From 9cada9526520f0b0ed0e3564dd95f9ce2a21dfb2 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:40:37 +0800 Subject: [PATCH 01/17] feat(dsv4): scaffold FP8 H100 dynamo-sglang multinode bring-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New SKU — no prior DeepSeek-V4-Pro data on H100. H100 (sm90) has no FP4 path, so this runs FP8 on the deepseek-v4-hopper image (shared with the H200 single-node dsv4 sglang recipe). ~671B FP8 weights don't fit 8xH100 (640GB), so prefill and decode workers each span 2 nodes at TP16 (matches dsr1-fp8-h100-dynamo-sglang); cross-node KV via NIXL. Conservative low-latency 1p1d STP topology (conc 1-64). Unvalidated on-cluster: needs the debug-runs loop for NIXL handshake + TP16 FP8 mem headroom before this leaves bring-up. pr-link intentionally TBD. 中文：搭建 H100 上 DeepSeek-V4-Pro FP8 分离式多节点 SGLang（Dynamo）脚手架 （H100 首次支持 dsv4，无原生 FP4 故走 FP8，复用 deepseek-v4-hopper 镜像； 671B 权重放不下 8xH100，故 prefill/decode 各跨 2 节点 TP16，跨节点 KV 用 NIXL）。尚未在集群验证，需 debug-runs 确认 NIXL 握手与 TP16 FP8 显存余量。 --- .../1k1k/disagg-h100-1p1d-tp16-tp16.yaml | 121 ++++++++++++++++++ configs/nvidia-master.yaml | 40 ++++++ perf-changelog.yaml | 8 ++ 3 files changed, 169 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml new file mode 100644 index 0000000000..301574e050 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml @@ -0,0 +1,121 @@ +name: "disagg-h100-1p1d-tp16-tp16" + +# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang, low-latency topology. +# +# BRING-UP scaffold (unvalidated on-cluster as of 2026-07). H100 (sm90) has +# no FP4 path, so this runs FP8 on the same deepseek-v4-hopper image the H200 +# single-node dsv4 sglang recipe uses. DeepSeek-V4-Pro FP8 weights (~671B) +# do not fit 8xH100 (640GB), so both prefill and decode workers span 2 nodes +# at TP16 (1280GB), matching the dsr1-fp8-h100-dynamo-sglang topology. Cross- +# node KV uses NIXL (no NVLink fabric across H100 nodes), unlike the GB200 +# mooncake/MNNVL recipes. +# +# Topology: 1 prefill (TP16, 2 nodes) + 1 decode (TP16, 2 nodes) = 4 nodes. +# STP only; a conservative low-concurrency first sweep. Server flags mirror +# the H200 dsv4 sglang recipe (marlin FP8 MoE, TP-only, radix cache off). +# +# Open questions for the debug-runs cluster loop before this leaves bring-up: +# 1. FP8 KV + marlin MoE headroom for TP16 on H100 (mem-fraction-static). +# 2. NIXL prefill/decode handshake on the h100-multinode pool. +# 3. Whether higher-throughput DEP topologies fit once low-latency is green. + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f" + precision: "fp8" + +dynamo: + install: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 1 + gpus_per_prefill: 16 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 16 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 16 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "marlin" + disable-flashinfer-autotune: true + kv-cache-dtype: "fp8_e4m3" + + mem-fraction-static: 0.85 + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 16 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "marlin" + disable-flashinfer-autotune: true + kv-cache-dtype: "fp8_e4m3" + + mem-fraction-static: 0.85 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.5 + context-length: 3072 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64" + req_rate: "inf" + use_chat_template: false diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f90c510186..9485116032 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4588,6 +4588,46 @@ dsr1-fp8-h100-dynamo-sglang: ep: 16 dp-attn: true +# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang via Dynamo. +# BRING-UP scaffold (unvalidated on-cluster as of 2026-07): no prior dsv4 +# data on H100. H100 (sm90) has no FP4 path, so this runs FP8 on the same +# deepseek-v4-hopper image the H200 single-node dsv4 sglang recipe uses. +# ~671B FP8 weights don't fit 8xH100 (640GB), so prefill and decode workers +# each span 2 nodes at TP16 (matches dsr1-fp8-h100-dynamo-sglang); cross-node +# KV uses NIXL (no NVLink fabric across H100 nodes). Conservative low-latency +# 1p1d STP topology only (conc 1-64); the debug-runs loop expands from here. +# Recipe: benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/. +dsv4-fp8-h100-dynamo-sglang: + image: lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: h100-multinode + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "0.8.0" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + # Low latency: 1 prefill (TP16, 2 nodes) + 1 decode (TP16, 2 nodes). + - conc-list: [1, 2, 4, 8, 16, 32, 64] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + minimaxm3-fp8-h200-vllm: image: vllm/vllm-openai:minimax-m3 model: MiniMaxAI/MiniMax-M3-MXFP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 57d510dd15..cfc3f1bd9e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4750,3 +4750,11 @@ - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 + +- config-keys: + - dsv4-fp8-h100-dynamo-sglang + description: + - "Add DeepSeek-V4-Pro FP8 H100 disaggregated multinode SGLang benchmark via Dynamo (new SKU, previously no dsv4 data on H100)" + - "H100 (sm90) has no FP4 path, so this runs FP8 on the deepseek-v4-hopper image (shared with the H200 single-node dsv4 sglang recipe); ~671B FP8 weights don't fit 8xH100 (640GB) so prefill and decode workers each span 2 nodes at TP16 (matches dsr1-fp8-h100-dynamo-sglang), cross-node KV via NIXL" + - "Conservative low-latency 1p1d STP topology (conc 1-64); recipe under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml" + pr-link: TBD From 8b28eecf517e06f6c36dc47e348b83b61d597d5f Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:41:29 +0800 Subject: [PATCH 02/17] refine(dsv4): extrapolate H100 dynamo-sglang topologies + 8k1k MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flesh out the H100 scaffold using same-SKU / same-model refs per recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro: - Add a throughput topology (1p1d TP16 prefill / DEP16 decode = tp16/ep16/dp-attn, DeepEP dispatch) alongside the low-latency TP16 pair, mirroring dsr1-fp8-h100-dynamo-sglang's max-throughput-TEP and max-throughput-DEP shapes. - Cover both 1k1k and 8k1k (4 recipes total). - Correct the FP8 sizing note (~960GB mixed -> ~1.05TB FP8). 中文：依据 recipes.vllm.ai 及同 SKU dsr1-h100、同型号 H200 参考，充实 H100 dynamo-sglang 拓扑：在低延迟 TP16 之外新增吞吐拓扑（DEP16 解码，DeepEP 跨节点分发），覆盖 1k1k 与 8k1k 共 4 个配方。仍需 debug-runs 集群验证。 --- .../1k1k/disagg-h100-1p1d-tp16-dep16.yaml | 119 ++++++++++++++++++ .../1k1k/disagg-h100-1p1d-tp16-tp16.yaml | 30 +++-- .../8k1k/disagg-h100-1p1d-tp16-dep16.yaml | 119 ++++++++++++++++++ .../8k1k/disagg-h100-1p1d-tp16-tp16.yaml | 119 ++++++++++++++++++ configs/nvidia-master.yaml | 64 ++++++++-- perf-changelog.yaml | 7 +- 6 files changed, 431 insertions(+), 27 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-dep16.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-dep16.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-tp16.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-dep16.yaml new file mode 100644 index 0000000000..7576e24cec --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-dep16.yaml @@ -0,0 +1,119 @@ +name: "disagg-h100-1p1d-tp16-dep16" + +# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang, throughput (TP16 prefill + DEP16 decode). +# +# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07), +# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus: +# * same SKU, different model: dsr1-fp8-h100-dynamo-sglang (TP16 2-node +# disagg topology, NIXL transfer, DEP decode tp16/ep16/dp-attn) +# * same model, adjacent Hopper SKU: the H200 dsv4 sglang recipe (FP8 +# marlin MoE, deepseek_v4 serving) — H100 (sm90) has no FP4 path, so the +# ~960GB mixed checkpoint is run in FP8. +# +# Sizing: ~1.05TB in FP8 does not fit 8xH100 (640GB), so each worker spans +# 2 nodes at TP16 (1280GB). Cross-node KV uses NIXL (no NVLink fabric across +# H100 nodes), unlike the GB200 mooncake/MNNVL recipes. +# +# Debug-runs must confirm before this leaves bring-up: NIXL prefill/decode +# handshake on h100-multinode, TP16 FP8 mem headroom and DeepEP cross-node dispatch on IB. + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f" + precision: "fp8" + +dynamo: + install: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 1 + gpus_per_prefill: 16 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 16 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 16 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "marlin" + disable-flashinfer-autotune: true + kv-cache-dtype: "fp8_e4m3" + + mem-fraction-static: 0.85 + max-running-requests: 512 + cuda-graph-max-bs: 128 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + + moe-a2a-backend: "deepep" + kv-cache-dtype: "fp8_e4m3" + + mem-fraction-static: 0.85 + max-running-requests: 8192 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.5 + context-length: 2304 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "64x128x256x512" + req_rate: "inf" + use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml index 301574e050..e84786fc32 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml @@ -1,23 +1,21 @@ name: "disagg-h100-1p1d-tp16-tp16" -# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang, low-latency topology. +# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang, low-latency (TP16 prefill + TP16 decode). # -# BRING-UP scaffold (unvalidated on-cluster as of 2026-07). H100 (sm90) has -# no FP4 path, so this runs FP8 on the same deepseek-v4-hopper image the H200 -# single-node dsv4 sglang recipe uses. DeepSeek-V4-Pro FP8 weights (~671B) -# do not fit 8xH100 (640GB), so both prefill and decode workers span 2 nodes -# at TP16 (1280GB), matching the dsr1-fp8-h100-dynamo-sglang topology. Cross- -# node KV uses NIXL (no NVLink fabric across H100 nodes), unlike the GB200 -# mooncake/MNNVL recipes. +# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07), +# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus: +# * same SKU, different model: dsr1-fp8-h100-dynamo-sglang (TP16 2-node +# disagg topology, NIXL transfer, prefill+decode TP16) +# * same model, adjacent Hopper SKU: the H200 dsv4 sglang recipe (FP8 +# marlin MoE, deepseek_v4 serving) — H100 (sm90) has no FP4 path, so the +# ~960GB mixed checkpoint is run in FP8. # -# Topology: 1 prefill (TP16, 2 nodes) + 1 decode (TP16, 2 nodes) = 4 nodes. -# STP only; a conservative low-concurrency first sweep. Server flags mirror -# the H200 dsv4 sglang recipe (marlin FP8 MoE, TP-only, radix cache off). +# Sizing: ~1.05TB in FP8 does not fit 8xH100 (640GB), so each worker spans +# 2 nodes at TP16 (1280GB). Cross-node KV uses NIXL (no NVLink fabric across +# H100 nodes), unlike the GB200 mooncake/MNNVL recipes. # -# Open questions for the debug-runs cluster loop before this leaves bring-up: -# 1. FP8 KV + marlin MoE headroom for TP16 on H100 (mem-fraction-static). -# 2. NIXL prefill/decode handshake on the h100-multinode pool. -# 3. Whether higher-throughput DEP topologies fit once low-latency is green. +# Debug-runs must confirm before this leaves bring-up: NIXL prefill/decode +# handshake on h100-multinode, TP16 FP8 mem headroom. model: path: "deepseek-v4-pro" @@ -110,7 +108,7 @@ backend: max-running-requests: 128 cuda-graph-max-bs: 128 swa-full-tokens-ratio: 0.5 - context-length: 3072 + context-length: 2304 benchmark: type: "sa-bench" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-dep16.yaml new file mode 100644 index 0000000000..967c40dd17 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-dep16.yaml @@ -0,0 +1,119 @@ +name: "disagg-h100-1p1d-tp16-dep16-8k" + +# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang, throughput (TP16 prefill + DEP16 decode). +# +# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07), +# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus: +# * same SKU, different model: dsr1-fp8-h100-dynamo-sglang (TP16 2-node +# disagg topology, NIXL transfer, DEP decode tp16/ep16/dp-attn) +# * same model, adjacent Hopper SKU: the H200 dsv4 sglang recipe (FP8 +# marlin MoE, deepseek_v4 serving) — H100 (sm90) has no FP4 path, so the +# ~960GB mixed checkpoint is run in FP8. +# +# Sizing: ~1.05TB in FP8 does not fit 8xH100 (640GB), so each worker spans +# 2 nodes at TP16 (1280GB). Cross-node KV uses NIXL (no NVLink fabric across +# H100 nodes), unlike the GB200 mooncake/MNNVL recipes. +# +# Debug-runs must confirm before this leaves bring-up: NIXL prefill/decode +# handshake on h100-multinode, TP16 FP8 mem headroom and DeepEP cross-node dispatch on IB. + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f" + precision: "fp8" + +dynamo: + install: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 1 + gpus_per_prefill: 16 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 16 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 16 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "marlin" + disable-flashinfer-autotune: true + kv-cache-dtype: "fp8_e4m3" + + mem-fraction-static: 0.85 + max-running-requests: 512 + cuda-graph-max-bs: 128 + chunked-prefill-size: 8192 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + + moe-a2a-backend: "deepep" + kv-cache-dtype: "fp8_e4m3" + + mem-fraction-static: 0.85 + max-running-requests: 8192 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.1 + context-length: 9472 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "64x128x256x512" + req_rate: "inf" + use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-tp16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-tp16.yaml new file mode 100644 index 0000000000..298e39f3d3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-tp16.yaml @@ -0,0 +1,119 @@ +name: "disagg-h100-1p1d-tp16-tp16-8k" + +# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang, low-latency (TP16 prefill + TP16 decode). +# +# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07), +# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus: +# * same SKU, different model: dsr1-fp8-h100-dynamo-sglang (TP16 2-node +# disagg topology, NIXL transfer, prefill+decode TP16) +# * same model, adjacent Hopper SKU: the H200 dsv4 sglang recipe (FP8 +# marlin MoE, deepseek_v4 serving) — H100 (sm90) has no FP4 path, so the +# ~960GB mixed checkpoint is run in FP8. +# +# Sizing: ~1.05TB in FP8 does not fit 8xH100 (640GB), so each worker spans +# 2 nodes at TP16 (1280GB). Cross-node KV uses NIXL (no NVLink fabric across +# H100 nodes), unlike the GB200 mooncake/MNNVL recipes. +# +# Debug-runs must confirm before this leaves bring-up: NIXL prefill/decode +# handshake on h100-multinode, TP16 FP8 mem headroom. + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f" + precision: "fp8" + +dynamo: + install: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 1 + gpus_per_prefill: 16 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 16 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 16 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "marlin" + disable-flashinfer-autotune: true + kv-cache-dtype: "fp8_e4m3" + + mem-fraction-static: 0.85 + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 8192 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 16 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "marlin" + disable-flashinfer-autotune: true + kv-cache-dtype: "fp8_e4m3" + + mem-fraction-static: 0.85 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + context-length: 9472 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64" + req_rate: "inf" + use_chat_template: false diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 9485116032..3b085de087 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4589,14 +4589,17 @@ dsr1-fp8-h100-dynamo-sglang: dp-attn: true # DeepSeek-V4-Pro FP8 H100 disaggregated SGLang via Dynamo. -# BRING-UP scaffold (unvalidated on-cluster as of 2026-07): no prior dsv4 -# data on H100. H100 (sm90) has no FP4 path, so this runs FP8 on the same -# deepseek-v4-hopper image the H200 single-node dsv4 sglang recipe uses. -# ~671B FP8 weights don't fit 8xH100 (640GB), so prefill and decode workers -# each span 2 nodes at TP16 (matches dsr1-fp8-h100-dynamo-sglang); cross-node -# KV uses NIXL (no NVLink fabric across H100 nodes). Conservative low-latency -# 1p1d STP topology only (conc 1-64); the debug-runs loop expands from here. -# Recipe: benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/. +# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07): no +# prior dsv4 data on H100. Derived per recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro +# plus the same-SKU dsr1-fp8-h100-dynamo-sglang (TP16 2-node disagg, NIXL, and +# its low-latency-TP + throughput-DEP topology pair) and the same-model H200 +# dsv4 sglang recipe (FP8 marlin serving). H100 (sm90) has no FP4 path, so the +# ~960GB mixed checkpoint runs in FP8 (~1.05TB), which doesn't fit 8xH100 +# (640GB) — so each worker spans 2 nodes at TP16 (1280GB); cross-node KV via +# NIXL. Two topologies per seq-len: low-latency (TP16 decode) and throughput +# (DEP16 decode = tp16/ep16/dp-attn, DeepEP dispatch), mirroring dsr1-h100's +# max-throughput-TEP and max-throughput-DEP shapes. +# Recipes: benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/{1k1k,8k1k}/disagg-h100-*. dsv4-fp8-h100-dynamo-sglang: image: lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f model: deepseek-ai/DeepSeek-V4-Pro @@ -4627,6 +4630,51 @@ dsv4-fp8-h100-dynamo-sglang: tp: 16 ep: 1 dp-attn: false + # Throughput: 1 prefill (TP16) + 1 decode (DEP16 = tp16/ep16/dp-attn). + - conc-list: [64, 128, 256, 512] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-dep16.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - isl: 8192 + osl: 1024 + search-space: + # Low latency: 1 prefill (TP16, 2 nodes) + 1 decode (TP16, 2 nodes). + - conc-list: [1, 2, 4, 8, 16, 32, 64] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-tp16.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + # Throughput: 1 prefill (TP16) + 1 decode (DEP16 = tp16/ep16/dp-attn). + - conc-list: [64, 128, 256, 512] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-dep16.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true minimaxm3-fp8-h200-vllm: image: vllm/vllm-openai:minimax-m3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cfc3f1bd9e..fcc0512d51 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4755,6 +4755,7 @@ - dsv4-fp8-h100-dynamo-sglang description: - "Add DeepSeek-V4-Pro FP8 H100 disaggregated multinode SGLang benchmark via Dynamo (new SKU, previously no dsv4 data on H100)" - - "H100 (sm90) has no FP4 path, so this runs FP8 on the deepseek-v4-hopper image (shared with the H200 single-node dsv4 sglang recipe); ~671B FP8 weights don't fit 8xH100 (640GB) so prefill and decode workers each span 2 nodes at TP16 (matches dsr1-fp8-h100-dynamo-sglang), cross-node KV via NIXL" - - "Conservative low-latency 1p1d STP topology (conc 1-64); recipe under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml" - pr-link: TBD + - "Extrapolated per recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro from the same-SKU dsr1-fp8-h100-dynamo-sglang (TP16 2-node disagg, NIXL, low-latency-TP + throughput-DEP topology pair) and the same-model H200 dsv4 sglang recipe (FP8 marlin serving)" + - "H100 (sm90) has no FP4 path, so the ~960GB mixed checkpoint runs in FP8 (~1.05TB), which doesn't fit 8xH100 (640GB); each worker spans 2 nodes at TP16, cross-node KV via NIXL" + - "Two topologies per seq-len (1k1k + 8k1k): low-latency 1p1d TP16/TP16 (conc 1-64) and throughput 1p1d TP16 prefill / DEP16 decode (tp16/ep16/dp-attn, DeepEP dispatch, conc 64-512); recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/{1k1k,8k1k}/disagg-h100-*.yaml" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 From 18eabcb6cda97d6a1b847e53303810e1d81c606c Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 19:04:56 +0800 Subject: [PATCH 03/17] fix(dsv4): route H100 dynamo-sglang to a runner label that exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dsv4-fp8-h100-dynamo-sglang used runner: h100-multinode, copied from the dsr1-fp8-h100-dynamo-* references. No registered runner carries the h100-multinode label, so run-sweep jobs (runs-on = the pool key verbatim) queued indefinitely (run 29326475700). Add an h100-disagg pool to runners.yaml (mirrors the mi300x-disagg / mi355x-disagg convention) mapping to the 3 runners that actually carry the h100-disagg label (h100-dgxc-slurm_17-19), and point the config at it. The launcher is selected from the runner name (h100-dgxc-slurm_NN -> launch_h100-dgxc-slurm.sh), so routing is unchanged. 中文：修复 H100 dynamo-sglang 的 runner 标签。原先用的 h100-multinode 无任何 runner 携带该标签，导致作业永久排队。新增 h100-disagg 资源池（对应真实携带 h100-disagg 标签的 3 台 runner），并将配置指向它。 --- configs/nvidia-master.yaml | 7 ++++++- configs/runners.yaml | 7 +++++++ perf-changelog.yaml | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3b085de087..939983532e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4604,7 +4604,12 @@ dsv4-fp8-h100-dynamo-sglang: image: lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: h100-multinode + # Runs-on label = this string verbatim. Must be a label a live runner + # carries: h100-disagg has 3 online runners (h100-dgxc-slurm_17-19) and is + # the disaggregated-H100 pool. NOT h100-multinode — no runner carries that + # label, so jobs requesting it queue forever (the dsr1-fp8-h100-dynamo-* + # reference configs above still use it and are non-functional for this). + runner: h100-disagg precision: fp8 framework: dynamo-sglang router: { name: dynamo-router, version: "0.8.0" } diff --git a/configs/runners.yaml b/configs/runners.yaml index 93da791b85..135c8fb9e8 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -27,6 +27,13 @@ labels: - h100-dgxc-slurm_17 - h100-dgxc-slurm_18 - h100-dgxc-slurm_19 + # Disaggregated-H100 pool. Unlike h100-multinode (whose key is not a label + # any runner carries, so jobs requesting it queue forever), these three + # runners carry the h100-disagg label, so runs-on: h100-disagg matches. + h100-disagg: + - h100-dgxc-slurm_17 + - h100-dgxc-slurm_18 + - h100-dgxc-slurm_19 h200: - h200-cw_00 - h200-cw_01 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d0a6eef405..d498d331d1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4758,6 +4758,7 @@ - "Extrapolated per recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro from the same-SKU dsr1-fp8-h100-dynamo-sglang (TP16 2-node disagg, NIXL, low-latency-TP + throughput-DEP topology pair) and the same-model H200 dsv4 sglang recipe (FP8 marlin serving)" - "H100 (sm90) has no FP4 path, so the ~960GB mixed checkpoint runs in FP8 (~1.05TB), which doesn't fit 8xH100 (640GB); each worker spans 2 nodes at TP16, cross-node KV via NIXL" - "Two topologies per seq-len (1k1k + 8k1k): low-latency 1p1d TP16/TP16 (conc 1-64) and throughput 1p1d TP16 prefill / DEP16 decode (tp16/ep16/dp-attn, DeepEP dispatch, conc 64-512); recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/{1k1k,8k1k}/disagg-h100-*.yaml" + - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 - config-keys: From 190057f886fb9f69f19e9fb80d66c993414c5f0a Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 19:38:38 +0800 Subject: [PATCH 04/17] refactor(dsv4): follow vLLM H100 recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the SGLang scaffold with a vLLM TEP16/TEP16 deployment, including the H100 launcher, DeepGEMM setup, NIXL recipe wiring, and append-only changelog update. 中文：将 SGLang 脚手架改为遵循 vLLM 配方的 TEP16/TEP16 部署，并补齐 H100 启动器、DeepGEMM 安装、NIXL 配方接入及追加式更新日志。 --- .../configs/dsv4-h100-vllm-deps.sh | 25 ++++ .../1k1k/disagg-h100-1p1d-tp16-dep16.yaml | 119 ----------------- .../1k1k/disagg-h100-1p1d-tp16-tp16.yaml | 119 ----------------- .../8k1k/disagg-h100-1p1d-tp16-dep16.yaml | 119 ----------------- .../8k1k/disagg-h100-1p1d-tp16-tp16.yaml | 119 ----------------- .../1k1k/disagg-h100-1p1d-tep16-tep16.yaml | 123 ++++++++++++++++++ .../8k1k/disagg-h100-1p1d-tep16-tep16.yaml | 123 ++++++++++++++++++ configs/nvidia-master.yaml | 77 ++++------- perf-changelog.yaml | 21 +-- runners/launch_h100-dgxc-slurm.sh | 59 ++++++++- 10 files changed, 361 insertions(+), 543 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-h100-vllm-deps.sh delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-dep16.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-dep16.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-tp16.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-h100-vllm-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-h100-vllm-deps.sh new file mode 100644 index 0000000000..56a2f40b3e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-h100-vllm-deps.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Container setup required by the upstream vLLM DeepSeek-V4-Pro recipe. + +set -euo pipefail + +apt-get -y update +apt-get install -y --no-install-recommends --allow-change-held-packages \ + curl \ + numactl +pip install msgpack + +# vLLM's recipe calls out DeepGEMM as a required extra. Prefer the installer +# shipped in the image so it matches the installed vLLM revision; retain the +# upstream recipe command as a fallback for images without the source tree. +if ! python3 -c 'import deep_gemm' >/dev/null 2>&1; then + if [[ -f /vllm-workspace/tools/install_deepgemm.sh ]]; then + bash /vllm-workspace/tools/install_deepgemm.sh + else + bash <(curl -fsSL https://raw.githubusercontent.com/vllm-project/vllm/v0.21.0/tools/install_deepgemm.sh) + fi +fi + +if [[ -f /configs/patches/vllm_numa_bind_hash_fix.py ]]; then + python3 /configs/patches/vllm_numa_bind_hash_fix.py +fi diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-dep16.yaml deleted file mode 100644 index 7576e24cec..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-dep16.yaml +++ /dev/null @@ -1,119 +0,0 @@ -name: "disagg-h100-1p1d-tp16-dep16" - -# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang, throughput (TP16 prefill + DEP16 decode). -# -# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07), -# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus: -# * same SKU, different model: dsr1-fp8-h100-dynamo-sglang (TP16 2-node -# disagg topology, NIXL transfer, DEP decode tp16/ep16/dp-attn) -# * same model, adjacent Hopper SKU: the H200 dsv4 sglang recipe (FP8 -# marlin MoE, deepseek_v4 serving) — H100 (sm90) has no FP4 path, so the -# ~960GB mixed checkpoint is run in FP8. -# -# Sizing: ~1.05TB in FP8 does not fit 8xH100 (640GB), so each worker spans -# 2 nodes at TP16 (1280GB). Cross-node KV uses NIXL (no NVLink fabric across -# H100 nodes), unlike the GB200 mooncake/MNNVL recipes. -# -# Debug-runs must confirm before this leaves bring-up: NIXL prefill/decode -# handshake on h100-multinode, TP16 FP8 mem headroom and DeepEP cross-node dispatch on IB. - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f" - precision: "fp8" - -dynamo: - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "h100" - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 16 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 16 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "marlin" - disable-flashinfer-autotune: true - kv-cache-dtype: "fp8_e4m3" - - mem-fraction-static: 0.85 - max-running-requests: 512 - cuda-graph-max-bs: 128 - chunked-prefill-size: 4096 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - - moe-a2a-backend: "deepep" - kv-cache-dtype: "fp8_e4m3" - - mem-fraction-static: 0.85 - max-running-requests: 8192 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.5 - context-length: 2304 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "64x128x256x512" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml deleted file mode 100644 index e84786fc32..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml +++ /dev/null @@ -1,119 +0,0 @@ -name: "disagg-h100-1p1d-tp16-tp16" - -# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang, low-latency (TP16 prefill + TP16 decode). -# -# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07), -# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus: -# * same SKU, different model: dsr1-fp8-h100-dynamo-sglang (TP16 2-node -# disagg topology, NIXL transfer, prefill+decode TP16) -# * same model, adjacent Hopper SKU: the H200 dsv4 sglang recipe (FP8 -# marlin MoE, deepseek_v4 serving) — H100 (sm90) has no FP4 path, so the -# ~960GB mixed checkpoint is run in FP8. -# -# Sizing: ~1.05TB in FP8 does not fit 8xH100 (640GB), so each worker spans -# 2 nodes at TP16 (1280GB). Cross-node KV uses NIXL (no NVLink fabric across -# H100 nodes), unlike the GB200 mooncake/MNNVL recipes. -# -# Debug-runs must confirm before this leaves bring-up: NIXL prefill/decode -# handshake on h100-multinode, TP16 FP8 mem headroom. - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f" - precision: "fp8" - -dynamo: - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "h100" - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 16 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 16 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "marlin" - disable-flashinfer-autotune: true - kv-cache-dtype: "fp8_e4m3" - - mem-fraction-static: 0.85 - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 4096 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 16 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "marlin" - disable-flashinfer-autotune: true - kv-cache-dtype: "fp8_e4m3" - - mem-fraction-static: 0.85 - max-running-requests: 128 - cuda-graph-max-bs: 128 - swa-full-tokens-ratio: 0.5 - context-length: 2304 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1x2x4x8x16x32x64" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-dep16.yaml deleted file mode 100644 index 967c40dd17..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-dep16.yaml +++ /dev/null @@ -1,119 +0,0 @@ -name: "disagg-h100-1p1d-tp16-dep16-8k" - -# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang, throughput (TP16 prefill + DEP16 decode). -# -# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07), -# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus: -# * same SKU, different model: dsr1-fp8-h100-dynamo-sglang (TP16 2-node -# disagg topology, NIXL transfer, DEP decode tp16/ep16/dp-attn) -# * same model, adjacent Hopper SKU: the H200 dsv4 sglang recipe (FP8 -# marlin MoE, deepseek_v4 serving) — H100 (sm90) has no FP4 path, so the -# ~960GB mixed checkpoint is run in FP8. -# -# Sizing: ~1.05TB in FP8 does not fit 8xH100 (640GB), so each worker spans -# 2 nodes at TP16 (1280GB). Cross-node KV uses NIXL (no NVLink fabric across -# H100 nodes), unlike the GB200 mooncake/MNNVL recipes. -# -# Debug-runs must confirm before this leaves bring-up: NIXL prefill/decode -# handshake on h100-multinode, TP16 FP8 mem headroom and DeepEP cross-node dispatch on IB. - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f" - precision: "fp8" - -dynamo: - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "h100" - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 16 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 16 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "marlin" - disable-flashinfer-autotune: true - kv-cache-dtype: "fp8_e4m3" - - mem-fraction-static: 0.85 - max-running-requests: 512 - cuda-graph-max-bs: 128 - chunked-prefill-size: 8192 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - - moe-a2a-backend: "deepep" - kv-cache-dtype: "fp8_e4m3" - - mem-fraction-static: 0.85 - max-running-requests: 8192 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.1 - context-length: 9472 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "64x128x256x512" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-tp16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-tp16.yaml deleted file mode 100644 index 298e39f3d3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-tp16.yaml +++ /dev/null @@ -1,119 +0,0 @@ -name: "disagg-h100-1p1d-tp16-tp16-8k" - -# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang, low-latency (TP16 prefill + TP16 decode). -# -# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07), -# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus: -# * same SKU, different model: dsr1-fp8-h100-dynamo-sglang (TP16 2-node -# disagg topology, NIXL transfer, prefill+decode TP16) -# * same model, adjacent Hopper SKU: the H200 dsv4 sglang recipe (FP8 -# marlin MoE, deepseek_v4 serving) — H100 (sm90) has no FP4 path, so the -# ~960GB mixed checkpoint is run in FP8. -# -# Sizing: ~1.05TB in FP8 does not fit 8xH100 (640GB), so each worker spans -# 2 nodes at TP16 (1280GB). Cross-node KV uses NIXL (no NVLink fabric across -# H100 nodes), unlike the GB200 mooncake/MNNVL recipes. -# -# Debug-runs must confirm before this leaves bring-up: NIXL prefill/decode -# handshake on h100-multinode, TP16 FP8 mem headroom. - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f" - precision: "fp8" - -dynamo: - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "h100" - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 16 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 16 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "marlin" - disable-flashinfer-autotune: true - kv-cache-dtype: "fp8_e4m3" - - mem-fraction-static: 0.85 - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 8192 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 16 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "marlin" - disable-flashinfer-autotune: true - kv-cache-dtype: "fp8_e4m3" - - mem-fraction-static: 0.85 - max-running-requests: 128 - cuda-graph-max-bs: 128 - swa-full-tokens-ratio: 0.1 - context-length: 9472 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1x2x4x8x16x32x64" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml new file mode 100644 index 0000000000..9d1ad7b628 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml @@ -0,0 +1,123 @@ +name: "dsv4-vllm-disagg-h100-1p1d-tep16-tep16-1k1k" + +# H100 adaptation of the upstream vLLM DeepSeek-V4-Pro recipe. The native +# mixed FP4+FP8 checkpoint needs about 960GB, so every prefill/decode replica +# spans two 8xH100 nodes (1280GB) as TP16 with expert parallelism. The 1P1D +# deployment therefore uses four GPU nodes total. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp8" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: dsv4-h100-vllm-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 1 + gpus_per_prefill: 16 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + prefill_environment: + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + decode_environment: + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + vllm_config: + prefill: + block-size: 256 + compilation-config: '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","pass_config":{"fuse_allreduce_rms":false}}' + enable-expert-parallel: true + enable-sleep-mode: true + enforce-eager: true + gpu-memory-utilization: 0.95 + kv-cache-dtype: "fp8" + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + max-model-len: 200000 + max-num-batched-tokens: 16384 + max-num-seqs: 16 + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + pipeline-parallel-size: 1 + reasoning-parser: deepseek_v4 + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + tensor-parallel-size: 16 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + decode: + block-size: 256 + compilation-config: '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","pass_config":{"fuse_allreduce_rms":false}}' + enable-expert-parallel: true + enable-sleep-mode: true + gpu-memory-utilization: 0.95 + kv-cache-dtype: "fp8" + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + max-model-len: 200000 + max-num-batched-tokens: 16384 + max-num-seqs: 16 + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + pipeline-parallel-size: 1 + reasoning-parser: deepseek_v4 + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + tensor-parallel-size: 16 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.21.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml new file mode 100644 index 0000000000..949117072f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml @@ -0,0 +1,123 @@ +name: "dsv4-vllm-disagg-h100-1p1d-tep16-tep16-8k1k" + +# H100 adaptation of the upstream vLLM DeepSeek-V4-Pro recipe. The native +# mixed FP4+FP8 checkpoint needs about 960GB, so every prefill/decode replica +# spans two 8xH100 nodes (1280GB) as TP16 with expert parallelism. The 1P1D +# deployment therefore uses four GPU nodes total. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp8" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: dsv4-h100-vllm-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 1 + gpus_per_prefill: 16 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + prefill_environment: + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + decode_environment: + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + vllm_config: + prefill: + block-size: 256 + compilation-config: '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","pass_config":{"fuse_allreduce_rms":false}}' + enable-expert-parallel: true + enable-sleep-mode: true + enforce-eager: true + gpu-memory-utilization: 0.95 + kv-cache-dtype: "fp8" + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + max-model-len: 200000 + max-num-batched-tokens: 16384 + max-num-seqs: 16 + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + pipeline-parallel-size: 1 + reasoning-parser: deepseek_v4 + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + tensor-parallel-size: 16 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + decode: + block-size: 256 + compilation-config: '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","pass_config":{"fuse_allreduce_rms":false}}' + enable-expert-parallel: true + enable-sleep-mode: true + gpu-memory-utilization: 0.95 + kv-cache-dtype: "fp8" + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + max-model-len: 200000 + max-num-batched-tokens: 16384 + max-num-seqs: 16 + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + pipeline-parallel-size: 1 + reasoning-parser: deepseek_v4 + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + tensor-parallel-size: 16 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.21.0" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 939983532e..58431bf47f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4588,20 +4588,19 @@ dsr1-fp8-h100-dynamo-sglang: ep: 16 dp-attn: true -# DeepSeek-V4-Pro FP8 H100 disaggregated SGLang via Dynamo. +# DeepSeek-V4-Pro FP8 H100 disaggregated vLLM via Dynamo. # EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07): no -# prior dsv4 data on H100. Derived per recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro -# plus the same-SKU dsr1-fp8-h100-dynamo-sglang (TP16 2-node disagg, NIXL, and -# its low-latency-TP + throughput-DEP topology pair) and the same-model H200 -# dsv4 sglang recipe (FP8 marlin serving). H100 (sm90) has no FP4 path, so the -# ~960GB mixed checkpoint runs in FP8 (~1.05TB), which doesn't fit 8xH100 -# (640GB) — so each worker spans 2 nodes at TP16 (1280GB); cross-node KV via -# NIXL. Two topologies per seq-len: low-latency (TP16 decode) and throughput -# (DEP16 decode = tp16/ep16/dp-attn, DeepEP dispatch), mirroring dsr1-h100's -# max-throughput-TEP and max-throughput-DEP shapes. -# Recipes: benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/{1k1k,8k1k}/disagg-h100-*. -dsv4-fp8-h100-dynamo-sglang: - image: lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f +# prior dsv4 data on H100. Follows the vLLM DeepSeek-V4-Pro recipe (vLLM +# 0.20.0+, FP8 KV cache, block size 256, expert parallelism, Hopper memory +# settings, and NIXL P/D transfer). The native checkpoint is mixed FP4+FP8 and +# requires about 960GB VRAM, so it cannot fit one 8xH100 node (640GB). Each +# prefill/decode worker therefore spans 2 nodes at TP16+EP (TEP16, 1280GB), +# avoiding the dense-weight replication that makes DEP unsuitable on H100. +# A 1P1D run consumes 4 H100 nodes total. H100 is not yet verified upstream, +# so the two-node-per-worker topology remains a bring-up target. +# Recipes: benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep16-tep16.yaml. +dsv4-fp8-h100-dynamo-vllm: + image: vllm/vllm-openai:v0.21.0 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 # Runs-on label = this string verbatim. Must be a label a live runner @@ -4611,8 +4610,8 @@ dsv4-fp8-h100-dynamo-sglang: # reference configs above still use it and are non-functional for this). runner: h100-disagg precision: fp8 - framework: dynamo-sglang - router: { name: dynamo-router, version: "0.8.0" } + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -4621,65 +4620,37 @@ dsv4-fp8-h100-dynamo-sglang: - isl: 1024 osl: 1024 search-space: - # Low latency: 1 prefill (TP16, 2 nodes) + 1 decode (TP16, 2 nodes). - - conc-list: [1, 2, 4, 8, 16, 32, 64] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-tp16.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - # Throughput: 1 prefill (TP16) + 1 decode (DEP16 = tp16/ep16/dp-attn). - - conc-list: [64, 128, 256, 512] + # Each TEP16 worker spans two H100 nodes; 1P1D uses four GPU nodes. + - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] prefill: num-worker: 1 tp: 16 - ep: 1 + ep: 16 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-h100-1p1d-tp16-dep16.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml" decode: num-worker: 1 tp: 16 ep: 16 - dp-attn: true + dp-attn: false - isl: 8192 osl: 1024 search-space: - # Low latency: 1 prefill (TP16, 2 nodes) + 1 decode (TP16, 2 nodes). - - conc-list: [1, 2, 4, 8, 16, 32, 64] + # Each TEP16 worker spans two H100 nodes; 1P1D uses four GPU nodes. + - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] prefill: num-worker: 1 tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-tp16.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - # Throughput: 1 prefill (TP16) + 1 decode (DEP16 = tp16/ep16/dp-attn). - - conc-list: [64, 128, 256, 512] - prefill: - num-worker: 1 - tp: 16 - ep: 1 + ep: 16 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-h100-1p1d-tp16-dep16.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml" decode: num-worker: 1 tp: 16 ep: 16 - dp-attn: true + dp-attn: false minimaxm3-fp8-h200-vllm: image: vllm/vllm-openai:minimax-m3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d498d331d1..7438fd483e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4751,16 +4751,6 @@ - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 -- config-keys: - - dsv4-fp8-h100-dynamo-sglang - description: - - "Add DeepSeek-V4-Pro FP8 H100 disaggregated multinode SGLang benchmark via Dynamo (new SKU, previously no dsv4 data on H100)" - - "Extrapolated per recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro from the same-SKU dsr1-fp8-h100-dynamo-sglang (TP16 2-node disagg, NIXL, low-latency-TP + throughput-DEP topology pair) and the same-model H200 dsv4 sglang recipe (FP8 marlin serving)" - - "H100 (sm90) has no FP4 path, so the ~960GB mixed checkpoint runs in FP8 (~1.05TB), which doesn't fit 8xH100 (640GB); each worker spans 2 nodes at TP16, cross-node KV via NIXL" - - "Two topologies per seq-len (1k1k + 8k1k): low-latency 1p1d TP16/TP16 (conc 1-64) and throughput 1p1d TP16 prefill / DEP16 decode (tp16/ep16/dp-attn, DeepEP dispatch, conc 64-512); recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/{1k1k,8k1k}/disagg-h100-*.yaml" - - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 - - config-keys: - dsv4-fp4-mi355x-sglang description: @@ -4775,3 +4765,14 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - dsv4-fp8-h100-dynamo-vllm + description: + - "Add DeepSeek-V4-Pro FP8 H100 disaggregated multinode vLLM benchmark via Dynamo (new SKU, previously no dsv4 data on H100)" + - "Follow recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro: vLLM 0.20.0+, native mixed FP4+FP8 checkpoint, FP8 KV cache, block size 256, expert parallelism, Hopper memory settings, and NIXL P/D transfer" + - "The checkpoint requires about 960GB VRAM and does not fit 8xH100 (640GB); each prefill/decode worker spans 2 nodes as TEP16 (TP16 with expert parallelism), so 1P1D uses 4 H100 nodes total" + - "One TEP16/TEP16 topology per seq-len (1k1k + 8k1k), conc 1-512; DEP16 is intentionally omitted because its replicated dense layers cannot fit H100 memory" + - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep16-tep16.yaml; H100 remains an unverified upstream bring-up target" + - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 1334c95542..906fea9cfb 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -16,7 +16,29 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # MODEL_PATH: Override with pre-downloaded paths on H100 runner # The yaml files specify HuggingFace model IDs for portability, but we use # local paths to avoid repeated downloading on the shared H100 cluster. - if [[ $FRAMEWORK == "dynamo-sglang" ]]; then + if [[ $FRAMEWORK == "dynamo-vllm" ]]; then + if [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp8" ]]; then + SELECTED_MODEL_PATH="" + if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then + SELECTED_MODEL_PATH="$MODEL_PATH" + else + for candidate in \ + /mnt/nfs/lustre/models/deepseek-v4-pro \ + /mnt/nfs/lustre/models/dsv4-pro \ + /mnt/nfs/lustre/models/DeepSeek-V4-Pro; do + if [[ -d "$candidate" ]]; then + SELECTED_MODEL_PATH="$candidate" + break + fi + done + fi + export MODEL_PATH="${SELECTED_MODEL_PATH:-/mnt/nfs/lustre/models/deepseek-v4-pro}" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" + else + echo "Unsupported model prefix/precision for dynamo-vllm: $MODEL_PREFIX/$PRECISION" + exit 1 + fi + elif [[ $FRAMEWORK == "dynamo-sglang" ]]; then if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/mnt/nfs/lustre/models/dsr1-fp8" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" @@ -34,7 +56,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then exit 1 fi else - echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang" + echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang, dynamo-vllm" exit 1 fi @@ -49,6 +71,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" + elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout aflowers/vllm-gb200-v0.20.0 + mkdir -p recipes/vllm/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 + cp \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-h100-vllm-deps.sh" \ + configs/patches/dsv4-h100-vllm-deps.sh else git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" @@ -84,6 +115,23 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # SGLang container mapping SQUASH_FILE="/mnt/nfs/lustre/containers/lmsysorg_sglang_v0.5.8.post1-cu130.sqsh" CONTAINER_KEY="lmsysorg/sglang:v0.5.8-cu130" + elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then + # vLLM container mapping. Import the recipe image once into the shared + # squash directory so every Slurm node uses the exact configured tag. + CONTAINER_KEY="$IMAGE" + SQUASH_FILE="/mnt/nfs/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/+/g').sqsh" + mkdir -p "$(dirname "$SQUASH_FILE")" + ( + exec 9>"${SQUASH_FILE}.lock" + flock -w 1800 9 || { echo "Failed to acquire lock for $SQUASH_FILE" >&2; exit 1; } + if unsquashfs -l "$SQUASH_FILE" >/dev/null 2>&1; then + echo "Squash file already exists and is valid: $SQUASH_FILE" + else + rm -f "$SQUASH_FILE" "${SQUASH_FILE}.tmp."* + enroot import -o "${SQUASH_FILE}.tmp.$$" "docker://$IMAGE" + mv -f "${SQUASH_FILE}.tmp.$$" "$SQUASH_FILE" + fi + ) elif [[ $FRAMEWORK == "dynamo-trt" ]]; then # TRT-LLM container mapping - convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) CONTAINER_KEY=$(echo "$IMAGE" | sed 's|nvcr.io/|nvcr.io#|') @@ -115,6 +163,7 @@ model_paths: containers: dynamo-trtllm: "${SQUASH_FILE}" dynamo-sglang: "${SQUASH_FILE}" + dynamo-vllm: "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" latest: "${SQUASH_FILE}" "${CONTAINER_KEY}": "${SQUASH_FILE}" @@ -143,8 +192,10 @@ EOF # Override the job name in the config file with the runner name sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE" - # Raise sglang's torch-distributed TCPStore timeout from the 600s gloo default - sed -i '/^ watchdog-timeout:/a\ dist-timeout: 1800' "${CONFIG_FILE%%:*}" + # Raise sglang's torch-distributed TCPStore timeout from the 600s gloo default. + if [[ $FRAMEWORK == "dynamo-sglang" ]]; then + sed -i '/^ watchdog-timeout:/a\ dist-timeout: 1800' "${CONFIG_FILE%%:*}" + fi SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h100,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT" From b04ebb701308275a1794ac24c87e16d95e046f46 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 19:54:42 +0800 Subject: [PATCH 05/17] fix(h100): install make before srt-slurm setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Install GNU make only when the H100 login-runner image does not provide it, then continue through the same upstream make setup ARCH=x86_64 flow used by the other srt-slurm launchers.\n\n中文：仅在 H100 登录 runner 镜像未提供 GNU make 时安装该依赖，随后继续使用与其他 srt-slurm 启动器一致的上游 make setup ARCH=x86_64 流程。 --- perf-changelog.yaml | 1 + runners/launch_h100-dgxc-slurm.sh | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7438fd483e..5a4fb42c23 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4775,4 +4775,5 @@ - "One TEP16/TEP16 topology per seq-len (1k1k + 8k1k), conc 1-512; DEP16 is intentionally omitted because its replicated dense layers cannot fit H100 memory" - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep16-tep16.yaml; H100 remains an unverified upstream bring-up target" - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" + - "Install GNU make when absent from the H100 login-runner image, then use the same upstream make setup ARCH=x86_64 flow as the other srt-slurm launchers" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 906fea9cfb..ec216276eb 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -176,6 +176,22 @@ EOF echo "Generated srtslurm.yaml:" cat srtslurm.yaml + # The other srt-slurm launchers rely on GNU make being provided by their + # runner image. The H100 login-runner image currently omits it, so install + # that same prerequisite before invoking the standard upstream target. + if ! command -v make >/dev/null 2>&1; then + if [[ "$(id -u)" -eq 0 ]]; then + apt-get update + apt-get install -y --no-install-recommends make + elif command -v sudo >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends make + else + echo "Error: GNU make is required by srt-slurm, but this runner cannot install it" >&2 + exit 1 + fi + fi + echo "Running make setup..." make setup ARCH=x86_64 From 6a4d35151d1500f2115d39fb11c651117fd5900e Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:27:07 +0800 Subject: [PATCH 06/17] revert(h100): avoid host package installation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep the H100 launcher aligned with the other srt-slurm launchers by invoking make setup directly and leaving GNU Make provisioning to the runner image.\n\n中文：让 H100 启动器与其他 srt-slurm 启动器保持一致，直接调用 make setup，并由 runner 镜像负责提供 GNU Make，不在宿主机安装软件包。 --- perf-changelog.yaml | 1 - runners/launch_h100-dgxc-slurm.sh | 16 ---------------- 2 files changed, 17 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5a4fb42c23..7438fd483e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4775,5 +4775,4 @@ - "One TEP16/TEP16 topology per seq-len (1k1k + 8k1k), conc 1-512; DEP16 is intentionally omitted because its replicated dense layers cannot fit H100 memory" - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep16-tep16.yaml; H100 remains an unverified upstream bring-up target" - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" - - "Install GNU make when absent from the H100 login-runner image, then use the same upstream make setup ARCH=x86_64 flow as the other srt-slurm launchers" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index ec216276eb..906fea9cfb 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -176,22 +176,6 @@ EOF echo "Generated srtslurm.yaml:" cat srtslurm.yaml - # The other srt-slurm launchers rely on GNU make being provided by their - # runner image. The H100 login-runner image currently omits it, so install - # that same prerequisite before invoking the standard upstream target. - if ! command -v make >/dev/null 2>&1; then - if [[ "$(id -u)" -eq 0 ]]; then - apt-get update - apt-get install -y --no-install-recommends make - elif command -v sudo >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install -y --no-install-recommends make - else - echo "Error: GNU make is required by srt-slurm, but this runner cannot install it" >&2 - exit 1 - fi - fi - echo "Running make setup..." make setup ARCH=x86_64 From edbaf8d5c9a1ff228e914704b459eeb1cf0fd867 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:38:54 +0800 Subject: [PATCH 07/17] fix(dsv4): use H100 checkpoint and healthy nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the existing dsv4-fp8 checkpoint on the H100 cluster and exclude nodes without healthy pod-network RDMA, including the node with mismatched mlx5 numbering.\n\n中文：使用 H100 集群现有的 dsv4-fp8 检查点，并排除 pod 网络 RDMA 异常的节点以及 mlx5 设备编号不一致的节点。 --- .../vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml | 1 + .../vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml | 1 + perf-changelog.yaml | 1 + runners/launch_h100-dgxc-slurm.sh | 3 ++- 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml index 9d1ad7b628..151c6bab72 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml @@ -25,6 +25,7 @@ health_check: sbatch_directives: cpus-per-task: "144" mem: "0" + exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" resources: gpu_type: "h100" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml index 949117072f..2fd1b11972 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml @@ -25,6 +25,7 @@ health_check: sbatch_directives: cpus-per-task: "144" mem: "0" + exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" resources: gpu_type: "h100" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7438fd483e..9b50a3cc5c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4775,4 +4775,5 @@ - "One TEP16/TEP16 topology per seq-len (1k1k + 8k1k), conc 1-512; DEP16 is intentionally omitted because its replicated dense layers cannot fit H100 memory" - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep16-tep16.yaml; H100 remains an unverified upstream bring-up target" - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" + - "Use the existing /mnt/nfs/lustre/models/dsv4-fp8 checkpoint and exclude H100 nodes without healthy pod-network RDMA plus hpc-gpu-1-16, whose mlx5 device numbering differs from the fleet" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 906fea9cfb..7851d6186d 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -23,6 +23,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then SELECTED_MODEL_PATH="$MODEL_PATH" else for candidate in \ + /mnt/nfs/lustre/models/dsv4-fp8 \ /mnt/nfs/lustre/models/deepseek-v4-pro \ /mnt/nfs/lustre/models/dsv4-pro \ /mnt/nfs/lustre/models/DeepSeek-V4-Pro; do @@ -32,7 +33,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then fi done fi - export MODEL_PATH="${SELECTED_MODEL_PATH:-/mnt/nfs/lustre/models/deepseek-v4-pro}" + export MODEL_PATH="${SELECTED_MODEL_PATH:-/mnt/nfs/lustre/models/dsv4-fp8}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" else echo "Unsupported model prefix/precision for dynamo-vllm: $MODEL_PREFIX/$PRECISION" From 94f1d11f2b942acb5237026e7feec5920f525d30 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:47:21 +0800 Subject: [PATCH 08/17] fix(dsv4): use standard vLLM container setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the custom DeepGEMM bootstrap with NVIDIA srt-slurm's standard vllm-container-deps.sh and wire both H100 recipes to it.\n\n中文：将自定义 DeepGEMM 引导脚本替换为 NVIDIA srt-slurm 标准的 vllm-container-deps.sh，并让两套 H100 配方统一使用该脚本。 --- .../configs/dsv4-h100-vllm-deps.sh | 25 ------------------- .../configs/vllm-container-deps.sh | 13 ++++++++++ .../1k1k/disagg-h100-1p1d-tep16-tep16.yaml | 2 +- .../8k1k/disagg-h100-1p1d-tep16-tep16.yaml | 2 +- perf-changelog.yaml | 1 + runners/launch_h100-dgxc-slurm.sh | 4 +-- 6 files changed, 18 insertions(+), 29 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-h100-vllm-deps.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-h100-vllm-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-h100-vllm-deps.sh deleted file mode 100644 index 56a2f40b3e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-h100-vllm-deps.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Container setup required by the upstream vLLM DeepSeek-V4-Pro recipe. - -set -euo pipefail - -apt-get -y update -apt-get install -y --no-install-recommends --allow-change-held-packages \ - curl \ - numactl -pip install msgpack - -# vLLM's recipe calls out DeepGEMM as a required extra. Prefer the installer -# shipped in the image so it matches the installed vLLM revision; retain the -# upstream recipe command as a fallback for images without the source tree. -if ! python3 -c 'import deep_gemm' >/dev/null 2>&1; then - if [[ -f /vllm-workspace/tools/install_deepgemm.sh ]]; then - bash /vllm-workspace/tools/install_deepgemm.sh - else - bash <(curl -fsSL https://raw.githubusercontent.com/vllm-project/vllm/v0.21.0/tools/install_deepgemm.sh) - fi -fi - -if [[ -f /configs/patches/vllm_numa_bind_hash_fix.py ]]; then - python3 /configs/patches/vllm_numa_bind_hash_fix.py -fi diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh new file mode 100644 index 0000000000..d11740d596 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +apt-get -y update && apt-get install -y --no-install-recommends --allow-change-held-packages numactl + +pip install msgpack + +if [ -f /configs/patches/vllm_numa_bind_hash_fix.py ]; then + python3 /configs/patches/vllm_numa_bind_hash_fix.py +fi diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml index 151c6bab72..249b8c690e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml @@ -13,7 +13,7 @@ dynamo: install: true wheel: "1.2.0.dev20260426" -setup_script: dsv4-h100-vllm-deps.sh +setup_script: vllm-container-deps.sh slurm: time_limit: "8:00:00" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml index 2fd1b11972..3079650c7c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml @@ -13,7 +13,7 @@ dynamo: install: true wheel: "1.2.0.dev20260426" -setup_script: dsv4-h100-vllm-deps.sh +setup_script: vllm-container-deps.sh slurm: time_limit: "8:00:00" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9b50a3cc5c..a99aaef166 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4776,4 +4776,5 @@ - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep16-tep16.yaml; H100 remains an unverified upstream bring-up target" - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" - "Use the existing /mnt/nfs/lustre/models/dsv4-fp8 checkpoint and exclude H100 nodes without healthy pod-network RDMA plus hpc-gpu-1-16, whose mlx5 device numbering differs from the fleet" + - "Use NVIDIA srt-slurm's standard vllm-container-deps.sh setup instead of a custom DeepGEMM bootstrap" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 7851d6186d..29aeab9330 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -79,8 +79,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 cp \ - "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-h100-vllm-deps.sh" \ - configs/patches/dsv4-h100-vllm-deps.sh + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh" \ + configs/patches/vllm-container-deps.sh else git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" From f0a3df204a50e58c87fd44e9316ab949afb3ad9b Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:54:50 +0800 Subject: [PATCH 09/17] fix(dsv4): use TP8 PP2 on H100 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split each two-node worker as TP8 x PP2 with EP8 so FP8 weight partitions remain divisible by the checkpoint's 128-column quantization blocks while retaining 16 GPUs of memory.\n\n中文：将每个双节点 worker 调整为 TP8 x PP2 并使用 EP8，在保留 16 张 GPU 显存容量的同时，确保 FP8 权重分片可被检查点的 128 列量化块整除。 --- ... => disagg-h100-1p1d-tep8pp2-tep8pp2.yaml} | 12 +++---- ... => disagg-h100-1p1d-tep8pp2-tep8pp2.yaml} | 12 +++---- configs/nvidia-master.yaml | 32 +++++++++++-------- perf-changelog.yaml | 6 ++-- 4 files changed, 33 insertions(+), 29 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/{disagg-h100-1p1d-tep16-tep16.yaml => disagg-h100-1p1d-tep8pp2-tep8pp2.yaml} (92%) rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/{disagg-h100-1p1d-tep16-tep16.yaml => disagg-h100-1p1d-tep8pp2-tep8pp2.yaml} (92%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml similarity index 92% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml index 249b8c690e..9118903c01 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml @@ -1,8 +1,8 @@ -name: "dsv4-vllm-disagg-h100-1p1d-tep16-tep16-1k1k" +name: "dsv4-vllm-disagg-h100-1p1d-tep8pp2-tep8pp2-1k1k" # H100 adaptation of the upstream vLLM DeepSeek-V4-Pro recipe. The native # mixed FP4+FP8 checkpoint needs about 960GB, so every prefill/decode replica -# spans two 8xH100 nodes (1280GB) as TP16 with expert parallelism. The 1P1D +# spans two 8xH100 nodes (1280GB) as TP8 x PP2 with expert parallelism. The 1P1D # deployment therefore uses four GPU nodes total. model: path: "deepseek-v4-pro" @@ -77,10 +77,10 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - pipeline-parallel-size: 1 + pipeline-parallel-size: 2 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 16 + tensor-parallel-size: 8 tokenizer-mode: deepseek_v4 trust-remote-code: true decode: @@ -97,10 +97,10 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - pipeline-parallel-size: 1 + pipeline-parallel-size: 2 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 16 + tensor-parallel-size: 8 tokenizer-mode: deepseek_v4 trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml similarity index 92% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml index 3079650c7c..38ae7ca88e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml @@ -1,8 +1,8 @@ -name: "dsv4-vllm-disagg-h100-1p1d-tep16-tep16-8k1k" +name: "dsv4-vllm-disagg-h100-1p1d-tep8pp2-tep8pp2-8k1k" # H100 adaptation of the upstream vLLM DeepSeek-V4-Pro recipe. The native # mixed FP4+FP8 checkpoint needs about 960GB, so every prefill/decode replica -# spans two 8xH100 nodes (1280GB) as TP16 with expert parallelism. The 1P1D +# spans two 8xH100 nodes (1280GB) as TP8 x PP2 with expert parallelism. The 1P1D # deployment therefore uses four GPU nodes total. model: path: "deepseek-v4-pro" @@ -77,10 +77,10 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - pipeline-parallel-size: 1 + pipeline-parallel-size: 2 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 16 + tensor-parallel-size: 8 tokenizer-mode: deepseek_v4 trust-remote-code: true decode: @@ -97,10 +97,10 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - pipeline-parallel-size: 1 + pipeline-parallel-size: 2 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 16 + tensor-parallel-size: 8 tokenizer-mode: deepseek_v4 trust-remote-code: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 58431bf47f..0301a81ec2 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4594,11 +4594,11 @@ dsr1-fp8-h100-dynamo-sglang: # 0.20.0+, FP8 KV cache, block size 256, expert parallelism, Hopper memory # settings, and NIXL P/D transfer). The native checkpoint is mixed FP4+FP8 and # requires about 960GB VRAM, so it cannot fit one 8xH100 node (640GB). Each -# prefill/decode worker therefore spans 2 nodes at TP16+EP (TEP16, 1280GB), +# prefill/decode worker therefore spans 2 nodes at TP8 x PP2 with EP8 (1280GB), # avoiding the dense-weight replication that makes DEP unsuitable on H100. # A 1P1D run consumes 4 H100 nodes total. H100 is not yet verified upstream, # so the two-node-per-worker topology remains a bring-up target. -# Recipes: benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep16-tep16.yaml. +# Recipes: benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml. dsv4-fp8-h100-dynamo-vllm: image: vllm/vllm-openai:v0.21.0 model: deepseek-ai/DeepSeek-V4-Pro @@ -4620,36 +4620,40 @@ dsv4-fp8-h100-dynamo-vllm: - isl: 1024 osl: 1024 search-space: - # Each TEP16 worker spans two H100 nodes; 1P1D uses four GPU nodes. + # Each TP8 x PP2 worker spans two H100 nodes; 1P1D uses four GPU nodes. - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] prefill: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + pp: 2 + ep: 8 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep16-tep16.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + pp: 2 + ep: 8 dp-attn: false - isl: 8192 osl: 1024 search-space: - # Each TEP16 worker spans two H100 nodes; 1P1D uses four GPU nodes. + # Each TP8 x PP2 worker spans two H100 nodes; 1P1D uses four GPU nodes. - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] prefill: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + pp: 2 + ep: 8 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep16-tep16.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml" decode: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + pp: 2 + ep: 8 dp-attn: false minimaxm3-fp8-h200-vllm: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a99aaef166..4d777d48ca 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4771,9 +4771,9 @@ description: - "Add DeepSeek-V4-Pro FP8 H100 disaggregated multinode vLLM benchmark via Dynamo (new SKU, previously no dsv4 data on H100)" - "Follow recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro: vLLM 0.20.0+, native mixed FP4+FP8 checkpoint, FP8 KV cache, block size 256, expert parallelism, Hopper memory settings, and NIXL P/D transfer" - - "The checkpoint requires about 960GB VRAM and does not fit 8xH100 (640GB); each prefill/decode worker spans 2 nodes as TEP16 (TP16 with expert parallelism), so 1P1D uses 4 H100 nodes total" - - "One TEP16/TEP16 topology per seq-len (1k1k + 8k1k), conc 1-512; DEP16 is intentionally omitted because its replicated dense layers cannot fit H100 memory" - - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep16-tep16.yaml; H100 remains an unverified upstream bring-up target" + - "The checkpoint requires about 960GB VRAM and does not fit 8xH100 (640GB); each prefill/decode worker spans 2 nodes as TP8 x PP2 with EP8, so 1P1D uses 4 H100 nodes total" + - "One TP8/PP2/EP8 topology per seq-len (1k1k + 8k1k), conc 1-512; PP2 supplies the extra memory while TP8 preserves the checkpoint's 128-column FP8 quantization-block divisibility" + - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml; H100 remains an unverified upstream bring-up target" - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" - "Use the existing /mnt/nfs/lustre/models/dsv4-fp8 checkpoint and exclude H100 nodes without healthy pod-network RDMA plus hpc-gpu-1-16, whose mlx5 device numbering differs from the fleet" - "Use NVIDIA srt-slurm's standard vllm-container-deps.sh setup instead of a custom DeepGEMM bootstrap" From 034d21972105e4328ae71e1fcdee588d60b74799 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 15 Jul 2026 00:31:36 +0800 Subject: [PATCH 10/17] fix(dsv4): resolve H100 TP8/PP2 CUDA OOM during model load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The multinode run failed with torch.OutOfMemoryError during decode CUDA-graph capture: each TP8/PP2 worker reached ~74.6/80GB and could not allocate the last ~1GB (~3.93GB was tied up in CUDA-graph pools). Fixes: - gpu-memory-utilization 0.95 -> 0.90 (frees ~4GB headroom for capture) - add PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to prefill/decode env (the allocator's own fragmentation hint from the OOM message) - max-model-len 200000 -> 3072 (1k1k) / 9472 (8k1k): benchmark-appropriate, drops the oversized KV/metadata reservation Applied to both disagg-h100-1p1d-tep8pp2-tep8pp2.yaml recipes. 中文：修复 H100 TP8/PP2 模型加载阶段的 CUDA OOM。每个 worker 在 0.95 显存利用率 下达到 ~74.6/80GB，decode CUDA graph 捕获时无法再分配 ~1GB。将显存利用率降到 0.90、加入 expandable_segments 以减少碎片、并把 max-model-len 从 200000 改为 基准合适的 3072/9472。 --- .../1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml | 10 ++++++---- .../8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml | 10 ++++++---- perf-changelog.yaml | 1 + 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml index 9118903c01..61d57fdd82 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml @@ -49,6 +49,7 @@ backend: connector: null prefill_environment: NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" @@ -56,6 +57,7 @@ backend: VLLM_SERVER_DEV_MODE: "1" decode_environment: NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" @@ -68,10 +70,10 @@ backend: enable-expert-parallel: true enable-sleep-mode: true enforce-eager: true - gpu-memory-utilization: 0.95 + gpu-memory-utilization: 0.90 kv-cache-dtype: "fp8" kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - max-model-len: 200000 + max-model-len: 3072 max-num-batched-tokens: 16384 max-num-seqs: 16 no-disable-hybrid-kv-cache-manager: true @@ -88,10 +90,10 @@ backend: compilation-config: '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","pass_config":{"fuse_allreduce_rms":false}}' enable-expert-parallel: true enable-sleep-mode: true - gpu-memory-utilization: 0.95 + gpu-memory-utilization: 0.90 kv-cache-dtype: "fp8" kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - max-model-len: 200000 + max-model-len: 3072 max-num-batched-tokens: 16384 max-num-seqs: 16 no-disable-hybrid-kv-cache-manager: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml index 38ae7ca88e..932a871b4e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml @@ -49,6 +49,7 @@ backend: connector: null prefill_environment: NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" @@ -56,6 +57,7 @@ backend: VLLM_SERVER_DEV_MODE: "1" decode_environment: NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" @@ -68,10 +70,10 @@ backend: enable-expert-parallel: true enable-sleep-mode: true enforce-eager: true - gpu-memory-utilization: 0.95 + gpu-memory-utilization: 0.90 kv-cache-dtype: "fp8" kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - max-model-len: 200000 + max-model-len: 9472 max-num-batched-tokens: 16384 max-num-seqs: 16 no-disable-hybrid-kv-cache-manager: true @@ -88,10 +90,10 @@ backend: compilation-config: '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","pass_config":{"fuse_allreduce_rms":false}}' enable-expert-parallel: true enable-sleep-mode: true - gpu-memory-utilization: 0.95 + gpu-memory-utilization: 0.90 kv-cache-dtype: "fp8" kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - max-model-len: 200000 + max-model-len: 9472 max-num-batched-tokens: 16384 max-num-seqs: 16 no-disable-hybrid-kv-cache-manager: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4d777d48ca..af7b828578 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4777,4 +4777,5 @@ - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" - "Use the existing /mnt/nfs/lustre/models/dsv4-fp8 checkpoint and exclude H100 nodes without healthy pod-network RDMA plus hpc-gpu-1-16, whose mlx5 device numbering differs from the fleet" - "Use NVIDIA srt-slurm's standard vllm-container-deps.sh setup instead of a custom DeepGEMM bootstrap" + - "Fix CUDA OOM during model load: at gpu-memory-utilization 0.95 each TP8/PP2 worker hit ~74.6/80GB and failed to allocate the last ~1GB during decode CUDA-graph capture. Lower gpu-memory-utilization to 0.90, add PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (per the allocator's own fragmentation hint), and set benchmark-appropriate max-model-len (3072 for 1k1k, 9472 for 8k1k) instead of 200000" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 From 464f4c64297758942188012834f9ca981e8e9470 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 15 Jul 2026 01:14:35 +0800 Subject: [PATCH 11/17] fix(dsv4): H100 use DP2+EP16 (multi_node_dep) instead of PP2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TP8xPP2 disaggregated prefill hung ~300s on the 8k1k prefill then the EngineCore died (dynamo-disagg + pipeline-parallel + NIXL KV-transfer stall; the server loaded and served, so OOM is resolved — this is a PP/disagg stall). PP is not a standard vLLM DeepSeek-V4 strategy; switch to the recipe's multi_node_dep pattern: TP8 x DP2 x EP16. TP8 stays intra-node (preserves the FP8 128-column block divisibility that cross-node TP16 breaks), data-parallel (not a pipeline) spans the 2 nodes, EP16 shards experts. Per-GPU ~= dense/8 + experts/16 ~= 46GB, fits 80GB. Applied to both 1k1k and 8k1k recipes. 中文：H100 由 TP8xPP2 改为 TP8 x DP2 x EP16（recipe 的 multi_node_dep 策略）。 PP2 分离式在 8k1k 预填充阶段挂起 ~300s 导致 EngineCore 死亡（PP+disagg+NIXL 传输停滞）。DP2 用数据并行跨 2 节点，消除流水线停滞；TP8 保持节点内，避免 TP16 破坏 FP8 128 列量化块整除性。 --- .../disagg-h100-1p1d-tep8pp2-tep8pp2.yaml | 4 ++-- .../disagg-h100-1p1d-tep8pp2-tep8pp2.yaml | 4 ++-- configs/nvidia-master.yaml | 20 ++++++++----------- perf-changelog.yaml | 1 + 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml index 61d57fdd82..4bc268dfa5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml @@ -79,7 +79,7 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - pipeline-parallel-size: 2 + data-parallel-size: 2 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" tensor-parallel-size: 8 @@ -99,7 +99,7 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - pipeline-parallel-size: 2 + data-parallel-size: 2 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" tensor-parallel-size: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml index 932a871b4e..b03b7618d5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml @@ -79,7 +79,7 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - pipeline-parallel-size: 2 + data-parallel-size: 2 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" tensor-parallel-size: 8 @@ -99,7 +99,7 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - pipeline-parallel-size: 2 + data-parallel-size: 2 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" tensor-parallel-size: 8 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0301a81ec2..a4c87f2459 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4625,17 +4625,15 @@ dsv4-fp8-h100-dynamo-vllm: prefill: num-worker: 1 tp: 8 - pp: 2 - ep: 8 - dp-attn: false + ep: 16 + dp-attn: true additional-settings: - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml" decode: num-worker: 1 tp: 8 - pp: 2 - ep: 8 - dp-attn: false + ep: 16 + dp-attn: true - isl: 8192 osl: 1024 search-space: @@ -4644,17 +4642,15 @@ dsv4-fp8-h100-dynamo-vllm: prefill: num-worker: 1 tp: 8 - pp: 2 - ep: 8 - dp-attn: false + ep: 16 + dp-attn: true additional-settings: - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml" decode: num-worker: 1 tp: 8 - pp: 2 - ep: 8 - dp-attn: false + ep: 16 + dp-attn: true minimaxm3-fp8-h200-vllm: image: vllm/vllm-openai:minimax-m3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index af7b828578..f8926b1004 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4778,4 +4778,5 @@ - "Use the existing /mnt/nfs/lustre/models/dsv4-fp8 checkpoint and exclude H100 nodes without healthy pod-network RDMA plus hpc-gpu-1-16, whose mlx5 device numbering differs from the fleet" - "Use NVIDIA srt-slurm's standard vllm-container-deps.sh setup instead of a custom DeepGEMM bootstrap" - "Fix CUDA OOM during model load: at gpu-memory-utilization 0.95 each TP8/PP2 worker hit ~74.6/80GB and failed to allocate the last ~1GB during decode CUDA-graph capture. Lower gpu-memory-utilization to 0.90, add PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (per the allocator's own fragmentation hint), and set benchmark-appropriate max-model-len (3072 for 1k1k, 9472 for 8k1k) instead of 200000" + - "Replace PP2 with DP2 (data-parallel + expert-parallel, per the recipe's multi_node_dep strategy): TP8xPP2 disaggregated prefill hung ~300s on the 8k1k prefill then EngineCore died (dynamo-disagg + pipeline-parallel + NIXL KV-transfer stall). TP8 x DP2 x EP16 keeps TP8 intra-node (preserving FP8 128-column block divisibility, which cross-node TP16 breaks) and puts data-parallelism, not a pipeline, across the 2 nodes, eliminating the pipeline stall; dense/8 + experts/16 ~= 46GB/GPU fits 80GB" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 From b8056c67720a2f169844d16441e556b553a59ad4 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 15 Jul 2026 01:21:45 +0800 Subject: [PATCH 12/17] fix(dsv4): rebase H100 perf-changelog onto origin/main (append-only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit origin/main advanced (H200 refresh #2191 merged, adding dsv4-fp8-h200-* entries) so the branch's changelog was stale and check-changelog flagged the main-only entries as deletions. Reset perf-changelog.yaml to origin/main and re-append only the dsv4-fp8-h100-dynamo-vllm entry. 中文：将 H100 分支的 perf-changelog 重置到 origin/main 并仅重新追加 H100 条目， 修复 origin/main 推进（#2191 合并）导致的 append-only 校验删除报错。 --- perf-changelog.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f8926b1004..08eb97d05c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4751,6 +4751,22 @@ - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 +- config-keys: + - dsv4-fp8-h200-vllm + - dsv4-fp8-h200-vllm-mtp + description: + - "Bump vLLM image from v0.21.0 to v0.25.0 for DeepSeek-V4-Pro FP8 on H200, matching the B200/B300 dsv4 vLLM bump (#2169)" + - "Refresh stale H200 dsv4 submissions (last run 2026-05-21)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2191 + +- config-keys: + - dsv4-fp8-h200-sglang + - dsv4-fp8-h200-sglang-mtp + description: + - "Bump the pinned lmsysorg/sglang:deepseek-v4-hopper digest from the 2026-05-02 push (7f19c6dc) to the current 2026-05-13 push (1bf5d508)" + - "Refresh stale H200 dsv4 SGLang submissions (last run 2026-05-04)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2191 + - config-keys: - dsv4-fp4-mi355x-sglang description: From 185c83f2afdd326e371a69020330d8bf9717ba5c Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 15 Jul 2026 01:30:13 +0800 Subject: [PATCH 13/17] fix(dsv4): H100 use pure DP16+EP16 (tp1), not TP8xDP2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TP8xDP2 failed at launch: dynamo assigns a data_parallel_rank per GPU, so data-parallel-size must equal the 16-GPU count ("data_parallel_rank (9) must be in range [0,2)"). Switch to the framework's DEP mode: tensor-parallel-size=1, data-parallel-size=16, expert-parallel across the 2 nodes — matching the GB200 dsv4 dynamo-vLLM DEP recipe (data-parallel-size=gpus, tp1). tp1 also removes the FP8 128-column TP-divisibility issue that motivated the original PP2. 中文：H100 改为纯 DP16+EP16（tp1）。TP8xDP2 启动失败，因 dynamo 按 GPU 分配 data_parallel_rank，故 data-parallel-size 必须等于 GPU 数（16）。参照 GB200 dsv4 DEP 配方（data-parallel-size=gpus, tp1）。 --- .../1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml | 8 ++++---- .../8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml | 8 ++++---- configs/nvidia-master.yaml | 8 ++++---- perf-changelog.yaml | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml index 4bc268dfa5..97c9a1b9cb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml @@ -79,10 +79,10 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - data-parallel-size: 2 + data-parallel-size: 16 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 8 + tensor-parallel-size: 1 tokenizer-mode: deepseek_v4 trust-remote-code: true decode: @@ -99,10 +99,10 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - data-parallel-size: 2 + data-parallel-size: 16 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 8 + tensor-parallel-size: 1 tokenizer-mode: deepseek_v4 trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml index b03b7618d5..cef73fc369 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml @@ -79,10 +79,10 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - data-parallel-size: 2 + data-parallel-size: 16 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 8 + tensor-parallel-size: 1 tokenizer-mode: deepseek_v4 trust-remote-code: true decode: @@ -99,10 +99,10 @@ backend: no-disable-hybrid-kv-cache-manager: true no-enable-flashinfer-autotune: true no-enable-prefix-caching: true - data-parallel-size: 2 + data-parallel-size: 16 reasoning-parser: deepseek_v4 served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 8 + tensor-parallel-size: 1 tokenizer-mode: deepseek_v4 trust-remote-code: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a4c87f2459..af6c6225a5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4624,14 +4624,14 @@ dsv4-fp8-h100-dynamo-vllm: - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] prefill: num-worker: 1 - tp: 8 + tp: 1 ep: 16 dp-attn: true additional-settings: - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml" decode: num-worker: 1 - tp: 8 + tp: 1 ep: 16 dp-attn: true - isl: 8192 @@ -4641,14 +4641,14 @@ dsv4-fp8-h100-dynamo-vllm: - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] prefill: num-worker: 1 - tp: 8 + tp: 1 ep: 16 dp-attn: true additional-settings: - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml" decode: num-worker: 1 - tp: 8 + tp: 1 ep: 16 dp-attn: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 08eb97d05c..349e19f3de 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4794,5 +4794,5 @@ - "Use the existing /mnt/nfs/lustre/models/dsv4-fp8 checkpoint and exclude H100 nodes without healthy pod-network RDMA plus hpc-gpu-1-16, whose mlx5 device numbering differs from the fleet" - "Use NVIDIA srt-slurm's standard vllm-container-deps.sh setup instead of a custom DeepGEMM bootstrap" - "Fix CUDA OOM during model load: at gpu-memory-utilization 0.95 each TP8/PP2 worker hit ~74.6/80GB and failed to allocate the last ~1GB during decode CUDA-graph capture. Lower gpu-memory-utilization to 0.90, add PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (per the allocator's own fragmentation hint), and set benchmark-appropriate max-model-len (3072 for 1k1k, 9472 for 8k1k) instead of 200000" - - "Replace PP2 with DP2 (data-parallel + expert-parallel, per the recipe's multi_node_dep strategy): TP8xPP2 disaggregated prefill hung ~300s on the 8k1k prefill then EngineCore died (dynamo-disagg + pipeline-parallel + NIXL KV-transfer stall). TP8 x DP2 x EP16 keeps TP8 intra-node (preserving FP8 128-column block divisibility, which cross-node TP16 breaks) and puts data-parallelism, not a pipeline, across the 2 nodes, eliminating the pipeline stall; dense/8 + experts/16 ~= 46GB/GPU fits 80GB" + - "Replace PP2 with pure DP16 + EP16 (the recipe's multi_node_dep strategy): TP8xPP2 disaggregated prefill hung ~300s on the 8k1k prefill then EngineCore died (dynamo-disagg + pipeline-parallel + NIXL KV-transfer stall). Use tensor-parallel-size=1, data-parallel-size=16, expert-parallel across the 2 nodes (16 GPUs) — dynamo assigns a data_parallel_rank per GPU so DP size must equal the GPU count (TP8xDP2 fails 'data_parallel_rank must be in [0,2)'). tp1 also removes the FP8 128-column TP-divisibility issue that pushed the original recipe to PP; matches the GB200 dsv4 dynamo-vLLM DEP pattern (data-parallel-size=gpus, tp1)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 From 3e2cbfcdf822f89b6d3aec1be6252e28d988575c Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 15 Jul 2026 02:00:47 +0800 Subject: [PATCH 14/17] fix(dsv4): H100 aggregated DEP32 over 4 nodes (fit ~1TB FP8 on 80GB) DP16 on 2 nodes OOM'd during model load (dense replicated ~16GB + experts EP16 ~62GB > 80GB). Aggregated DP32+EP32 across 4 nodes shards experts to ~31GB/GPU, leaving headroom. TP1 keeps FP8 128-col block divisibility. Aggregated recipe (prefill worker 1 / decode worker 0), enforce-eager for the first green shot to isolate the memory-fit variable. --- .../vllm/deepseek-v4/1k1k/agg-h100-dep32.yaml | 95 +++++++++++++ .../disagg-h100-1p1d-tep8pp2-tep8pp2.yaml | 126 ------------------ .../vllm/deepseek-v4/8k1k/agg-h100-dep32.yaml | 95 +++++++++++++ .../disagg-h100-1p1d-tep8pp2-tep8pp2.yaml | 126 ------------------ configs/nvidia-master.yaml | 46 ++++--- perf-changelog.yaml | 16 +-- 6 files changed, 222 insertions(+), 282 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-dep32.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-dep32.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-dep32.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-dep32.yaml new file mode 100644 index 0000000000..8419d963bb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-dep32.yaml @@ -0,0 +1,95 @@ +name: "dsv4-vllm-agg-h100-dep32-1k1k" + +# DeepSeek-V4-Pro FP8 H100 aggregated multi_node_dep (throughput) recipe. +# The ~1.1TB FP8 model + tight 80GB H100s: pure DP16 on 2 nodes OOMs (dense +# weights replicate per DP rank ~16GB + experts EP16 ~62GB > 80GB), and PP +# across nodes stalls the disagg prefill. Aggregated DP32 + EP32 across 4 +# nodes (32 GPUs) shards experts enough (~31GB/GPU) that the replicated dense +# (~16GB) leaves headroom (~47GB weights/GPU). TP=1 so nothing is TP-split, +# avoiding the FP8 128-column block-divisibility limit that blocks TP16. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp8" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + agg_nodes: 4 + agg_workers: 1 + gpus_per_agg: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + block-size: 256 + tensor-parallel-size: 1 + data-parallel-size: 32 + pipeline-parallel-size: 1 + enable-expert-parallel: true + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.90 + max-model-len: 2304 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + # First green shot: eager to isolate the memory-fit variable (no + # graph-capture allocation). Once green, swap for the GB200-proven + # compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'. + enforce-eager: true + reasoning-parser: deepseek_v4 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.21.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml deleted file mode 100644 index 97c9a1b9cb..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml +++ /dev/null @@ -1,126 +0,0 @@ -name: "dsv4-vllm-disagg-h100-1p1d-tep8pp2-tep8pp2-1k1k" - -# H100 adaptation of the upstream vLLM DeepSeek-V4-Pro recipe. The native -# mixed FP4+FP8 checkpoint needs about 960GB, so every prefill/decode replica -# spans two 8xH100 nodes (1280GB) as TP8 x PP2 with expert parallelism. The 1P1D -# deployment therefore uses four GPU nodes total. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp8" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 1440 - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" - -resources: - gpu_type: "h100" - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 16 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: false - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - NCCL_CUMEM_ENABLE: "1" - PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "all" - VLLM_SERVER_DEV_MODE: "1" - decode_environment: - NCCL_CUMEM_ENABLE: "1" - PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "all" - VLLM_SERVER_DEV_MODE: "1" - vllm_config: - prefill: - block-size: 256 - compilation-config: '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","pass_config":{"fuse_allreduce_rms":false}}' - enable-expert-parallel: true - enable-sleep-mode: true - enforce-eager: true - gpu-memory-utilization: 0.90 - kv-cache-dtype: "fp8" - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - max-model-len: 3072 - max-num-batched-tokens: 16384 - max-num-seqs: 16 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - data-parallel-size: 16 - reasoning-parser: deepseek_v4 - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - decode: - block-size: 256 - compilation-config: '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","pass_config":{"fuse_allreduce_rms":false}}' - enable-expert-parallel: true - enable-sleep-mode: true - gpu-memory-utilization: 0.90 - kv-cache-dtype: "fp8" - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - max-model-len: 3072 - max-num-batched-tokens: 16384 - max-num-seqs: 16 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - data-parallel-size: 16 - reasoning-parser: deepseek_v4 - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1x2x4x8x16x32x64x128x256x512" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.21.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.21.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-dep32.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-dep32.yaml new file mode 100644 index 0000000000..da8a50b36c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-dep32.yaml @@ -0,0 +1,95 @@ +name: "dsv4-vllm-agg-h100-dep32-8k1k" + +# DeepSeek-V4-Pro FP8 H100 aggregated multi_node_dep (throughput) recipe. +# The ~1.1TB FP8 model + tight 80GB H100s: pure DP16 on 2 nodes OOMs (dense +# weights replicate per DP rank ~16GB + experts EP16 ~62GB > 80GB), and PP +# across nodes stalls the disagg prefill. Aggregated DP32 + EP32 across 4 +# nodes (32 GPUs) shards experts enough (~31GB/GPU) that the replicated dense +# (~16GB) leaves headroom (~47GB weights/GPU). TP=1 so nothing is TP-split, +# avoiding the FP8 128-column block-divisibility limit that blocks TP16. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp8" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + agg_nodes: 4 + agg_workers: 1 + gpus_per_agg: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + block-size: 256 + tensor-parallel-size: 1 + data-parallel-size: 32 + pipeline-parallel-size: 1 + enable-expert-parallel: true + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.90 + max-model-len: 9472 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + # First green shot: eager to isolate the memory-fit variable (no + # graph-capture allocation). Once green, swap for the GB200-proven + # compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'. + enforce-eager: true + reasoning-parser: deepseek_v4 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.21.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml deleted file mode 100644 index cef73fc369..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml +++ /dev/null @@ -1,126 +0,0 @@ -name: "dsv4-vllm-disagg-h100-1p1d-tep8pp2-tep8pp2-8k1k" - -# H100 adaptation of the upstream vLLM DeepSeek-V4-Pro recipe. The native -# mixed FP4+FP8 checkpoint needs about 960GB, so every prefill/decode replica -# spans two 8xH100 nodes (1280GB) as TP8 x PP2 with expert parallelism. The 1P1D -# deployment therefore uses four GPU nodes total. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp8" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 1440 - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" - -resources: - gpu_type: "h100" - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 16 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: false - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - NCCL_CUMEM_ENABLE: "1" - PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "all" - VLLM_SERVER_DEV_MODE: "1" - decode_environment: - NCCL_CUMEM_ENABLE: "1" - PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "all" - VLLM_SERVER_DEV_MODE: "1" - vllm_config: - prefill: - block-size: 256 - compilation-config: '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","pass_config":{"fuse_allreduce_rms":false}}' - enable-expert-parallel: true - enable-sleep-mode: true - enforce-eager: true - gpu-memory-utilization: 0.90 - kv-cache-dtype: "fp8" - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - max-model-len: 9472 - max-num-batched-tokens: 16384 - max-num-seqs: 16 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - data-parallel-size: 16 - reasoning-parser: deepseek_v4 - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - decode: - block-size: 256 - compilation-config: '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","pass_config":{"fuse_allreduce_rms":false}}' - enable-expert-parallel: true - enable-sleep-mode: true - gpu-memory-utilization: 0.90 - kv-cache-dtype: "fp8" - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - max-model-len: 9472 - max-num-batched-tokens: 16384 - max-num-seqs: 16 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - data-parallel-size: 16 - reasoning-parser: deepseek_v4 - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1x2x4x8x16x32x64x128x256x512" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.21.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.21.0" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index af6c6225a5..848e3056d5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4588,17 +4588,15 @@ dsr1-fp8-h100-dynamo-sglang: ep: 16 dp-attn: true -# DeepSeek-V4-Pro FP8 H100 disaggregated vLLM via Dynamo. -# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07): no -# prior dsv4 data on H100. Follows the vLLM DeepSeek-V4-Pro recipe (vLLM -# 0.20.0+, FP8 KV cache, block size 256, expert parallelism, Hopper memory -# settings, and NIXL P/D transfer). The native checkpoint is mixed FP4+FP8 and -# requires about 960GB VRAM, so it cannot fit one 8xH100 node (640GB). Each -# prefill/decode worker therefore spans 2 nodes at TP8 x PP2 with EP8 (1280GB), -# avoiding the dense-weight replication that makes DEP unsuitable on H100. -# A 1P1D run consumes 4 H100 nodes total. H100 is not yet verified upstream, -# so the two-node-per-worker topology remains a bring-up target. -# Recipes: benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml. +# DeepSeek-V4-Pro FP8 H100 aggregated vLLM via Dynamo (multi_node_dep). +# The native checkpoint is mixed FP4+FP8 (~1TB), too large for one 8xH100 node +# (640GB). Prior bring-up attempts: TP8xPP2 disagg (prefill NIXL stall) and +# pure DP16 on 2 nodes (dense replicates ~16GB + experts EP16 ~62GB -> 80GB +# OOM). The working topology is Data-Expert-Parallel across 4 nodes (32 GPUs): +# DP32 + EP32 at TP1 shards experts to ~31GB/GPU, so dense (~16GB) fits with +# headroom (~47GB weights/GPU). TP1 also sidesteps the FP8 128-column +# block-divisibility limit that blocks TP16. +# Recipes: benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-dep32.yaml. dsv4-fp8-h100-dynamo-vllm: image: vllm/vllm-openai:v0.21.0 model: deepseek-ai/DeepSeek-V4-Pro @@ -4614,42 +4612,48 @@ dsv4-fp8-h100-dynamo-vllm: router: { name: dynamo-router, version: "1.2.0.dev20260426" } kv-p2p-transfer: nixl multinode: true + # Aggregated (not P/D-disaggregated): prefill num-worker 1 + decode + # num-worker 0 is how srtslurm signals AGG mode (see RECIPES.md §5). DSv4-Pro + # (~1TB FP8) does not fit 80GB H100s under TP8xPP2 disagg (prefill stalls) nor + # pure DP16/2-node (dense replicates ~16GB + experts EP16 ~62GB > 80GB OOM). + # The throughput/multi_node_dep answer is Data-Expert-Parallel across 4 nodes: + # tp:1, dp-attn (DP32), ep:32 shards experts to ~31GB/GPU, leaving headroom. disagg: true scenarios: fixed-seq-len: - isl: 1024 osl: 1024 search-space: - # Each TP8 x PP2 worker spans two H100 nodes; 1P1D uses four GPU nodes. + # Aggregated DEP32 over 4 H100 nodes (32 GPUs): DP32 + EP32, TP1. - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] prefill: num-worker: 1 tp: 1 - ep: 16 + ep: 32 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/agg-h100-dep32.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 1 - ep: 16 + ep: 32 dp-attn: true - isl: 8192 osl: 1024 search-space: - # Each TP8 x PP2 worker spans two H100 nodes; 1P1D uses four GPU nodes. + # Aggregated DEP32 over 4 H100 nodes (32 GPUs): DP32 + EP32, TP1. - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] prefill: num-worker: 1 tp: 1 - ep: 16 + ep: 32 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/agg-h100-dep32.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 1 - ep: 16 + ep: 32 dp-attn: true minimaxm3-fp8-h200-vllm: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 349e19f3de..33f28bfcf3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4785,14 +4785,12 @@ - config-keys: - dsv4-fp8-h100-dynamo-vllm description: - - "Add DeepSeek-V4-Pro FP8 H100 disaggregated multinode vLLM benchmark via Dynamo (new SKU, previously no dsv4 data on H100)" - - "Follow recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro: vLLM 0.20.0+, native mixed FP4+FP8 checkpoint, FP8 KV cache, block size 256, expert parallelism, Hopper memory settings, and NIXL P/D transfer" - - "The checkpoint requires about 960GB VRAM and does not fit 8xH100 (640GB); each prefill/decode worker spans 2 nodes as TP8 x PP2 with EP8, so 1P1D uses 4 H100 nodes total" - - "One TP8/PP2/EP8 topology per seq-len (1k1k + 8k1k), conc 1-512; PP2 supplies the extra memory while TP8 preserves the checkpoint's 128-column FP8 quantization-block divisibility" - - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/disagg-h100-1p1d-tep8pp2-tep8pp2.yaml; H100 remains an unverified upstream bring-up target" + - "Add DeepSeek-V4-Pro FP8 H100 aggregated multinode vLLM benchmark via Dynamo (new SKU, previously no dsv4 data on H100)" + - "Native checkpoint is mixed FP4+FP8 (~1TB) and does not fit one 8xH100 node (640GB); vLLM auto-resolves the DSv4 quant config to a Hopper-compatible path (no explicit --quantization), so the only real blocker on H100 is memory, not FP4 support" + - "Aggregated Data-Expert-Parallel across 4 H100 nodes (32 GPUs), the recipe's multi_node_dep strategy: tensor-parallel-size 1, data-parallel-size 32, enable-expert-parallel (EP32). Experts shard to ~31GB/GPU and the replicated dense weights (~16GB) leave headroom (~47GB weights/GPU at gpu-memory-utilization 0.90). TP1 also sidesteps the FP8 128-column quantization-block divisibility that blocks TP16" + - "Supersedes two failed bring-up topologies: TP8xPP2 disaggregated prefill hung ~300s then EngineCore died (dynamo-disagg + pipeline-parallel + NIXL KV-transfer stall); pure DP16 on 2 nodes OOM'd during model load (dense replicates ~16GB + experts EP16 ~62GB > 80GB). Doubling to DP32/EP32 on 4 nodes is the memory-fitting fix" + - "Aggregated mode is signalled by prefill num-worker 1 + decode num-worker 0 (RECIPES.md section 5); the full vLLM serve config lives under backend.vllm_config.aggregated in the recipe. First green shot uses enforce-eager to isolate the memory-fit variable; cudagraphs (FULL_AND_PIECEWISE, custom_ops all) can be enabled once green" + - "One DEP32 topology per seq-len (1k1k + 8k1k), conc 1-512; FP8 KV cache, block size 256, max-model-len 2304/9472, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" + - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-dep32.yaml; setup via NVIDIA srt-slurm's standard vllm-container-deps.sh" - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" - - "Use the existing /mnt/nfs/lustre/models/dsv4-fp8 checkpoint and exclude H100 nodes without healthy pod-network RDMA plus hpc-gpu-1-16, whose mlx5 device numbering differs from the fleet" - - "Use NVIDIA srt-slurm's standard vllm-container-deps.sh setup instead of a custom DeepGEMM bootstrap" - - "Fix CUDA OOM during model load: at gpu-memory-utilization 0.95 each TP8/PP2 worker hit ~74.6/80GB and failed to allocate the last ~1GB during decode CUDA-graph capture. Lower gpu-memory-utilization to 0.90, add PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (per the allocator's own fragmentation hint), and set benchmark-appropriate max-model-len (3072 for 1k1k, 9472 for 8k1k) instead of 200000" - - "Replace PP2 with pure DP16 + EP16 (the recipe's multi_node_dep strategy): TP8xPP2 disaggregated prefill hung ~300s on the 8k1k prefill then EngineCore died (dynamo-disagg + pipeline-parallel + NIXL KV-transfer stall). Use tensor-parallel-size=1, data-parallel-size=16, expert-parallel across the 2 nodes (16 GPUs) — dynamo assigns a data_parallel_rank per GPU so DP size must equal the GPU count (TP8xDP2 fails 'data_parallel_rank must be in [0,2)'). tp1 also removes the FP8 128-column TP-divisibility issue that pushed the original recipe to PP; matches the GB200 dsv4 dynamo-vLLM DEP pattern (data-parallel-size=gpus, tp1)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 From fe5a2fe97f8e127e24474eeb8735519a51684fe7 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 15 Jul 2026 02:34:24 +0800 Subject: [PATCH 15/17] fix(dsv4): H100 aggregated TP8xPP2 (2 nodes), not DP32 DP32 'aggregated' collapsed onto 1 node: this framework sizes nodes as tp*pp/gpus_per_node and EP/DP-attn add no GPUs, so tp1 gave 1 node and vLLM could not place 32 DP ranks on 8 GPUs. Use the memory-proven TP8xPP2 topology (the original disagg run loaded + reached prefill; only the NIXL P/D transfer stalled) but AGGREGATED (decode num-worker 0) so there is no P/D transfer. tp8*pp2 = 16 GPUs = 2 nodes; TP8 keeps FP8 128-col divisibility. --- ...-h100-dep32.yaml => agg-h100-tep8pp2.yaml} | 30 +++++---- ...-h100-dep32.yaml => agg-h100-tep8pp2.yaml} | 30 +++++---- configs/nvidia-master.yaml | 61 ++++++++++--------- perf-changelog.yaml | 9 +-- 4 files changed, 71 insertions(+), 59 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/{agg-h100-dep32.yaml => agg-h100-tep8pp2.yaml} (66%) rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/{agg-h100-dep32.yaml => agg-h100-tep8pp2.yaml} (66%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-dep32.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml similarity index 66% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-dep32.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml index 8419d963bb..1aedc75118 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-dep32.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml @@ -1,12 +1,17 @@ -name: "dsv4-vllm-agg-h100-dep32-1k1k" +name: "dsv4-vllm-agg-h100-tep8pp2-1k1k" -# DeepSeek-V4-Pro FP8 H100 aggregated multi_node_dep (throughput) recipe. -# The ~1.1TB FP8 model + tight 80GB H100s: pure DP16 on 2 nodes OOMs (dense -# weights replicate per DP rank ~16GB + experts EP16 ~62GB > 80GB), and PP -# across nodes stalls the disagg prefill. Aggregated DP32 + EP32 across 4 -# nodes (32 GPUs) shards experts enough (~31GB/GPU) that the replicated dense -# (~16GB) leaves headroom (~47GB weights/GPU). TP=1 so nothing is TP-split, -# avoiding the FP8 128-column block-divisibility limit that blocks TP16. +# DeepSeek-V4-Pro FP8 H100 AGGREGATED TP8 x PP2 recipe (2 nodes / 16 GPUs). +# The ~1TB FP8 checkpoint does not fit one 8xH100 node (640GB). TP8 x PP2 +# shards attention/dense (/8) and layers (/2) across 16 GPUs -- the original +# memory-correct sizing that loaded the model and reached prefill. That +# topology's ONLY failure was the disaggregated NIXL KV-transfer stall between +# the prefill and decode workers; running AGGREGATED (single worker, decode +# num-worker 0) removes the P/D split entirely, so there is no NIXL transfer to +# stall on. TP8 (not TP16) preserves the FP8 128-column block divisibility +# (moe_intermediate 3072 / 8 = 384, divisible by 128; / 16 = 192 is not). +# Node allocation for this framework is tp*pp/gpus_per_node = 8*2/8 = 2 nodes; +# EP and DP-attention do NOT contribute GPUs, which is why the earlier DP32 +# aggregated attempt collapsed onto a single node. model: path: "deepseek-v4-pro" container: "vllm/vllm-openai:v0.21.0" @@ -33,9 +38,9 @@ sbatch_directives: resources: gpu_type: "h100" gpus_per_node: 8 - agg_nodes: 4 + agg_nodes: 2 agg_workers: 1 - gpus_per_agg: 32 + gpus_per_agg: 16 frontend: type: dynamo @@ -56,9 +61,8 @@ backend: aggregated: served-model-name: "deepseek-ai/DeepSeek-V4-Pro" block-size: 256 - tensor-parallel-size: 1 - data-parallel-size: 32 - pipeline-parallel-size: 1 + tensor-parallel-size: 8 + pipeline-parallel-size: 2 enable-expert-parallel: true kv-cache-dtype: "fp8" gpu-memory-utilization: 0.90 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-dep32.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml similarity index 66% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-dep32.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml index da8a50b36c..b4466e5452 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-dep32.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml @@ -1,12 +1,17 @@ -name: "dsv4-vllm-agg-h100-dep32-8k1k" +name: "dsv4-vllm-agg-h100-tep8pp2-8k1k" -# DeepSeek-V4-Pro FP8 H100 aggregated multi_node_dep (throughput) recipe. -# The ~1.1TB FP8 model + tight 80GB H100s: pure DP16 on 2 nodes OOMs (dense -# weights replicate per DP rank ~16GB + experts EP16 ~62GB > 80GB), and PP -# across nodes stalls the disagg prefill. Aggregated DP32 + EP32 across 4 -# nodes (32 GPUs) shards experts enough (~31GB/GPU) that the replicated dense -# (~16GB) leaves headroom (~47GB weights/GPU). TP=1 so nothing is TP-split, -# avoiding the FP8 128-column block-divisibility limit that blocks TP16. +# DeepSeek-V4-Pro FP8 H100 AGGREGATED TP8 x PP2 recipe (2 nodes / 16 GPUs). +# The ~1TB FP8 checkpoint does not fit one 8xH100 node (640GB). TP8 x PP2 +# shards attention/dense (/8) and layers (/2) across 16 GPUs -- the original +# memory-correct sizing that loaded the model and reached prefill. That +# topology's ONLY failure was the disaggregated NIXL KV-transfer stall between +# the prefill and decode workers; running AGGREGATED (single worker, decode +# num-worker 0) removes the P/D split entirely, so there is no NIXL transfer to +# stall on. TP8 (not TP16) preserves the FP8 128-column block divisibility +# (moe_intermediate 3072 / 8 = 384, divisible by 128; / 16 = 192 is not). +# Node allocation for this framework is tp*pp/gpus_per_node = 8*2/8 = 2 nodes; +# EP and DP-attention do NOT contribute GPUs, which is why the earlier DP32 +# aggregated attempt collapsed onto a single node. model: path: "deepseek-v4-pro" container: "vllm/vllm-openai:v0.21.0" @@ -33,9 +38,9 @@ sbatch_directives: resources: gpu_type: "h100" gpus_per_node: 8 - agg_nodes: 4 + agg_nodes: 2 agg_workers: 1 - gpus_per_agg: 32 + gpus_per_agg: 16 frontend: type: dynamo @@ -56,9 +61,8 @@ backend: aggregated: served-model-name: "deepseek-ai/DeepSeek-V4-Pro" block-size: 256 - tensor-parallel-size: 1 - data-parallel-size: 32 - pipeline-parallel-size: 1 + tensor-parallel-size: 8 + pipeline-parallel-size: 2 enable-expert-parallel: true kv-cache-dtype: "fp8" gpu-memory-utilization: 0.90 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 848e3056d5..76b1784cec 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4588,15 +4588,16 @@ dsr1-fp8-h100-dynamo-sglang: ep: 16 dp-attn: true -# DeepSeek-V4-Pro FP8 H100 aggregated vLLM via Dynamo (multi_node_dep). +# DeepSeek-V4-Pro FP8 H100 aggregated vLLM via Dynamo (TP8 x PP2, 2 nodes). # The native checkpoint is mixed FP4+FP8 (~1TB), too large for one 8xH100 node -# (640GB). Prior bring-up attempts: TP8xPP2 disagg (prefill NIXL stall) and -# pure DP16 on 2 nodes (dense replicates ~16GB + experts EP16 ~62GB -> 80GB -# OOM). The working topology is Data-Expert-Parallel across 4 nodes (32 GPUs): -# DP32 + EP32 at TP1 shards experts to ~31GB/GPU, so dense (~16GB) fits with -# headroom (~47GB weights/GPU). TP1 also sidesteps the FP8 128-column -# block-divisibility limit that blocks TP16. -# Recipes: benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-dep32.yaml. +# (640GB). Prior attempts: TP8xPP2 *disagg* loaded the model and reached prefill +# but hung on the NIXL P/D KV-transfer; DP16/2-node OOM'd; DP32 "aggregated" +# collapsed onto ONE node because this framework sizes nodes as tp*pp/gpus_per_node +# (EP and DP-attention add no GPUs), so tp1 => 1 node and vLLM could not place 32 +# DP ranks on 8 GPUs. The working topology is the memory-proven TP8xPP2 run made +# AGGREGATED (single worker, no P/D split => no NIXL transfer to stall on): tp8*pp2 +# = 16 GPUs = 2 nodes. TP8 (not TP16) keeps FP8 128-column block divisibility. +# Recipes: benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-tep8pp2.yaml. dsv4-fp8-h100-dynamo-vllm: image: vllm/vllm-openai:v0.21.0 model: deepseek-ai/DeepSeek-V4-Pro @@ -4613,48 +4614,50 @@ dsv4-fp8-h100-dynamo-vllm: kv-p2p-transfer: nixl multinode: true # Aggregated (not P/D-disaggregated): prefill num-worker 1 + decode - # num-worker 0 is how srtslurm signals AGG mode (see RECIPES.md §5). DSv4-Pro - # (~1TB FP8) does not fit 80GB H100s under TP8xPP2 disagg (prefill stalls) nor - # pure DP16/2-node (dense replicates ~16GB + experts EP16 ~62GB > 80GB OOM). - # The throughput/multi_node_dep answer is Data-Expert-Parallel across 4 nodes: - # tp:1, dp-attn (DP32), ep:32 shards experts to ~31GB/GPU, leaving headroom. + # num-worker 0 is how srtslurm signals AGG mode (see RECIPES.md §5). Node + # count = tp*pp/gpus_per_node = 8*2/8 = 2. tp:8 pp:2 with expert-parallel; + # dp-attn false (TP8 attention). Aggregated => no NIXL transfer to stall on. disagg: true scenarios: fixed-seq-len: - isl: 1024 osl: 1024 search-space: - # Aggregated DEP32 over 4 H100 nodes (32 GPUs): DP32 + EP32, TP1. + # Aggregated TP8 x PP2 over 2 H100 nodes (16 GPUs), EP8. - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] prefill: num-worker: 1 - tp: 1 - ep: 32 - dp-attn: true + tp: 8 + pp: 2 + ep: 8 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/agg-h100-dep32.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml" decode: num-worker: 0 - tp: 1 - ep: 32 - dp-attn: true + tp: 8 + pp: 2 + ep: 8 + dp-attn: false - isl: 8192 osl: 1024 search-space: - # Aggregated DEP32 over 4 H100 nodes (32 GPUs): DP32 + EP32, TP1. + # Aggregated TP8 x PP2 over 2 H100 nodes (16 GPUs), EP8. - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] prefill: num-worker: 1 - tp: 1 - ep: 32 - dp-attn: true + tp: 8 + pp: 2 + ep: 8 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/agg-h100-dep32.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml" decode: num-worker: 0 - tp: 1 - ep: 32 - dp-attn: true + tp: 8 + pp: 2 + ep: 8 + dp-attn: false minimaxm3-fp8-h200-vllm: image: vllm/vllm-openai:minimax-m3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 33f28bfcf3..ce50321791 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4787,10 +4787,11 @@ description: - "Add DeepSeek-V4-Pro FP8 H100 aggregated multinode vLLM benchmark via Dynamo (new SKU, previously no dsv4 data on H100)" - "Native checkpoint is mixed FP4+FP8 (~1TB) and does not fit one 8xH100 node (640GB); vLLM auto-resolves the DSv4 quant config to a Hopper-compatible path (no explicit --quantization), so the only real blocker on H100 is memory, not FP4 support" - - "Aggregated Data-Expert-Parallel across 4 H100 nodes (32 GPUs), the recipe's multi_node_dep strategy: tensor-parallel-size 1, data-parallel-size 32, enable-expert-parallel (EP32). Experts shard to ~31GB/GPU and the replicated dense weights (~16GB) leave headroom (~47GB weights/GPU at gpu-memory-utilization 0.90). TP1 also sidesteps the FP8 128-column quantization-block divisibility that blocks TP16" - - "Supersedes two failed bring-up topologies: TP8xPP2 disaggregated prefill hung ~300s then EngineCore died (dynamo-disagg + pipeline-parallel + NIXL KV-transfer stall); pure DP16 on 2 nodes OOM'd during model load (dense replicates ~16GB + experts EP16 ~62GB > 80GB). Doubling to DP32/EP32 on 4 nodes is the memory-fitting fix" + - "Aggregated TP8 x PP2 across 2 H100 nodes (16 GPUs) with expert-parallel: tensor-parallel-size 8, pipeline-parallel-size 2, enable-expert-parallel. TP8 shards attention/dense (/8) and PP2 splits layers (/2) -- the original memory-correct sizing that loaded the model and reached prefill. TP8 (not TP16) preserves the FP8 128-column quantization-block divisibility (moe_intermediate 3072/8=384 divisible by 128; /16=192 not)" + - "Native checkpoint quant note: vLLM auto-resolves the DSv4 config to a Hopper-compatible path (no explicit --quantization); the only blocker on H100 is memory, not FP4 support (all failed attempts loaded far enough to fail on placement/transfer, not on FP4)" + - "Supersedes three failed bring-up topologies: (1) TP8xPP2 *disaggregated* loaded the model and reached prefill but hung ~300s on the NIXL P/D KV-transfer then EngineCore died; (2) pure DP16 on 2 nodes OOM'd at model load; (3) DP32 'aggregated' collapsed onto ONE node -- this framework sizes nodes as tp*pp/gpus_per_node and EP/DP-attention add no GPUs, so tp1 => 1 node and vLLM could not place 32 DP ranks on 8 GPUs. Running the memory-proven TP8xPP2 AGGREGATED (single worker, no P/D split) removes the NIXL transfer entirely and allocates 2 nodes deterministically (8*2/8)" - "Aggregated mode is signalled by prefill num-worker 1 + decode num-worker 0 (RECIPES.md section 5); the full vLLM serve config lives under backend.vllm_config.aggregated in the recipe. First green shot uses enforce-eager to isolate the memory-fit variable; cudagraphs (FULL_AND_PIECEWISE, custom_ops all) can be enabled once green" - - "One DEP32 topology per seq-len (1k1k + 8k1k), conc 1-512; FP8 KV cache, block size 256, max-model-len 2304/9472, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" - - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-dep32.yaml; setup via NVIDIA srt-slurm's standard vllm-container-deps.sh" + - "One TP8xPP2 topology per seq-len (1k1k + 8k1k), conc 1-512; FP8 KV cache, block size 256, max-model-len 2304/9472, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" + - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-tep8pp2.yaml; setup via NVIDIA srt-slurm's standard vllm-container-deps.sh" - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 From 756a429faec4bb632c618788e0afa097cf634f4b Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:33:17 +0800 Subject: [PATCH 16/17] perf(dsv4): H100 use CUDA graphs (FULL_AND_PIECEWISE) not eager On-cluster prototype (srtctl job 11256, 2-node TP8xPP2, 1x8) confirmed FULL_AND_PIECEWISE graphs fit at gpu-mem-util 0.92 (no OOM) and cut decode TPOT from the eager ~144ms to ~15ms (~9.4x). The eager sweep served correctly but was dispatch-bound and could not finish in 8h; graphs make it viable on the same 2 nodes (no extra nodes needed). --- .../vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml | 12 +++++++----- .../vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml | 12 +++++++----- perf-changelog.yaml | 3 ++- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml index 1aedc75118..c548caf6cb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml @@ -65,16 +65,18 @@ backend: pipeline-parallel-size: 2 enable-expert-parallel: true kv-cache-dtype: "fp8" - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.92 max-model-len: 2304 max-num-seqs: 16 max-num-batched-tokens: 16384 no-enable-prefix-caching: true no-enable-flashinfer-autotune: true - # First green shot: eager to isolate the memory-fit variable (no - # graph-capture allocation). Once green, swap for the GB200-proven - # compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'. - enforce-eager: true + # CUDA graphs (not eager): the eager run served correctly but decode ran + # at ~144ms/token (Python dispatch-bound, flat vs concurrency), so the + # conc sweep could not finish in 8h. On-cluster prototype (job 11256) + # confirmed FULL_AND_PIECEWISE graphs FIT this 2-node TP8xPP2 layout at + # gpu-mem-util 0.92 and drop TPOT to ~15ms (~9.4x). No extra nodes needed. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' reasoning-parser: deepseek_v4 tokenizer-mode: deepseek_v4 trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml index b4466e5452..c6f19ad0ec 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml @@ -65,16 +65,18 @@ backend: pipeline-parallel-size: 2 enable-expert-parallel: true kv-cache-dtype: "fp8" - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.92 max-model-len: 9472 max-num-seqs: 16 max-num-batched-tokens: 16384 no-enable-prefix-caching: true no-enable-flashinfer-autotune: true - # First green shot: eager to isolate the memory-fit variable (no - # graph-capture allocation). Once green, swap for the GB200-proven - # compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'. - enforce-eager: true + # CUDA graphs (not eager): the eager run served correctly but decode ran + # at ~144ms/token (Python dispatch-bound, flat vs concurrency), so the + # conc sweep could not finish in 8h. On-cluster prototype (job 11256) + # confirmed FULL_AND_PIECEWISE graphs FIT this 2-node TP8xPP2 layout at + # gpu-mem-util 0.92 and drop TPOT to ~15ms (~9.4x). No extra nodes needed. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' reasoning-parser: deepseek_v4 tokenizer-mode: deepseek_v4 trust-remote-code: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ce50321791..57b49fd16f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4790,7 +4790,8 @@ - "Aggregated TP8 x PP2 across 2 H100 nodes (16 GPUs) with expert-parallel: tensor-parallel-size 8, pipeline-parallel-size 2, enable-expert-parallel. TP8 shards attention/dense (/8) and PP2 splits layers (/2) -- the original memory-correct sizing that loaded the model and reached prefill. TP8 (not TP16) preserves the FP8 128-column quantization-block divisibility (moe_intermediate 3072/8=384 divisible by 128; /16=192 not)" - "Native checkpoint quant note: vLLM auto-resolves the DSv4 config to a Hopper-compatible path (no explicit --quantization); the only blocker on H100 is memory, not FP4 support (all failed attempts loaded far enough to fail on placement/transfer, not on FP4)" - "Supersedes three failed bring-up topologies: (1) TP8xPP2 *disaggregated* loaded the model and reached prefill but hung ~300s on the NIXL P/D KV-transfer then EngineCore died; (2) pure DP16 on 2 nodes OOM'd at model load; (3) DP32 'aggregated' collapsed onto ONE node -- this framework sizes nodes as tp*pp/gpus_per_node and EP/DP-attention add no GPUs, so tp1 => 1 node and vLLM could not place 32 DP ranks on 8 GPUs. Running the memory-proven TP8xPP2 AGGREGATED (single worker, no P/D split) removes the NIXL transfer entirely and allocates 2 nodes deterministically (8*2/8)" - - "Aggregated mode is signalled by prefill num-worker 1 + decode num-worker 0 (RECIPES.md section 5); the full vLLM serve config lives under backend.vllm_config.aggregated in the recipe. First green shot uses enforce-eager to isolate the memory-fit variable; cudagraphs (FULL_AND_PIECEWISE, custom_ops all) can be enabled once green" + - "Aggregated mode is signalled by prefill num-worker 1 + decode num-worker 0 (RECIPES.md section 5); the full vLLM serve config lives under backend.vllm_config.aggregated in the recipe" + - "CUDA graphs (compilation-config cudagraph_mode FULL_AND_PIECEWISE, custom_ops all) at gpu-memory-utilization 0.92 -- NOT enforce-eager. An eager run served correctly (eval passed) but decode ran at ~144ms/token (Python-dispatch-bound, flat vs concurrency), so the conc sweep could not finish in the 8h SLURM limit (killed mid-conc-128 after ~7h). An on-cluster srtctl prototype (2-node TP8xPP2, 1x8 sweep) confirmed FULL_AND_PIECEWISE graphs FIT this layout (no OOM at 0.92) and cut TPOT to ~15ms (~9.4x); no extra nodes required" - "One TP8xPP2 topology per seq-len (1k1k + 8k1k), conc 1-512; FP8 KV cache, block size 256, max-model-len 2304/9472, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-tep8pp2.yaml; setup via NVIDIA srt-slurm's standard vllm-container-deps.sh" - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" From d71c6fa6c00df54f4fefa4e5b0a6089665038dfa Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:17:30 +0800 Subject: [PATCH 17/17] perf(dsv4): cap H100 8k1k at conc256 (conc512 exceeds 8h SLURM limit) 8k1k conc512 = 5120 x 8k-token requests needs ~3.6h for that single level; the run blew the 8h SLURM limit mid-conc512 (1k1k + eval were already green). conc256 completes in ~71min and fits with margin. 1k1k keeps conc512 (short requests finish fast). Optimized cudagraph config otherwise unchanged. --- .../vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml | 4 +++- configs/nvidia-master.yaml | 5 ++++- perf-changelog.yaml | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml index c6f19ad0ec..b55cdc4b13 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml @@ -85,7 +85,9 @@ benchmark: type: "sa-bench" isl: 8192 osl: 1024 - concurrencies: "1x2x4x8x16x32x64x128x256x512" + # Capped at 256: conc512 (5120 x 8k-token requests) needs ~3.6h for that one + # level and exceeds the 8h SLURM limit; conc256 completes (~71min) and fits. + concurrencies: "1x2x4x8x16x32x64x128x256" req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 76b1784cec..3d99512a8a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4643,7 +4643,10 @@ dsv4-fp8-h100-dynamo-vllm: osl: 1024 search-space: # Aggregated TP8 x PP2 over 2 H100 nodes (16 GPUs), EP8. - - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + # 8k1k capped at conc256: conc512 (5120 x 8k-token requests) needs ~3.6h + # for that single level and blew the 8h SLURM limit; conc256 completes in + # ~71min and fits with margin. 1k1k keeps 512 (short requests finish fast). + - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256] prefill: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 57b49fd16f..b92f24cd37 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4792,7 +4792,7 @@ - "Supersedes three failed bring-up topologies: (1) TP8xPP2 *disaggregated* loaded the model and reached prefill but hung ~300s on the NIXL P/D KV-transfer then EngineCore died; (2) pure DP16 on 2 nodes OOM'd at model load; (3) DP32 'aggregated' collapsed onto ONE node -- this framework sizes nodes as tp*pp/gpus_per_node and EP/DP-attention add no GPUs, so tp1 => 1 node and vLLM could not place 32 DP ranks on 8 GPUs. Running the memory-proven TP8xPP2 AGGREGATED (single worker, no P/D split) removes the NIXL transfer entirely and allocates 2 nodes deterministically (8*2/8)" - "Aggregated mode is signalled by prefill num-worker 1 + decode num-worker 0 (RECIPES.md section 5); the full vLLM serve config lives under backend.vllm_config.aggregated in the recipe" - "CUDA graphs (compilation-config cudagraph_mode FULL_AND_PIECEWISE, custom_ops all) at gpu-memory-utilization 0.92 -- NOT enforce-eager. An eager run served correctly (eval passed) but decode ran at ~144ms/token (Python-dispatch-bound, flat vs concurrency), so the conc sweep could not finish in the 8h SLURM limit (killed mid-conc-128 after ~7h). An on-cluster srtctl prototype (2-node TP8xPP2, 1x8 sweep) confirmed FULL_AND_PIECEWISE graphs FIT this layout (no OOM at 0.92) and cut TPOT to ~15ms (~9.4x); no extra nodes required" - - "One TP8xPP2 topology per seq-len (1k1k + 8k1k), conc 1-512; FP8 KV cache, block size 256, max-model-len 2304/9472, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" + - "One TP8xPP2 topology per seq-len; 1k1k conc 1-512, 8k1k conc 1-256 (8k1k conc512 = 5120 x 8k-token requests needs ~3.6h for that single level and exceeds the 8h SLURM limit; conc256 completes in ~71min). FP8 KV cache, block size 256, max-model-len 2304/9472, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-tep8pp2.yaml; setup via NVIDIA srt-slurm's standard vllm-container-deps.sh" - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196