From e93f00a7be242f339d77d76825de391828004ffc Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:11:42 +0800 Subject: [PATCH] feat(dsv4): restore GB200 dynamo-vllm 1k1k sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The day-0 1k1k recipes (#1129) were dropped by the deepseek-v4-pro-sa curation (5b0347f43), freezing GB200 dsv4 1k1k submissions at the 2026-04-25 day-0 run. Re-add the three day-0 topologies (1p1d-dep8-tep8, 1p1d-dep8-dep16, 3p1d-dep8-dep16) with engine tuning unchanged; recipe container/dynamo/infra fields re-based on the current 8k1k -sa set. 中文：恢复 GB200 dsv4 dynamo-vLLM 的 1k1k 扫描。day-0 的 1k1k 配方在 deepseek-v4-pro-sa 整理时被移除，导致官网 GB200 1k1k 数据停留在 2026-04-25。重新加入三个 day-0 拓扑（引擎调优不变），配方的 container/dynamo/infra 字段与当前 8k1k -sa 集合对齐。 --- .../1k1k/disagg-gb200-1p1d-dep8-dep16.yaml | 136 +++++++++++++++ .../1k1k/disagg-gb200-1p1d-dep8-tep8.yaml | 155 ++++++++++++++++++ .../1k1k/disagg-gb200-3p1d-dep8-dep16.yaml | 128 +++++++++++++++ configs/nvidia-master.yaml | 58 +++++++ perf-changelog.yaml | 8 + 5 files changed, 485 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-1p1d-dep8-dep16.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-1p1d-dep8-tep8.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-3p1d-dep8-dep16.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-1p1d-dep8-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-1p1d-dep8-dep16.yaml new file mode 100644 index 0000000000..3a00e11b18 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-1p1d-dep8-dep16.yaml @@ -0,0 +1,136 @@ +name: "dsv4-vllm-disagg-gb200-1p1d-dep8-dep16" + +# 1k/1k mid-to-high throughput topology. Extrapolated from +# kimi-k2.5/1k1k/disagg-gb200-1p1d-dep4-dep16.yaml adjusted for DSV4-Pro's +# DP>=8 minimum. Single prefill worker feeding a wide DP=16 decode handles +# conc 256-4096 cleanly for 1k prompts (prefill throughput per rank is high +# enough at this prompt length; see kimi precedent). +# +# Differences from our 8k1k 7p1d-dep8-dep16: +# * prefill_workers: 1 (vs 7) — 1k prompts don't need 14 prefill nodes +# * max-model-len: 3072 instead of auto +# * prefill max-num-seqs: 16 (fills 16384-token budget at 1k per seq) +# * decode max-num-seqs: 512 instead of 256 (shorter KV, more parallelism) +# * max-cudagraph-capture-size / max-num-batched-tokens (decode): 512 +# +# Restored 2026-07 for the GB200 dsv4 1k1k refresh: these day-0 recipes +# (#1129) were dropped by the deepseek-v4-pro-sa curation (5b0347f43, +# "only 8k/1k in scope"), which froze the site's GB200 1k1k rows at the +# 2026-04-25 day-0 run. Container/dynamo/infra fields updated to match +# the current 8k1k -sa set (v0.20.0-ubuntu2404, dynamo wheel +# 1.2.0.dev20260426, dedicated NATS/etcd infra node); topology and engine +# tuning unchanged from day-0. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.20.0-ubuntu2404" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +# Also set slurm.time_limit explicitly (above srtslurm.yaml's 6h default) so +# a slow first-time Lustre load + cudagraph capture can't get cut off by the +# SLURM wall clock. +slurm: + time_limit: "8:00:00" + +# Bumped from the 1800s default to 4 hours. DSV4-Pro weights load slowly from +# Lustre with multiple workers contending for the same OSTs — previous 1k/1k +# run hit the default 1800s. Make this *very* generous since the cost of an +# over-long deadline is just sitting idle, not wasted compute. +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enforce-eager: true + max-model-len: 3072 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.88 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "128x256x1024x2048x4096" + req_rate: "inf" + use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-1p1d-dep8-tep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-1p1d-dep8-tep8.yaml new file mode 100644 index 0000000000..55d11dda61 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-1p1d-dep8-tep8.yaml @@ -0,0 +1,155 @@ +name: "dsv4-vllm-disagg-gb200-1p1d-dep8-tep8" + +# 1k/1k variant of NVIDIA's 8k/1k 1p1d-dep8-tep8 recipe (mirrored from +# aflowers/gb200-dsv4-recipes branch). Same topology and tuning; only +# max-model-len shrinks from 9280 (8k+1k+pad) to 3072 (1k+1k+pad). No +# upstream NVIDIA reference for DSV4-Pro 1k/1k vLLM disagg yet. +# +# Topology: 1 prefill (DP=8) + 1 decode (TP=8). 4 nodes total. Targets +# very low concurrency (1-64). +# +# Local deltas vs upstream 8k/1k sibling: same as the 8k/1k recipe — see +# ../8k1k/disagg-gb200-1p1d-dep8-tep8.yaml for the full deviation list. +# +# Restored 2026-07 for the GB200 dsv4 1k1k refresh: these day-0 recipes +# (#1129) were dropped by the deepseek-v4-pro-sa curation (5b0347f43, +# "only 8k/1k in scope"), which froze the site's GB200 1k1k rows at the +# 2026-04-25 day-0 run. Container/dynamo/infra fields updated to match +# the current 8k1k -sa set (v0.20.0-ubuntu2404, dynamo wheel +# 1.2.0.dev20260426, dedicated NATS/etcd infra node); topology and engine +# tuning unchanged from day-0. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.20.0-ubuntu2404" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enforce-eager: true + max-model-len: 3072 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.8 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # CPU/DRAM expert offload — required for fit. Without these the prefill + # rank reports `Available KV cache memory: -16 GiB` and the engine + # refuses to start. Numa-bind from upstream is still off because our + # NVIDIA/srt-slurm@sa-submission-q2-2026 clone doesn't ship the + # vllm_numa_bind_hash_fix.py patch. + offload-group-size: 3 + offload-num-in-group: 1 + offload-prefetch-step: 2 + tokenizer-mode: deepseek_v4 + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 64 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 64 + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 256 + attention-config: '{"use_fp4_indexer_cache":true}' + compilation-config: '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","pass_config":{"fuse_allreduce_rms":false}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1x4x8x16x32x64" + req_rate: "inf" + use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-3p1d-dep8-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-3p1d-dep8-dep16.yaml new file mode 100644 index 0000000000..9cf0bdc195 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-gb200-3p1d-dep8-dep16.yaml @@ -0,0 +1,128 @@ +name: "dsv4-vllm-disagg-gb200-3p1d-dep8-dep16" + +# 1k/1k high-throughput topology: 3 prefill workers (DP=8) feeding a single +# wide decode (DP=16). 10 nodes total. Sized for conc 4096-8192 — at those +# concurrencies a single prefill worker (the 1p1d-dep8-dep16 sibling) +# becomes the bottleneck since 1k prefill arrival rate ~200-300 req/s +# exceeds what one DP=8 worker can sustain. +# +# Decode capacity: +# max-num-seqs: 1024 with DP=16 -> 16384 total simultaneous slots, which +# leaves headroom over the conc=8192 working set (per-rank avg 512). +# max-cudagraph-capture-size kept at 512: per-rank batch at conc=8192 is +# ~512 so cudagraphs still apply at steady state. +# +# Restored 2026-07 for the GB200 dsv4 1k1k refresh: these day-0 recipes +# (#1129) were dropped by the deepseek-v4-pro-sa curation (5b0347f43, +# "only 8k/1k in scope"), which froze the site's GB200 1k1k rows at the +# 2026-04-25 day-0 run. Container/dynamo/infra fields updated to match +# the current 8k1k -sa set (v0.20.0-ubuntu2404, dynamo wheel +# 1.2.0.dev20260426, dedicated NATS/etcd infra node); topology and engine +# tuning unchanged from day-0. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.20.0-ubuntu2404" + precision: "fp4" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 6 + decode_nodes: 4 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + + prefill_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + + decode_environment: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enforce-eager: true + max-model-len: 3072 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.88 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 1024 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4096x8192" + req_rate: "inf" + use_chat_template: false diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f90c510186..5c8196f42f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8671,6 +8671,64 @@ dsv4-fp4-gb200-dynamo-vllm: disagg: true scenarios: fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + # 1k/1k restored 2026-07: the day-0 recipes (#1129) were dropped by the + # deepseek-v4-pro-sa curation (5b0347f43, "only 8k/1k in scope"), which + # froze the site's GB200 dsv4 1k1k rows at the 2026-04-25 day-0 run. + # Topologies and conc-lists match day-0; recipes under + # recipes/vllm/deepseek-v4/1k1k/ are re-based on the current 8k1k -sa + # container/dynamo/infra layout. + + # Low latency: 1 prefill (DEP=8) + 1 decode (TP=8). 5 nodes total with + # a dedicated NATS/etcd infra node. + - conc-list: [1, 4, 8, 16, 32, 64] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/disagg-gb200-1p1d-dep8-tep8.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + + # Mid curve: 1 prefill (DEP=8) + 1 decode (DEP=16). 7 nodes total with + # a dedicated NATS/etcd infra node. + - conc-list: [128, 256, 1024, 2048, 4096] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/disagg-gb200-1p1d-dep8-dep16.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + + # High throughput: 3 prefill (DEP=8 each) + 1 decode (DEP=16). 11 nodes + # total with a dedicated NATS/etcd infra node. + - conc-list: [4096, 8192] + prefill: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/disagg-gb200-3p1d-dep8-dep16.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - isl: 8192 osl: 1024 search-space: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 57d510dd15..cf6399d06e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4750,3 +4750,11 @@ - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm + description: + - "Restore the GB200 dsv4 1k1k sweep: the day-0 1k1k recipes (#1129) were dropped by the deepseek-v4-pro-sa curation (only 8k/1k kept), freezing the site's GB200 dsv4 1k1k submissions at the 2026-04-25 day-0 run (217 out tok/s/GPU at 30 tok/s/user vs GB300's 9050 — not representative)" + - "Re-add the three day-0 1k1k topologies with unchanged engine tuning: 1p1d-dep8-tep8 (conc 1-64), 1p1d-dep8-dep16 (conc 128-4096), 3p1d-dep8-dep16 (conc 4096, 8192)" + - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/ re-based on the current 8k1k -sa layout: container vllm/vllm-openai:v0.20.0-ubuntu2404 (matches the config image/alias key), dynamo wheel 1.2.0.dev20260426, dedicated NATS/etcd infra node" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2190