From ebc6af9af2edb1f5b2654dfc23a28ea2288a6795 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 15 Jul 2026 11:20:53 -0500 Subject: [PATCH 1/3] feat(agentic): split DeepSeek-V4 B200 vLLM recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry the B200-only launcher, config search space, and changelog scope from PR #2202. 中文：拆分 DeepSeek-V4 B200 vLLM AgentX 配方，仅保留 PR #2202 中的 B200 启动器、配置搜索空间和变更日志范围。 --- .../single_node/agentic/dsv4_fp4_b200_vllm.sh | 106 ++++++++++++------ configs/nvidia-master.yaml | 16 ++- perf-changelog.yaml | 8 ++ 3 files changed, 89 insertions(+), 41 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index 35cc55c04..a16114a49 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail set -x # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B200 using vLLM. @@ -14,21 +14,25 @@ set -x # Highest aggregate throughput at large CONC. # # Image is configured in nvidia-master.yaml. block_size=256, -# kv-cache-dtype=fp8, FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph -# capture with custom_ops=all (per the vLLM blog recipe at -# https://vllm.ai/blog/deepseek-v4). +# kv-cache-dtype=fp8, FLASHINFER_MLA_SPARSE_DSV4 attention with the FP4 indexer +# cache, FULL_DECODE_ONLY cudagraph capture, and (in EP tiers) mega-MoE backend. # # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. +# Pure TP is GPU-resident (KV_OFFLOADING=none). DEP tiers offload KV to host +# DRAM: KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-simple or mooncake. source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -DCP_SIZE="${DCP_SIZE:-1}" -PCP_SIZE="${PCP_SIZE:-1}" +if [ -z "$DCP_SIZE" ]; then + DCP_SIZE=1 +fi +if [ -z "$PCP_SIZE" ]; then + PCP_SIZE=1 +fi VLLM_CP_ARGS=() if [ "$DCP_SIZE" -gt 1 ]; then VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE") @@ -37,21 +41,21 @@ if [ "$PCP_SIZE" -gt 1 ]; then VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE") fi -GPU_COUNT="${GPU_COUNT:-$((TP * PCP_SIZE))}" +GPU_COUNT=$TP if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2 exit 1 fi export GPU_COUNT -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi # `hf download` creates the target dir if missing and is itself idempotent. # When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE # Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then +if [[ -n "$MODEL_PATH" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" fi @@ -65,12 +69,6 @@ nvidia-smi resolve_trace_source install_agentic_deps -# vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings, -# which fails DSV4 indexer compilation with mlir_global_dtors(..., data). -# Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved. -agentic_pip_install --quiet --force-reinstall --no-deps \ - 'nvidia-cutlass-dsl-libs-cu13==4.5.2' - # vllm-project/router expands the one HTTP backend into one logical worker per # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's # X-Correlation-ID is stable for every turn of a conversation; alias it to the @@ -94,6 +92,10 @@ export VLLM_ENGINE_READY_TIMEOUT_S=3600 # vllm-project/vllm#44774 applies the same reachability policy to Mooncake's # store mask. 32k matches the trace-replay tuning validated for this workload. export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 +export VLLM_USE_V2_MODEL_RUNNER=1 +export VLLM_USE_RUST_FRONTEND=1 +export VLLM_DSV4_MEGA_FP8_COMBINE=1 +export VLLM_RPC_TIMEOUT=600000 # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" @@ -106,8 +108,34 @@ ROUTER_PID="" MOONCAKE_MASTER_PID="" OFFLOAD_ARGS=() - -if require_agentic_kv_offload_backend mooncake; then +case "$KV_OFFLOAD_BACKEND" in + "") + require_agentic_kv_offload_none + ;; + vllm-simple) + require_agentic_kv_offload_backend vllm-simple + CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT )) + # Identical prefixes must hash to identical block keys across DP ranks. + export PYTHONHASHSEED=42 + OFFLOAD_CONFIG=$(cat <&2 + exit 1 + ;; +esac PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) if [ "$DP_ATTENTION" = "true" ]; then @@ -175,8 +206,14 @@ if [ "$DP_ATTENTION" = "true" ]; then fi EP_ARGS=() +FAST_MOE_ARGS=() if [ "$EP_SIZE" -gt 1 ]; then EP_ARGS=(--enable-expert-parallel) + FAST_MOE_ARGS=( + --moe-backend deep_gemm_amxf4_mega_moe + --enable-ep-weight-filter + --prefill-schedule-interval 16 + ) fi # AgentX concurrency counts live session trees, not individual requests. @@ -197,18 +234,23 @@ VLLM_CMD=( --trust-remote-code --kv-cache-dtype fp8 --block-size 256 - "${PARALLEL_ARGS[@]}" - "${VLLM_CP_ARGS[@]}" - "${EP_ARGS[@]}" - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - --attention_config.use_fp4_indexer_cache=True + --max-model-len 1048576 + --gpu-memory-utilization 0.92 + --numa-bind + --enable-cumem-allocator + --no-enable-flashinfer-autotune --tokenizer-mode deepseek_v4 - --tool-call-parser deepseek_v4 - --enable-auto-tool-choice --reasoning-parser deepseek_v4 - --enable-prefix-caching + --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' --no-disable-hybrid-kv-cache-manager + --disable-uvicorn-access-log + --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' --max-num-seqs "$MAX_NUM_SEQS" + --max-cudagraph-capture-size "$MAX_NUM_SEQS" + "${PARALLEL_ARGS[@]}" + "${VLLM_CP_ARGS[@]}" + "${EP_ARGS[@]}" + "${FAST_MOE_ARGS[@]}" "${OFFLOAD_ARGS[@]}" ) printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 73327351e..854675bfb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1771,7 +1771,7 @@ dsv4-fp4-b200-vllm: - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 } dsv4-fp4-b200-vllm-agentic: - image: vllm/vllm-openai:v0.23.0 + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b200-dgxc @@ -1782,14 +1782,12 @@ dsv4-fp4-b200-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - # Pure TP is only competitive at very low concurrency. - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - # Sample the useful MooncakeStore range without repeating its collapsed - # high-concurrency tail. - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8, 10, 16] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 50], router: { name: vllm-router, version: "0.1.14" } } - # Retain the external-cache transition and peak-throughput region. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [16, 38, 44, 56, 64, 66, 68], router: { name: vllm-router, version: "0.1.14" } } + # Pure TP is GPU-resident (cache fits) at low concurrency. + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 12, 16] } + # DEP compares SimpleCPUOffloadConnector and MooncakeStoreConnector. + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 68, 72], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 68, 72], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cd0320fa9..6cd7db1cf 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4788,3 +4788,11 @@ - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2222 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic + description: + - "Update B200 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs." + - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec" + - "B200: GPU-resident TP8 at conc [1,2,4,6,8] and SimpleCPU-offload TP8 at conc [8,12,16]; DEP8 at conc [8,16,24,32,40,48,56,64,68,72] with both SimpleCPU and Mooncake 0.3.11.post1." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2202 From 64973b30c6a8bfb71f36707ad2d2e78fccb1cb42 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 15 Jul 2026 11:22:39 -0500 Subject: [PATCH 2/3] docs(changelog): link B200 replacement PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Point the split B200 changelog entry to PR #2224. 中文：将拆分后的 B200 变更日志条目链接到 PR #2224。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6cd7db1cf..ac3c75678 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4795,4 +4795,4 @@ - "Update B200 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs." - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec" - "B200: GPU-resident TP8 at conc [1,2,4,6,8] and SimpleCPU-offload TP8 at conc [8,12,16]; DEP8 at conc [8,16,24,32,40,48,56,64,68,72] with both SimpleCPU and Mooncake 0.3.11.post1." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2202 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2224 From 455eabceeee9ac22a616c70b644afff14ea11876 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 15 Jul 2026 19:48:08 +0000 Subject: [PATCH 3/3] chore: fix mooncake dev and conc --- benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh | 2 +- configs/nvidia-master.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index a16114a49..de5947e36 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -155,7 +155,7 @@ EOF "global_segment_size": "${PER_RANK_GB}GB", "local_buffer_size": "4GB", "protocol": "rdma", - "device_name": "", + "device_name": "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11", "enable_offload": false } EOF diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 854675bfb..c7b2f5223 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1782,12 +1782,12 @@ dsv4-fp4-b200-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - # Pure TP is GPU-resident (cache fits) at low concurrency. + # Pure TP at low concurrency. - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8] } - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 12, 16] } - # DEP compares SimpleCPUOffloadConnector and MooncakeStoreConnector. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 68, 72], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 68, 72], router: { name: vllm-router, version: "0.1.14" } } + # DEP + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 68, 72, 80], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 @@ -3001,7 +3001,7 @@ dsr1-fp8-b200-trt: osl: 1024 search-space: - { tp: 8, ep: 1, conc-start: 64, conc-end: 128 } - - { tp: 4, ep: 1, conc-start: 8, conc-end: 16 } + - { tp: 4, ep: 1, conc-start: 8, conc-end: 16 } - { tp: 8, ep: 1, conc-start: 4, conc-end: 8 } - isl: 8192 osl: 1024