diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index 35cc55c04..de5947e36 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail set -x # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B200 using vLLM. @@ -14,21 +14,25 @@ set -x # Highest aggregate throughput at large CONC. # # Image is configured in nvidia-master.yaml. block_size=256, -# kv-cache-dtype=fp8, FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph -# capture with custom_ops=all (per the vLLM blog recipe at -# https://vllm.ai/blog/deepseek-v4). +# kv-cache-dtype=fp8, FLASHINFER_MLA_SPARSE_DSV4 attention with the FP4 indexer +# cache, FULL_DECODE_ONLY cudagraph capture, and (in EP tiers) mega-MoE backend. # # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. +# Pure TP is GPU-resident (KV_OFFLOADING=none). DEP tiers offload KV to host +# DRAM: KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-simple or mooncake. source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -DCP_SIZE="${DCP_SIZE:-1}" -PCP_SIZE="${PCP_SIZE:-1}" +if [ -z "$DCP_SIZE" ]; then + DCP_SIZE=1 +fi +if [ -z "$PCP_SIZE" ]; then + PCP_SIZE=1 +fi VLLM_CP_ARGS=() if [ "$DCP_SIZE" -gt 1 ]; then VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE") @@ -37,21 +41,21 @@ if [ "$PCP_SIZE" -gt 1 ]; then VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE") fi -GPU_COUNT="${GPU_COUNT:-$((TP * PCP_SIZE))}" +GPU_COUNT=$TP if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2 exit 1 fi export GPU_COUNT -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi # `hf download` creates the target dir if missing and is itself idempotent. # When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE # Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then +if [[ -n "$MODEL_PATH" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" fi @@ -65,12 +69,6 @@ nvidia-smi resolve_trace_source install_agentic_deps -# vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings, -# which fails DSV4 indexer compilation with mlir_global_dtors(..., data). -# Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved. -agentic_pip_install --quiet --force-reinstall --no-deps \ - 'nvidia-cutlass-dsl-libs-cu13==4.5.2' - # vllm-project/router expands the one HTTP backend into one logical worker per # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's # X-Correlation-ID is stable for every turn of a conversation; alias it to the @@ -94,6 +92,10 @@ export VLLM_ENGINE_READY_TIMEOUT_S=3600 # vllm-project/vllm#44774 applies the same reachability policy to Mooncake's # store mask. 32k matches the trace-replay tuning validated for this workload. export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 +export VLLM_USE_V2_MODEL_RUNNER=1 +export VLLM_USE_RUST_FRONTEND=1 +export VLLM_DSV4_MEGA_FP8_COMBINE=1 +export VLLM_RPC_TIMEOUT=600000 # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" @@ -106,8 +108,34 @@ ROUTER_PID="" MOONCAKE_MASTER_PID="" OFFLOAD_ARGS=() - -if require_agentic_kv_offload_backend mooncake; then +case "$KV_OFFLOAD_BACKEND" in + "") + require_agentic_kv_offload_none + ;; + vllm-simple) + require_agentic_kv_offload_backend vllm-simple + CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT )) + # Identical prefixes must hash to identical block keys across DP ranks. + export PYTHONHASHSEED=42 + OFFLOAD_CONFIG=$(cat <&2 + exit 1 + ;; +esac PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) if [ "$DP_ATTENTION" = "true" ]; then @@ -175,8 +206,14 @@ if [ "$DP_ATTENTION" = "true" ]; then fi EP_ARGS=() +FAST_MOE_ARGS=() if [ "$EP_SIZE" -gt 1 ]; then EP_ARGS=(--enable-expert-parallel) + FAST_MOE_ARGS=( + --moe-backend deep_gemm_amxf4_mega_moe + --enable-ep-weight-filter + --prefill-schedule-interval 16 + ) fi # AgentX concurrency counts live session trees, not individual requests. @@ -197,18 +234,23 @@ VLLM_CMD=( --trust-remote-code --kv-cache-dtype fp8 --block-size 256 - "${PARALLEL_ARGS[@]}" - "${VLLM_CP_ARGS[@]}" - "${EP_ARGS[@]}" - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - --attention_config.use_fp4_indexer_cache=True + --max-model-len 1048576 + --gpu-memory-utilization 0.92 + --numa-bind + --enable-cumem-allocator + --no-enable-flashinfer-autotune --tokenizer-mode deepseek_v4 - --tool-call-parser deepseek_v4 - --enable-auto-tool-choice --reasoning-parser deepseek_v4 - --enable-prefix-caching + --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' --no-disable-hybrid-kv-cache-manager + --disable-uvicorn-access-log + --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' --max-num-seqs "$MAX_NUM_SEQS" + --max-cudagraph-capture-size "$MAX_NUM_SEQS" + "${PARALLEL_ARGS[@]}" + "${VLLM_CP_ARGS[@]}" + "${EP_ARGS[@]}" + "${FAST_MOE_ARGS[@]}" "${OFFLOAD_ARGS[@]}" ) printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 33e9118b7..1f40b27fb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1771,7 +1771,7 @@ dsv4-fp4-b200-vllm: - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 } dsv4-fp4-b200-vllm-agentic: - image: vllm/vllm-openai:v0.23.0 + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b200-dgxc @@ -1782,14 +1782,12 @@ dsv4-fp4-b200-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - # Pure TP is only competitive at very low concurrency. - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - # Sample the useful MooncakeStore range without repeating its collapsed - # high-concurrency tail. - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8, 10, 16] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 50], router: { name: vllm-router, version: "0.1.14" } } - # Retain the external-cache transition and peak-throughput region. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [16, 38, 44, 56, 64, 66, 68], router: { name: vllm-router, version: "0.1.14" } } + # Pure TP at low concurrency. + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 12, 16] } + # DEP + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 68, 72, 80], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 @@ -3346,7 +3344,7 @@ dsr1-fp8-b200-trt: osl: 1024 search-space: - { tp: 8, ep: 1, conc-start: 64, conc-end: 128 } - - { tp: 4, ep: 1, conc-start: 8, conc-end: 16 } + - { tp: 4, ep: 1, conc-start: 8, conc-end: 16 } - { tp: 8, ep: 1, conc-start: 4, conc-end: 8 } - isl: 8192 osl: 1024 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 761f55587..8c02a14d6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4887,3 +4887,10 @@ - "Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201 +- config-keys: + - dsv4-fp4-b200-vllm-agentic + description: + - "Update B200 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs." + - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec" + - "B200: GPU-resident TP8 at conc [1,2,4,6,8] and SimpleCPU-offload TP8 at conc [8,12,16]; DEP8 at conc [8,16,24,32,40,48,56,64,68,72] with both SimpleCPU and Mooncake 0.3.11.post1." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2224