diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh index 3a5be7b57..72f2736fe 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh @@ -1,60 +1,46 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail set -x # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM. -# Mirrors the fixed-seq-len parallelism options (pure TP and DEP) so the -# agentic sweep can probe both interactivity and throughput regimes: -# pure TP (DP_ATTENTION=false, EP_SIZE=1): attention TP-sharded across -# all $TP GPUs in a single engine. Lower TPOT, lower batch. -# TP+EP (DP_ATTENTION=false, EP_SIZE>1): attention TP-sharded, MoE -# experts EP-sharded within the TP group. -# DEP (DP_ATTENTION=true, EP_SIZE>1): per-DP-rank attention with -# experts EP-sharded across DP ranks (per the vLLM blog recipe). -# Highest aggregate throughput at large CONC. +# v4pro-b300.yaml TP4 and DEP4 recipe. SimpleCPUOffload / MooncakeStore # -# Image is vllm/vllm-openai:v0.20.0-cu130. block_size=256, kv-cache-dtype=fp8, -# FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph capture with -# custom_ops=all (per the vLLM blog recipe at https://vllm.ai/blog/deepseek-v4). +# Image is configured in nvidia-master.yaml. The recipe uses FP8 KV cache, +# sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer cache, mega-MoE, +# and FULL_DECODE_ONLY CUDA graphs with every batch size captured explicitly. # # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. +# KV_OFFLOADING=none is used by TP4. DEP4 uses KV_OFFLOADING=dram with +# KV_OFFLOAD_BACKEND=vllm-simple or mooncake. source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -DCP_SIZE="${DCP_SIZE:-1}" -PCP_SIZE="${PCP_SIZE:-1}" -VLLM_CP_ARGS=() -if [ "$DCP_SIZE" -gt 1 ]; then - VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE") -fi -if [ "$PCP_SIZE" -gt 1 ]; then - VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE") -fi - -GPU_COUNT="${GPU_COUNT:-$((TP * PCP_SIZE))}" +GPU_COUNT=$TP if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2 exit 1 fi export GPU_COUNT -if declare -p SLURM_JOB_ID >/dev/null 2>&1 && [ -n "$SLURM_JOB_ID" ]; then - SLURM_NODE=unknown - if declare -p SLURMD_NODENAME >/dev/null 2>&1 && [ -n "$SLURMD_NODENAME" ]; then - SLURM_NODE="$SLURMD_NODENAME" - fi - echo "JOB $SLURM_JOB_ID running on $SLURM_NODE" +# Under DP-attention the DP world size equals TP, and the DEP recipe sizes +# per-rank batch as MAX_NUM_SEQS = 2*CONC/TP, which must be an integer. +if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then + echo "Error: DEP requires 2*CONC divisible by TP, got CONC='$CONC' and TP='$TP'" >&2 + exit 1 +fi + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi # `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE. # Either way, MODEL_PATH is what the server is launched with. -if declare -p MODEL_PATH >/dev/null 2>&1 && [ -n "$MODEL_PATH" ]; then +if [[ -n "$MODEL_PATH" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" fi @@ -68,17 +54,9 @@ nvidia-smi resolve_trace_source install_agentic_deps -# vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings, -# which fails DSV4 indexer compilation with mlir_global_dtors(..., data). -# Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved. -agentic_pip_install --quiet --force-reinstall --no-deps \ - 'nvidia-cutlass-dsl-libs-cu13==4.5.2' - # vllm-project/router expands the one HTTP backend into one logical worker per -# DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's -# X-Correlation-ID is stable for every turn of a conversation; alias it to the -# router's preferred X-Session-ID header. This also keeps affinity correct when -# testing older wheels that prioritize per-request X-Request-ID. +# DP rank. Bind every turn of a conversation to the same rank by mapping +# AIPerf's stable correlation ID to the router's X-Session-ID header. USE_VLLM_ROUTER=false VLLM_BACKEND_PORT="$PORT" if [ "$DP_ATTENTION" = "true" ]; then @@ -91,13 +69,13 @@ if [ "$DP_ATTENTION" = "true" ]; then agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION" fi -# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. +# Match the environment used by v4pro-b300.yaml. +export VLLM_USE_V2_MODEL_RUNNER=1 export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while -# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's -# store mask. 32k matches the trace-replay tuning validated for this workload. export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 +export VLLM_DSV4_MEGA_FP8_COMBINE=1 +export NCCL_NVLS_ENABLE=1 +export VLLM_USE_RUST_FRONTEND=1 # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" @@ -109,13 +87,40 @@ SERVER_PID="" ROUTER_PID="" MOONCAKE_MASTER_PID="" +# The generated TOTAL_CPU_DRAM_GB budget is proportional to allocated GPUs. +# On cluster:b300-nv, dram-utilization=0.80 and DEP4 resolve to roughly the +# source recipe's 280 GiB per DP rank. TP4 remains GPU-resident. OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then - # Mooncake embedded mode contributes one global segment per GPU rank to - # a shared distributed store. Pre-divide the aggregate host budget - # across those rank-contributed segments. +case "$KV_OFFLOAD_BACKEND" in + "") + require_agentic_kv_offload_none + ;; + vllm-simple) + require_agentic_kv_offload_backend vllm-simple + CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT )) + # Identical prefixes must hash to identical block keys across DP ranks. + export PYTHONHASHSEED=42 + OFFLOAD_CONFIG=$(cat <&2 + exit 1 + ;; +esac PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) if [ "$DP_ATTENTION" = "true" ]; then PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") fi -EP_ARGS=() +TP_ARGS=() +if [ "$DP_ATTENTION" = "true" ]; then + export PYTORCH_ALLOC_CONF=expandable_segments:True +else + export VLLM_ALLREDUCE_USE_FLASHINFER=1 + export VLLM_FLASHINFER_ALLREDUCE_BACKEND=auto + TP_ARGS+=(--disable-custom-all-reduce) +fi + +MODE_ARGS=() if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) + MODE_ARGS+=( + --enable-expert-parallel + --enable-ep-weight-filter + --moe-backend deep_gemm_amxf4_mega_moe + ) +fi +if [ "$DP_ATTENTION" = "true" ]; then + MODE_ARGS+=( + --prefill-schedule-interval 8 + --max-num-batched-tokens 8192 + ) fi -# AgentX concurrency counts live session trees, not individual requests. -# Subagent fan-out can push instantaneous request concurrency above CONC, so -# leave 2x headroom rather than clipping those bursts at the scheduler. -MAX_NUM_SEQS=$((2 * CONC)) -if [ "$MAX_NUM_SEQS" -eq 128 ]; then - MAX_NUM_SEQS=136 +if [ "$DP_ATTENTION" = "true" ]; then + # The DEP source recipe enforces 2*CONC = DP_WORLD_SIZE*MAX_NUM_SEQS. + MAX_NUM_SEQS=$((2 * CONC / TP)) +else + # Preserve the previous TP4 scheduler headroom for agentic fan-out. + MAX_NUM_SEQS=$((2 * CONC)) fi +CUDA_GRAPH_CAPTURE_SIZES="" +for ((capture_size = 1; capture_size <= MAX_NUM_SEQS; capture_size++)); do + if [ -n "$CUDA_GRAPH_CAPTURE_SIZES" ]; then + CUDA_GRAPH_CAPTURE_SIZES+="," + fi + CUDA_GRAPH_CAPTURE_SIZES+="$capture_size" +done +COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[${CUDA_GRAPH_CAPTURE_SIZES}],\"mode\":0}" echo "Starting vllm server..." export TORCH_CUDA_ARCH_LIST="10.0" export PYTHONNOUSERSITE=1 export VLLM_FLOAT32_MATMUL_PRECISION=high -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ ---host 0.0.0.0 \ ---port "$VLLM_BACKEND_PORT" \ ---trust-remote-code \ ---kv-cache-dtype fp8 \ ---block-size 256 \ -"${PARALLEL_ARGS[@]}" \ -"${VLLM_CP_ARGS[@]}" \ -"${EP_ARGS[@]}" \ ---compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ ---attention_config.use_fp4_indexer_cache=True \ ---tokenizer-mode deepseek_v4 \ ---tool-call-parser deepseek_v4 \ ---enable-auto-tool-choice \ ---reasoning-parser deepseek_v4 \ ---enable-prefix-caching \ ---no-disable-hybrid-kv-cache-manager \ ---max-num-seqs "$MAX_NUM_SEQS" \ -"${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$VLLM_BACKEND_PORT" + --gpu-memory-utilization 0.96 + --trust-remote-code + --no-enable-flashinfer-autotune + --no-disable-hybrid-kv-cache-manager + --max-num-seqs "$MAX_NUM_SEQS" + --kv-cache-dtype fp8 + --block-size 256 + --max-model-len 1048576 + --attention-config '{"use_fp4_indexer_cache":true,"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' + --disable-uvicorn-access-log + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 + --compilation-config "$COMPILATION_CONFIG" + "${PARALLEL_ARGS[@]}" + "${TP_ARGS[@]}" + "${MODE_ARGS[@]}" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! echo "Server PID: $SERVER_PID" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1f40b27fb..1cf99c58b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -3558,7 +3558,7 @@ dsv4-fp4-b300-vllm: - { tp: 8, ep: 8, dp-attn: true, conc-start: 2048, conc-end: 2048 } dsv4-fp4-b300-vllm-agentic: - image: vllm/vllm-openai:v0.23.0 + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b300-nv @@ -3569,16 +3569,13 @@ dsv4-fp4-b300-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - # Compare native GPU-cache and MooncakeStore CPU-offload cliffs. - - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [16, 18, 20, 24] } - # TP8 remains cache-resident through conc 52. - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16, 32, 40, 48, 52] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [52] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [32], router: { name: vllm-router, version: "0.1.14" } } - # TP8 DEP retains representative low, peak, and transition points. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144], router: { name: vllm-router, version: "0.1.14" } } + # Preserve the previous GPU-resident TP4 search space on the new image. + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] } + - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 64, 96, 128, 160, 192, 224, 228], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [40, 72, 104, 136, 168, 200], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b300-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8c02a14d6..34be81c6a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4894,3 +4894,12 @@ - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec" - "B200: GPU-resident TP8 at conc [1,2,4,6,8] and SimpleCPU-offload TP8 at conc [8,12,16]; DEP8 at conc [8,16,24,32,40,48,56,64,68,72] with both SimpleCPU and Mooncake 0.3.11.post1." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2224 + +- config-keys: + - dsv4-fp4-b300-vllm-agentic + description: + - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs." + - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec" + - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16]; DEP4 at conc [8,16,24,32,40,48,56,64,72]; DEP8 at conc [32,64,96,128,160,192,224,228]; with both SimpleCPU and Mooncake 0.3.11.post1." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225 +