diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index 35cc55c04..67678359b 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail set -x # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B200 using vLLM. @@ -14,21 +14,26 @@ set -x # Highest aggregate throughput at large CONC. # # Image is configured in nvidia-master.yaml. block_size=256, -# kv-cache-dtype=fp8, FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph -# capture with custom_ops=all (per the vLLM blog recipe at -# https://vllm.ai/blog/deepseek-v4). +# kv-cache-dtype=fp8, FLASHINFER_MLA_SPARSE_DSV4 attention with the FP4 indexer +# cache, FULL_DECODE_ONLY cudagraph capture, and (in EP tiers) mega-MoE backend. # # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. +# Pure TP is GPU-resident (KV_OFFLOADING=none). DEP tiers offload KV to host +# DRAM: KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-simple, mooncake, +# or lmcache. source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -DCP_SIZE="${DCP_SIZE:-1}" -PCP_SIZE="${PCP_SIZE:-1}" +if [ -z "$DCP_SIZE" ]; then + DCP_SIZE=1 +fi +if [ -z "$PCP_SIZE" ]; then + PCP_SIZE=1 +fi VLLM_CP_ARGS=() if [ "$DCP_SIZE" -gt 1 ]; then VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE") @@ -37,21 +42,21 @@ if [ "$PCP_SIZE" -gt 1 ]; then VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE") fi -GPU_COUNT="${GPU_COUNT:-$((TP * PCP_SIZE))}" +GPU_COUNT=$TP if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2 exit 1 fi export GPU_COUNT -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi # `hf download` creates the target dir if missing and is itself idempotent. # When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE # Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then +if [[ -n "$MODEL_PATH" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" fi @@ -65,12 +70,6 @@ nvidia-smi resolve_trace_source install_agentic_deps -# vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings, -# which fails DSV4 indexer compilation with mlir_global_dtors(..., data). -# Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved. -agentic_pip_install --quiet --force-reinstall --no-deps \ - 'nvidia-cutlass-dsl-libs-cu13==4.5.2' - # vllm-project/router expands the one HTTP backend into one logical worker per # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's # X-Correlation-ID is stable for every turn of a conversation; alias it to the @@ -94,20 +93,55 @@ export VLLM_ENGINE_READY_TIMEOUT_S=3600 # vllm-project/vllm#44774 applies the same reachability policy to Mooncake's # store mask. 32k matches the trace-replay tuning validated for this workload. export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 +export VLLM_USE_V2_MODEL_RUNNER=1 +export VLLM_USE_RUST_FRONTEND=1 +export VLLM_DSV4_MEGA_FP8_COMBINE=1 +export VLLM_RPC_TIMEOUT=600000 # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" ROUTER_LOG="$RESULT_DIR/router.log" MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" SERVER_PID="" ROUTER_PID="" MOONCAKE_MASTER_PID="" +LMCACHE_SERVER_PID="" OFFLOAD_ARGS=() - -if require_agentic_kv_offload_backend mooncake; then +# vLLM's cuMem/VMM allocator is the recipe default; the lmcache arm clears +# this (see the lmcache case below). +CUMEM_ARGS=(--enable-cumem-allocator) +case "$KV_OFFLOAD_BACKEND" in + "") + require_agentic_kv_offload_none + ;; + vllm-simple) + require_agentic_kv_offload_backend vllm-simple + CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT )) + # Identical prefixes must hash to identical block keys across DP ranks. + export PYTHONHASHSEED=42 + OFFLOAD_CONFIG=$(cat </dev/null + + LMCACHE_HOST=127.0.0.1 + LMCACHE_PORT=$((PORT + 12000)) + LMCACHE_HTTP_PORT=$((PORT + 13000)) + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector + # a ZMQ-style host string. + LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" + # Pool target derated to 75% of the aggregate budget: pinned host + # memory is unswappable and also consumes GPU-side mapping + # resources, so leave headroom for vLLM host buffers and the OS. + # Full-budget targets OOM-killed the node (host OOM-killer or + # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during + # PR #2153 bring-up. + LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4)) + # The pool grows lazily from the initial allocation, so the full + # --l1-size-gb target is not pinned at startup. + LMCACHE_L1_INIT_SIZE_GB=20 + LMCACHE_MQ_TIMEOUT=300 + # Identical prefixes must hash to identical cache keys across DP ranks. + export PYTHONHASHSEED=0 + # LMCacheMPConnector exports the KV cache to the LMCache server + # through legacy CUDA IPC handles, and cuMem/VMM allocations cannot + # be exported that way (register_kv_caches fails with + # cudaErrorInvalidValue), so drop the allocator for this arm only. + CUMEM_ARGS=() + # Per-engine scheduler stats every 5s, to diagnose per-DP-rank KV + # cache imbalance under the session-sticky router. + export VLLM_LOG_STATS_INTERVAL=5 + + echo "Starting LMCache MP server on port $LMCACHE_PORT..." + # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls + # under heavy async-load pressure; CPU-side workers stay at 8. + lmcache server \ + --host "$LMCACHE_HOST" \ + --port "$LMCACHE_PORT" \ + --http-host "$LMCACHE_HOST" \ + --http-port "$LMCACHE_HTTP_PORT" \ + --l1-size-gb "$LMCACHE_L1_SIZE_GB" \ + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \ + --max-gpu-workers 1 \ + --max-cpu-workers 8 \ + --chunk-size 1024 \ + --l1-align-bytes 16384 \ + --eviction-trigger-watermark 0.85 \ + --eviction-ratio 0.10 \ + --eviction-policy LRU \ + --supported-transfer-mode lmcache_driven \ + --no-separate-object-groups \ + > "$LMCACHE_SERVER_LOG" 2>&1 & + LMCACHE_SERVER_PID=$! + LMCACHE_READY=0 + for _ in $(seq 1 60); do + if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then + echo "LMCache server died during startup." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + if curl --output /dev/null --silent --fail \ + "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then + LMCACHE_READY=1 + break + fi + sleep 2 + done + if [ "$LMCACHE_READY" -ne 1 ]; then + echo "LMCache server did not become healthy in time." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}" + ) + ;; + *) + echo "Error: unsupported B200 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2 + exit 1 + ;; +esac PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) if [ "$DP_ATTENTION" = "true" ]; then @@ -175,8 +300,14 @@ if [ "$DP_ATTENTION" = "true" ]; then fi EP_ARGS=() +FAST_MOE_ARGS=() if [ "$EP_SIZE" -gt 1 ]; then EP_ARGS=(--enable-expert-parallel) + FAST_MOE_ARGS=( + --moe-backend deep_gemm_amxf4_mega_moe + --enable-ep-weight-filter + --prefill-schedule-interval 16 + ) fi # AgentX concurrency counts live session trees, not individual requests. @@ -197,18 +328,23 @@ VLLM_CMD=( --trust-remote-code --kv-cache-dtype fp8 --block-size 256 - "${PARALLEL_ARGS[@]}" - "${VLLM_CP_ARGS[@]}" - "${EP_ARGS[@]}" - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - --attention_config.use_fp4_indexer_cache=True + --max-model-len 1048576 + --gpu-memory-utilization 0.92 + --numa-bind + "${CUMEM_ARGS[@]}" + --no-enable-flashinfer-autotune --tokenizer-mode deepseek_v4 - --tool-call-parser deepseek_v4 - --enable-auto-tool-choice --reasoning-parser deepseek_v4 - --enable-prefix-caching + --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' --no-disable-hybrid-kv-cache-manager + --disable-uvicorn-access-log + --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' --max-num-seqs "$MAX_NUM_SEQS" + --max-cudagraph-capture-size "$MAX_NUM_SEQS" + "${PARALLEL_ARGS[@]}" + "${VLLM_CP_ARGS[@]}" + "${EP_ARGS[@]}" + "${FAST_MOE_ARGS[@]}" "${OFFLOAD_ARGS[@]}" ) printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e4e0a7a75..260c64213 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1771,7 +1771,30 @@ dsv4-fp4-b200-vllm: - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 } dsv4-fp4-b200-vllm-agentic: - image: vllm/vllm-openai:v0.23.0 + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b200-dgxc + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # Pure TP at low concurrency. + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 12, 16] } + # DEP + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 68, 72, 80], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } } + +# LMCache CPU-offload arm of dsv4-fp4-b200-vllm-agentic, split into its own +# config so it can be triggered/tested independently of the vllm-simple and +# Mooncake curves. Points mirror the vllm-simple offload ladder above on the +# same tuned image, so the backends are directly comparable. +dsv4-fp4-b200-vllm-agentic-lmcache: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b200-dgxc @@ -1782,14 +1805,8 @@ dsv4-fp4-b200-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - # Pure TP is only competitive at very low concurrency. - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - # Sample the useful MooncakeStore range without repeating its collapsed - # high-concurrency tail. - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8, 10, 16] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 50], router: { name: vllm-router, version: "0.1.14" } } - # Retain the external-cache transition and peak-throughput region. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [16, 38, 44, 56, 64, 66, 68], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [8, 12, 16] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 68, 72, 80], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 44b403ae9..04c1ef739 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4815,3 +4815,13 @@ - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2123 +- config-keys: + - dsv4-fp4-b200-vllm-agentic-lmcache + scenario-type: + - agentic-coding + description: + - "Add LMCache 0.5.1 DRAM KV-offload arm on the tuned B200 AgentX recipe from PR #2224/#2225 (sparse DSV4 FlashInfer attention, AMXF4 mega-MoE, FULL_DECODE_ONLY CUDA graphs) with image vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec" + - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector per PR #2153; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB; --enable-cumem-allocator dropped on the lmcache arm only (cuMem/VMM allocations cannot be CUDA-IPC-exported to the LMCache server)" + - "Points mirror the vllm-simple ladder: TP8 conc [8, 12, 16]; TP8-DEP8 conc [8, 16, 24, 32, 40, 48, 56, 64, 68, 72, 80]" + - "Parent dsv4-fp4-b200-vllm-agentic updated in the same change to the PR #2224 recipe and search space (image bump, vllm-simple + Mooncake arms) so the shared script stays consistent; the parent points are re-benchmarked by PR #2224 and not re-triggered here" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2231