From fee87d8cffaf803f6dcff667fa83f3d2f3e13fa9 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Wed, 15 Jul 2026 11:20:53 -0500
Subject: [PATCH 1/7] feat(agentic): split DeepSeek-V4 B300 vLLM recipe
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Carry the B300-only launcher, config search space, and changelog scope from PR #2202.

中文：拆分 DeepSeek-V4 B300 vLLM AgentX 配方，仅保留 PR #2202 中的 B300 启动器、配置搜索空间和变更日志范围。
---
 .../single_node/agentic/dsv4_fp4_b300_vllm.sh | 218 +++++++++++-------
 configs/nvidia-master.yaml                    |  17 +-
 perf-changelog.yaml                           |   8 +
 3 files changed, 144 insertions(+), 99 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index 3a5be7b57..72f2736fe 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -1,60 +1,46 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 set -x
 
 # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM.
-# Mirrors the fixed-seq-len parallelism options (pure TP and DEP) so the
-# agentic sweep can probe both interactivity and throughput regimes:
-#   pure TP (DP_ATTENTION=false, EP_SIZE=1):  attention TP-sharded across
-#       all $TP GPUs in a single engine. Lower TPOT, lower batch.
-#   TP+EP   (DP_ATTENTION=false, EP_SIZE>1):  attention TP-sharded, MoE
-#       experts EP-sharded within the TP group.
-#   DEP     (DP_ATTENTION=true, EP_SIZE>1):   per-DP-rank attention with
-#       experts EP-sharded across DP ranks (per the vLLM blog recipe).
-#       Highest aggregate throughput at large CONC.
+# v4pro-b300.yaml TP4 and DEP4 recipe. SimpleCPUOffload / MooncakeStore
 #
-# Image is vllm/vllm-openai:v0.20.0-cu130. block_size=256, kv-cache-dtype=fp8,
-# FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph capture with
-# custom_ops=all (per the vLLM blog recipe at https://vllm.ai/blog/deepseek-v4).
+# Image is configured in nvidia-master.yaml. The recipe uses FP8 KV cache,
+# sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer cache, mega-MoE,
+# and FULL_DECODE_ONLY CUDA graphs with every batch size captured explicitly.
 #
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake.
+# KV_OFFLOADING=none is used by TP4. DEP4 uses KV_OFFLOADING=dram with
+# KV_OFFLOAD_BACKEND=vllm-simple or mooncake.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
 check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
 
-DCP_SIZE="${DCP_SIZE:-1}"
-PCP_SIZE="${PCP_SIZE:-1}"
-VLLM_CP_ARGS=()
-if [ "$DCP_SIZE" -gt 1 ]; then
-    VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE")
-fi
-if [ "$PCP_SIZE" -gt 1 ]; then
-    VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE")
-fi
-
-GPU_COUNT="${GPU_COUNT:-$((TP * PCP_SIZE))}"
+GPU_COUNT=$TP
 if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then
     echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2
     exit 1
 fi
 export GPU_COUNT
 
-if declare -p SLURM_JOB_ID >/dev/null 2>&1 && [ -n "$SLURM_JOB_ID" ]; then
-    SLURM_NODE=unknown
-    if declare -p SLURMD_NODENAME >/dev/null 2>&1 && [ -n "$SLURMD_NODENAME" ]; then
-        SLURM_NODE="$SLURMD_NODENAME"
-    fi
-    echo "JOB $SLURM_JOB_ID running on $SLURM_NODE"
+# Under DP-attention the DP world size equals TP, and the DEP recipe sizes
+# per-rank batch as MAX_NUM_SEQS = 2*CONC/TP, which must be an integer.
+if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then
+    echo "Error: DEP requires 2*CONC divisible by TP, got CONC='$CONC' and TP='$TP'" >&2
+    exit 1
+fi
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
 fi
 
 # `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
+# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE.
 # Either way, MODEL_PATH is what the server is launched with.
-if declare -p MODEL_PATH >/dev/null 2>&1 && [ -n "$MODEL_PATH" ]; then
+if [[ -n "$MODEL_PATH" ]]; then
     if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
         hf download "$MODEL" --local-dir "$MODEL_PATH"
     fi
@@ -68,17 +54,9 @@ nvidia-smi
 resolve_trace_source
 install_agentic_deps
 
-# vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings,
-# which fails DSV4 indexer compilation with mlir_global_dtors(..., data).
-# Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved.
-agentic_pip_install --quiet --force-reinstall --no-deps \
-    'nvidia-cutlass-dsl-libs-cu13==4.5.2'
-
 # vllm-project/router expands the one HTTP backend into one logical worker per
-# DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's
-# X-Correlation-ID is stable for every turn of a conversation; alias it to the
-# router's preferred X-Session-ID header. This also keeps affinity correct when
-# testing older wheels that prioritize per-request X-Request-ID.
+# DP rank. Bind every turn of a conversation to the same rank by mapping
+# AIPerf's stable correlation ID to the router's X-Session-ID header.
 USE_VLLM_ROUTER=false
 VLLM_BACKEND_PORT="$PORT"
 if [ "$DP_ATTENTION" = "true" ]; then
@@ -91,13 +69,13 @@ if [ "$DP_ATTENTION" = "true" ]; then
     agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION"
 fi
 
-# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s.
+# Match the environment used by v4pro-b300.yaml.
+export VLLM_USE_V2_MODEL_RUNNER=1
 export VLLM_ENGINE_READY_TIMEOUT_S=3600
-
-# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while
-# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's
-# store mask. 32k matches the trace-replay tuning validated for this workload.
 export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
+export VLLM_DSV4_MEGA_FP8_COMBINE=1
+export NCCL_NVLS_ENABLE=1
+export VLLM_USE_RUST_FRONTEND=1
 
 # ---- Server config ----------------------------------------------------------
 SERVER_LOG="$RESULT_DIR/server.log"
@@ -109,13 +87,40 @@ SERVER_PID=""
 ROUTER_PID=""
 MOONCAKE_MASTER_PID=""
 
+# The generated TOTAL_CPU_DRAM_GB budget is proportional to allocated GPUs.
+# On cluster:b300-nv, dram-utilization=0.80 and DEP4 resolve to roughly the
+# source recipe's 280 GiB per DP rank. TP4 remains GPU-resident.
 OFFLOAD_ARGS=()
-if require_agentic_kv_offload_backend mooncake; then
-        # Mooncake embedded mode contributes one global segment per GPU rank to
-        # a shared distributed store. Pre-divide the aggregate host budget
-        # across those rank-contributed segments.
+case "$KV_OFFLOAD_BACKEND" in
+    "")
+        require_agentic_kv_offload_none
+        ;;
+    vllm-simple)
+        require_agentic_kv_offload_backend vllm-simple
+        CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT ))
+        # Identical prefixes must hash to identical block keys across DP ranks.
+        export PYTHONHASHSEED=42
+        OFFLOAD_CONFIG=$(cat <<EOF
+{
+  "kv_connector": "SimpleCPUOffloadConnector",
+  "kv_role": "kv_both",
+  "kv_connector_extra_config": {
+    "cpu_bytes_to_use": ${CPU_BYTES_PER_RANK},
+    "enable_cross_layers_blocks": "true"
+  }
+}
+EOF
+)
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "$OFFLOAD_CONFIG"
+        )
+        ;;
+    mooncake)
+        require_agentic_kv_offload_backend mooncake
+        # Embedded mode contributes one global segment per DP rank to the
+        # shared store, so divide the aggregate host budget across ranks.
         PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT))
-
         MOONCAKE_VERSION=0.3.11.post1
         agentic_pip_install --quiet --no-cache-dir --no-deps \
             --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
@@ -139,9 +144,7 @@ EOF
         export MC_ENABLE_DEST_DEVICE_AFFINITY=1
         # Identical prefixes must hash to identical store keys across DP ranks.
         export PYTHONHASHSEED=0
-        # Large agentic KV writes can exceed Mooncake Store's fixed 60-second
-        # transfer deadline at the default 64 KiB RDMA slice size. Reduce
-        # per-transfer bookkeeping and give the shared RNIC more workers.
+        export WITH_NVIDIA_PEERMEM=0
         export MC_SLICE_SIZE=1048576
         export MC_WORKERS_PER_CTX=4
 
@@ -165,54 +168,93 @@ EOF
         fi
 
         unset VLLM_USE_SIMPLE_KV_OFFLOAD
-        OFFLOAD_ARGS=(
-            --kv-transfer-config
-            '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
-        )
-fi
+        OFFLOAD_CONFIG='{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
+        OFFLOAD_ARGS=(--kv-transfer-config "$OFFLOAD_CONFIG")
+        ;;
+    *)
+        echo "Error: unsupported B300 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2
+        exit 1
+        ;;
+esac
 
 PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
 if [ "$DP_ATTENTION" = "true" ]; then
     PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
 fi
 
-EP_ARGS=()
+TP_ARGS=()
+if [ "$DP_ATTENTION" = "true" ]; then
+    export PYTORCH_ALLOC_CONF=expandable_segments:True
+else
+    export VLLM_ALLREDUCE_USE_FLASHINFER=1
+    export VLLM_FLASHINFER_ALLREDUCE_BACKEND=auto
+    TP_ARGS+=(--disable-custom-all-reduce)
+fi
+
+MODE_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
-    EP_ARGS=(--enable-expert-parallel)
+    MODE_ARGS+=(
+        --enable-expert-parallel
+        --enable-ep-weight-filter
+        --moe-backend deep_gemm_amxf4_mega_moe
+    )
+fi
+if [ "$DP_ATTENTION" = "true" ]; then
+    MODE_ARGS+=(
+        --prefill-schedule-interval 8
+        --max-num-batched-tokens 8192
+    )
 fi
 
-# AgentX concurrency counts live session trees, not individual requests.
-# Subagent fan-out can push instantaneous request concurrency above CONC, so
-# leave 2x headroom rather than clipping those bursts at the scheduler.
-MAX_NUM_SEQS=$((2 * CONC))
-if [ "$MAX_NUM_SEQS" -eq 128 ]; then
-    MAX_NUM_SEQS=136
+if [ "$DP_ATTENTION" = "true" ]; then
+    # The DEP source recipe enforces 2*CONC = DP_WORLD_SIZE*MAX_NUM_SEQS.
+    MAX_NUM_SEQS=$((2 * CONC / TP))
+else
+    # Preserve the previous TP4 scheduler headroom for agentic fan-out.
+    MAX_NUM_SEQS=$((2 * CONC))
 fi
+CUDA_GRAPH_CAPTURE_SIZES=""
+for ((capture_size = 1; capture_size <= MAX_NUM_SEQS; capture_size++)); do
+    if [ -n "$CUDA_GRAPH_CAPTURE_SIZES" ]; then
+        CUDA_GRAPH_CAPTURE_SIZES+=","
+    fi
+    CUDA_GRAPH_CAPTURE_SIZES+="$capture_size"
+done
+COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[${CUDA_GRAPH_CAPTURE_SIZES}],\"mode\":0}"
 
 echo "Starting vllm server..."
 export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
 export VLLM_FLOAT32_MATMUL_PRECISION=high
 
-vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
---host 0.0.0.0 \
---port "$VLLM_BACKEND_PORT" \
---trust-remote-code \
---kv-cache-dtype fp8 \
---block-size 256 \
-"${PARALLEL_ARGS[@]}" \
-"${VLLM_CP_ARGS[@]}" \
-"${EP_ARGS[@]}" \
---compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \
---attention_config.use_fp4_indexer_cache=True \
---tokenizer-mode deepseek_v4 \
---tool-call-parser deepseek_v4 \
---enable-auto-tool-choice \
---reasoning-parser deepseek_v4 \
---enable-prefix-caching \
---no-disable-hybrid-kv-cache-manager \
---max-num-seqs "$MAX_NUM_SEQS" \
-"${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 &
+{ set +x; } 2>/dev/null
+VLLM_CMD=(
+    vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$VLLM_BACKEND_PORT"
+    --gpu-memory-utilization 0.96
+    --trust-remote-code
+    --no-enable-flashinfer-autotune
+    --no-disable-hybrid-kv-cache-manager
+    --max-num-seqs "$MAX_NUM_SEQS"
+    --kv-cache-dtype fp8
+    --block-size 256
+    --max-model-len 1048576
+    --attention-config '{"use_fp4_indexer_cache":true,"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}'
+    --disable-uvicorn-access-log
+    --tokenizer-mode deepseek_v4
+    --tool-call-parser deepseek_v4
+    --enable-auto-tool-choice
+    --reasoning-parser deepseek_v4
+    --compilation-config "$COMPILATION_CONFIG"
+    "${PARALLEL_ARGS[@]}"
+    "${TP_ARGS[@]}"
+    "${MODE_ARGS[@]}"
+    "${OFFLOAD_ARGS[@]}"
+)
+printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
+"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
 SERVER_PID=$!
 echo "Server PID: $SERVER_PID"
 
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 73327351e..f598c300c 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3217,7 +3217,7 @@ dsv4-fp4-b300-vllm:
       - { tp: 8, ep: 8, dp-attn: true, conc-start: 2048, conc-end: 2048 }
 
 dsv4-fp4-b300-vllm-agentic:
-  image: vllm/vllm-openai:v0.23.0
+  image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:b300-nv
@@ -3228,16 +3228,11 @@ dsv4-fp4-b300-vllm-agentic:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      # Compare native GPU-cache and MooncakeStore CPU-offload cliffs.
-      - { tp: 4, kv-offloading: none,  conc-list: [1, 2, 4, 6, 8, 16] }
-      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [16, 18, 20, 24] }
-      # TP8 remains cache-resident through conc 52.
-      - { tp: 8, kv-offloading: none,  conc-list: [1, 2, 4, 6, 8, 16, 32, 40, 48, 52] }
-      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [52] }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none,  conc-list: [8, 16], router: { name: vllm-router, version: "0.1.14" } }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [32], router: { name: vllm-router, version: "0.1.14" } }
-      # TP8 DEP retains representative low, peak, and transition points.
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,  conc-list: [52, 72, 100, 128, 144], router: { name: vllm-router, version: "0.1.14" } }
+      # Preserve the previous GPU-resident TP4 search space on the new image.
+      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4] }
+      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index cd0320fa9..c3f9f9d8f 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4788,3 +4788,11 @@
     - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)."
     - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2222
+
+- config-keys:
+    - dsv4-fp4-b300-vllm-agentic
+  description:
+    - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
+    - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
+    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2202

From 2d1725bf4b170b00169675f5d5c57302a3bb13e0 Mon Sep 17 00:00:00 2001
From: Cam Quilici <cjquilici@gmail.com>
Date: Wed, 15 Jul 2026 11:22:39 -0500
Subject: [PATCH 2/7] docs(changelog): link B300 replacement PR
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Point the split B300 changelog entry to PR #2225.

中文：将拆分后的 B300 变更日志条目链接到 PR #2225。
---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index c3f9f9d8f..edae9793a 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4795,4 +4795,4 @@
     - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
     - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2202
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225

From 407404b991e77aabfb5cc9f6d5a75ce9aadbfc4b Mon Sep 17 00:00:00 2001
From: Yifan Qiao <yifanqiao@inferact.ai>
Date: Wed, 15 Jul 2026 19:38:02 +0000
Subject: [PATCH 3/7] chore: update conc

---
 configs/nvidia-master.yaml | 8 +++++---
 perf-changelog.yaml        | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index f598c300c..ad03cc091 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3003,7 +3003,7 @@ dsr1-fp8-b200-trt:
       osl: 1024
       search-space:
       - { tp: 8, ep: 1, conc-start: 64, conc-end: 128 }
-      - { tp: 4, ep: 1, conc-start: 8, conc-end: 16 } 
+      - { tp: 4, ep: 1, conc-start: 8, conc-end: 16 }
       - { tp: 8, ep: 1, conc-start: 4, conc-end: 8 }
     - isl: 8192
       osl: 1024
@@ -3229,10 +3229,12 @@ dsv4-fp4-b300-vllm-agentic:
     - dram-utilization: 0.80
       search-space:
       # Preserve the previous GPU-resident TP4 search space on the new image.
-      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4] }
+      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
       - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 64, 96, 128, 160, 192, 224, 228], router: { name: vllm-router, version: "0.1.14" } }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [40, 72, 104, 136, 168, 200], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index edae9793a..abc9f1a7e 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4794,5 +4794,5 @@
   description:
     - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
-    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
+    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16]; DEP4 at conc [8,16,24,32,40,48,56,64,72]; DEP8 at conc [32,64,96,128,160,192,224,228]; with both SimpleCPU and Mooncake 0.3.11.post1."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225

From 29b6fa5882280d3737087ea730d02e5a7e8afd6b Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Wed, 15 Jul 2026 17:54:42 -0700
Subject: [PATCH 4/7] update dep8 args

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
---
 .../single_node/agentic/dsv4_fp4_b300_vllm.sh | 37 +++++++++++++++----
 configs/nvidia-master.yaml                    |  4 +-
 perf-changelog.yaml                           |  2 +-
 3 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index 72f2736fe..0f31a1881 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -3,7 +3,7 @@ set -eo pipefail
 set -x
 
 # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM.
-# v4pro-b300.yaml TP4 and DEP4 recipe. SimpleCPUOffload / MooncakeStore
+# v4pro-b300.yaml TP4, DEP4, and DEP8 recipe. SimpleCPUOffload / MooncakeStore
 #
 # Image is configured in nvidia-master.yaml. The recipe uses FP8 KV cache,
 # sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer cache, mega-MoE,
@@ -12,8 +12,8 @@ set -x
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# KV_OFFLOADING=none is used by TP4. DEP4 uses KV_OFFLOADING=dram with
-# KV_OFFLOAD_BACKEND=vllm-simple or mooncake.
+# TP4, TP8, and DEP8 (TP8 + DP-attention) are GPU-resident (KV_OFFLOADING=none).
+# DEP4 uses KV_OFFLOADING=dram with KV_OFFLOAD_BACKEND=vllm-simple or mooncake.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -33,6 +33,14 @@ if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then
     exit 1
 fi
 
+# DEP8 (TP8 + DP-attention) is a GPU-resident, high-concurrency arm that is
+# tuned separately from the smaller DEP4 arm (larger prefill token budget,
+# long-prefill chunking, and a lower GPU-memory-utilization headroom).
+IS_DEP8=false
+if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then
+    IS_DEP8=true
+fi
+
 if [[ -n "$SLURM_JOB_ID" ]]; then
     echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
 fi
@@ -200,10 +208,17 @@ if [ "$EP_SIZE" -gt 1 ]; then
     )
 fi
 if [ "$DP_ATTENTION" = "true" ]; then
-    MODE_ARGS+=(
-        --prefill-schedule-interval 8
-        --max-num-batched-tokens 8192
-    )
+    MODE_ARGS+=(--prefill-schedule-interval 8)
+    if [ "$IS_DEP8" = "true" ]; then
+        # GPU-resident DEP8 gets a larger prefill token budget and chunks long
+        # prefills so decode latency stays bounded at high concurrency.
+        MODE_ARGS+=(
+            --max-num-batched-tokens 16384
+            --long-prefill-token-threshold 4096
+        )
+    else
+        MODE_ARGS+=(--max-num-batched-tokens 8192)
+    fi
 fi
 
 if [ "$DP_ATTENTION" = "true" ]; then
@@ -227,12 +242,18 @@ export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
 export VLLM_FLOAT32_MATMUL_PRECISION=high
 
+# DEP8 leaves slightly more headroom for its larger prefill token budget.
+GPU_MEM_UTIL=0.96
+if [ "$IS_DEP8" = "true" ]; then
+    GPU_MEM_UTIL=0.95
+fi
+
 { set +x; } 2>/dev/null
 VLLM_CMD=(
     vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
     --host 0.0.0.0
     --port "$VLLM_BACKEND_PORT"
-    --gpu-memory-utilization 0.96
+    --gpu-memory-utilization "$GPU_MEM_UTIL"
     --trust-remote-code
     --no-enable-flashinfer-autotune
     --no-disable-hybrid-kv-cache-manager
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index ad03cc091..926f582ab 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3233,8 +3233,8 @@ dsv4-fp4-b300-vllm-agentic:
       - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } }
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 64, 96, 128, 160, 192, 224, 228], router: { name: vllm-router, version: "0.1.14" } }
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [40, 72, 104, 136, 168, 200], router: { name: vllm-router, version: "0.1.14" } }
+      # DEP8 is GPU-resident (no KV offload) with a DEP8-tuned server config.
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [32, 64, 96, 128, 160, 192, 196, 224, 228], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index abc9f1a7e..4711b0993 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4794,5 +4794,5 @@
   description:
     - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
-    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16]; DEP4 at conc [8,16,24,32,40,48,56,64,72]; DEP8 at conc [32,64,96,128,160,192,224,228]; with both SimpleCPU and Mooncake 0.3.11.post1."
+    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.95); DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225

From 5d3da845b31d993b436ac6afccb89afcb384fc09 Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Thu, 16 Jul 2026 00:54:48 -0700
Subject: [PATCH 5/7] add more tp configs

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
---
 benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh | 9 ++++++++-
 configs/nvidia-master.yaml                           | 6 ++++--
 perf-changelog.yaml                                  | 2 +-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index 0f31a1881..bfae06d7c 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -108,13 +108,20 @@ case "$KV_OFFLOAD_BACKEND" in
         CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT ))
         # Identical prefixes must hash to identical block keys across DP ranks.
         export PYTHONHASHSEED=42
+        # The plain-TP (non-DP-attention) offload ladder uses lazy offload;
+        # DEP keeps eager offload for cross-rank block-hash stability.
+        SIMPLE_LAZY_OFFLOAD=false
+        if [ "$DP_ATTENTION" != "true" ]; then
+            SIMPLE_LAZY_OFFLOAD=true
+        fi
         OFFLOAD_CONFIG=$(cat <<EOF
 {
   "kv_connector": "SimpleCPUOffloadConnector",
   "kv_role": "kv_both",
   "kv_connector_extra_config": {
     "cpu_bytes_to_use": ${CPU_BYTES_PER_RANK},
-    "enable_cross_layers_blocks": "true"
+    "enable_cross_layers_blocks": "true",
+    "lazy_offload": ${SIMPLE_LAZY_OFFLOAD}
   }
 }
 EOF
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 926f582ab..831a29d2c 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3229,8 +3229,10 @@ dsv4-fp4-b300-vllm-agentic:
     - dram-utilization: 0.80
       search-space:
       # Preserve the previous GPU-resident TP4 search space on the new image.
-      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
-      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] }
+      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32] }
+      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32] }
+      # TP4 SimpleCPU lazy-offload ladder (extends the GPU-resident TP4 arm).
+      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [28, 32, 36, 40] }
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } }
       # DEP8 is GPU-resident (no KV offload) with a DEP8-tuned server config.
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 4711b0993..fc63d2067 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4794,5 +4794,5 @@
   description:
     - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
-    - "B300: GPU-resident TP4 at conc [1,2,4,6,8,16] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.95); DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
+    - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.95); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225

From 06101fe2464e98b0308c2f9b15a136ecfceceada Mon Sep 17 00:00:00 2001
From: Jeff Ma <jeffjma@umich.edu>
Date: Thu, 16 Jul 2026 00:58:39 -0700
Subject: [PATCH 6/7] dep8 reduce gpu mem util to 0.92

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
---
 benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh | 4 ++--
 perf-changelog.yaml                                  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
index bfae06d7c..d4c0ca054 100755
--- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -249,10 +249,10 @@ export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
 export VLLM_FLOAT32_MATMUL_PRECISION=high
 
-# DEP8 leaves slightly more headroom for its larger prefill token budget.
+# DEP8 leaves more headroom for its larger prefill token budget.
 GPU_MEM_UTIL=0.96
 if [ "$IS_DEP8" = "true" ]; then
-    GPU_MEM_UTIL=0.95
+    GPU_MEM_UTIL=0.92
 fi
 
 { set +x; } 2>/dev/null
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index fc63d2067..eeb8f3f32 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4794,5 +4794,5 @@
   description:
     - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
     - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
-    - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.95); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
+    - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.92); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2225

From 0a1612d917b49f5eec5f606aee549dd4f28bfc3d Mon Sep 17 00:00:00 2001
From: Yifan Qiao <yifanqiao@inferact.ai>
Date: Thu, 16 Jul 2026 08:43:49 +0000
Subject: [PATCH 7/7] chore: update conc list

---
 configs/nvidia-master.yaml | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 831a29d2c..d8e31a188 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -3228,15 +3228,14 @@ dsv4-fp4-b300-vllm-agentic:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      # Preserve the previous GPU-resident TP4 search space on the new image.
-      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32] }
+      # TP4 GPU-resident
       - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32] }
-      # TP4 SimpleCPU lazy-offload ladder (extends the GPU-resident TP4 arm).
+      # TP4 SimpleCPU
       - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [28, 32, 36, 40] }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } }
-      # DEP8 is GPU-resident (no KV offload) with a DEP8-tuned server config.
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [32, 64, 96, 128, 160, 192, 196, 224, 228], router: { name: vllm-router, version: "0.1.14" } }
+      # DEP4 SimpleCPU
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
+      # DEP8 SimpleCPU
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066