SemiAnalysisAI · Oasis-Git · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026
diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh
@@ -56,6 +56,8 @@ install_agentic_deps
 SERVER_LOG="$RESULT_DIR/server.log"
 mkdir -p "$RESULT_DIR"
 
+export SGLANG_DEFAULT_THINKING=1
+export SGLANG_DSV4_REASONING_EFFORT=high
 export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1
 export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1
 
@@ -99,13 +101,20 @@ fi
 PARALLEL_ARGS=(--tp "$TP")
 METRICS_ARGS=(--enable-metrics)
 CHUNKED_PREFILL_SIZE=8192
+# DEP recipe splits by concurrency. The high-throughput tail (conc >= 54) uses
+# the SGLang cookbook recipe (chunked 65536 + prefill delayer + tighter mem/SWA
+# + cuda-graph-max-bs-decode 544, plus the 8192 tokens/rank cap that chunked
+# 65536 needs to stay on the DeepGEMM MoE path). conc < 54 keeps the
+# conservative recipe (chunked 32768) that avoids the pre-54 throughput cliff.
+DEP_HIGH_CONC=false
+[ "$DP_ATTENTION" = "true" ] && [ "$CONC" -ge 54 ] && DEP_HIGH_CONC=true
+DEP_EXTRA_ARGS=()
 if [ "$DP_ATTENTION" = "true" ]; then
     DEEPEP_CONFIG='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}'
     export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1
     export SGLANG_OPT_FIX_HASH_MEGA_MOE=1
     export SGLANG_OPT_USE_FAST_MASK_EP=1
     export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1
-    export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096
     export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1
     export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0
     PARALLEL_ARGS+=(
@@ -120,7 +129,14 @@ if [ "$DP_ATTENTION" = "true" ]; then
         --moe-a2a-backend deepep
         --deepep-config "$DEEPEP_CONFIG"
     )
-    CHUNKED_PREFILL_SIZE=32768
+    if [ "$DEP_HIGH_CONC" = "true" ]; then
+        export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8192
+        CHUNKED_PREFILL_SIZE=65536
+        DEP_EXTRA_ARGS=(--enable-prefill-delayer)
+    else
+        export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096
+        CHUNKED_PREFILL_SIZE=32768
+    fi
 else
     PARALLEL_ARGS+=(
         --moe-runner-backend flashinfer_mxfp4
@@ -134,13 +150,32 @@ MODEL_ARGS=()
 # DeepGEMM's DSv4 indexer needs a multi-GiB temporary allocation at long
 # contexts. Leave the same HBM headroom used by the B300 recipe so a nearly
 # full GPU KV cache does not OOM while HiCache is spilling to host memory.
-MEM_FRACTION_STATIC=0.88
+# The low-latency TP-only path (conc <= 16) runs GPU-only with no HiCache
+# spill, so it can take a larger static fraction for more KV headroom.
+if [ "$DP_ATTENTION" = "true" ]; then
+    if [ "$DEP_HIGH_CONC" = "true" ]; then
+        MEM_FRACTION_STATIC=0.85
+    else
+        MEM_FRACTION_STATIC=0.88
+    fi
+else
+    MEM_FRACTION_STATIC=0.90
+fi
 
 # AgentX concurrency counts live session trees, not individual requests.
 # Allow subagent fan-out to exceed CONC without clipping request bursts.
 MAX_RUNNING_REQUESTS=$((2 * CONC))
-CUDA_GRAPH_MAX_BS=$CONC
-[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64
+# The cookbook DEP tail captures a large decode graph (batch 544); other paths
+# scale the unified cuda-graph batch with CONC (capped at 64).
+if [ "$DEP_HIGH_CONC" = "true" ]; then
+    CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544)
+else
+    CUDA_GRAPH_MAX_BS=$CONC
+    [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64
+    CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS")
+fi
+SWA_FULL_TOKENS_RATIO=0.1
+[ "$DEP_HIGH_CONC" = "true" ] && SWA_FULL_TOKENS_RATIO=0.075
 
 export PYTHONNOUSERSITE=1
 export TORCH_CUDA_ARCH_LIST=10.0
@@ -175,10 +210,11 @@ SGLANG_CMD=(
     --trust-remote-code
     "${PARALLEL_ARGS[@]}"
     --mem-fraction-static "$MEM_FRACTION_STATIC"
-    --swa-full-tokens-ratio 0.1
+    --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO"
     --max-running-requests "$MAX_RUNNING_REQUESTS"
-    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    "${CUDA_GRAPH_ARGS[@]}"
     --chunked-prefill-size "$CHUNKED_PREFILL_SIZE"
+    "${DEP_EXTRA_ARGS[@]}"
     --tool-call-parser deepseekv4
     --reasoning-parser deepseek-v4
     --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja"

diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh
@@ -56,6 +56,8 @@ install_agentic_deps
 SERVER_LOG="$RESULT_DIR/server.log"
 mkdir -p "$RESULT_DIR"
 
+export SGLANG_DEFAULT_THINKING=1
+export SGLANG_DSV4_REASONING_EFFORT=high
 export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1
 export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1
 
@@ -105,7 +107,28 @@ PARALLEL_ARGS=(--tp "$TP")
 METRICS_ARGS=(--enable-metrics)
 MEM_FRACTION_STATIC=0.88
 CHUNKED_PREFILL_SIZE=8192
+SWA_FULL_TOKENS_RATIO=0.1
+# Default (non-DP) attention path: compressed attention, shared-expert fusion off.
+MODEL_ARGS=(
+    --attention-backend compressed
+    --page-size 256
+    --disable-shared-experts-fusion
+)
+CUDA_GRAPH_ARGS=()
+EXTRA_ARGS=()
 if [ "$DP_ATTENTION" = "true" ]; then
+    # DP-attention high-throughput agentic recipe (SGLang cookbook: DSV4-Pro FP4
+    # B300, high-throughput, single-node): MegaMoE DeepGEMM MoE (DeepEP dispatch +
+    # fused shared experts + kernel autotune), tuned for conc128.
+    # Measured DEP8 conc128 vs the previous flashinfer_mxfp4 path:
+    # 24,466 -> 33,220 tok/s/gpu (surpasses the vLLM reference of 28,962).
+    export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1
+    export SGLANG_OPT_FIX_HASH_MEGA_MOE=1
+    export SGLANG_OPT_USE_FAST_MASK_EP=1
+    export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1
+    export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320
+    export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1
+    export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0
     PARALLEL_ARGS+=(
         --dp "$TP"
         --tokenizer-worker-num "$TP"
@@ -115,29 +138,57 @@ if [ "$DP_ATTENTION" = "true" ]; then
         --stream-interval 20
         --dist-init-addr "127.0.0.1:$((PORT + 2000))"
         --ep-size "$EP_SIZE"
+        --moe-a2a-backend megamoe
+    )
+    MEM_FRACTION_STATIC=0.85
+    SWA_FULL_TOKENS_RATIO=0.075
+    # MegaMoE requires the default (non-compressed) attention path and fused shared experts.
+    MODEL_ARGS=()
+    EXTRA_ARGS=(--enable-prefill-delayer)
+    # chunked-prefill is divided by dp_size (=TP) under DP-attention; keep the
+    # effective chunk at 8192. Size the decode graph to served concurrency:
+    # DEP8 (dp 8) serves conc up to 512, DEP4 (dp 4) tops out near 128.
+    if [ "$TP" = "8" ]; then
+        CHUNKED_PREFILL_SIZE=65536
+        CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544)
+    else
+        # DEP4 (TP4/EP4) shards the model across only 4 GPUs, so per-GPU weights
+        # are ~2x DEP8 (233 GB loaded, ~32 GB free). At the DEP8-tuned 0.835 the
+        # KV pool cannot allocate (profiler floor ~0.879). Set 0.93: ~12 GB KV
+        # for the no-offload conc points, while keeping ~18 GB of activation/
+        # CUDA-graph headroom for the MegaMoE workspace (DEP4's footprint is
+        # lighter than DEP8's: decode graph 128 vs 544, same 8192 eff chunk).
+        MEM_FRACTION_STATIC=0.93
+        CHUNKED_PREFILL_SIZE=32768
+        CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 128)
+    fi
+elif { [ "$TP" = "8" ] || [ "$TP" = "4" ]; } && [ "${CONC:-999}" -le 16 ]; then
+    # TP-only low-latency (TP4 or TP8, non-DP, conc <= 16): SGLang cookbook
+    # DSV4-Pro FP4 B300 low-latency single-node recipe (mirrors the fixed_seq_len
+    # MTP TP-only path) with speculative decoding (EAGLE) removed. Default
+    # attention + fused shared experts + DSV4 FP4 sparse-attention indexer;
+    # mem-fraction 0.90.
+    PARALLEL_ARGS+=(
         --moe-runner-backend flashinfer_mxfp4
         --disable-flashinfer-autotune
+        --enable-deepseek-v4-fp4-indexer
     )
-    MEM_FRACTION_STATIC=0.95
-    CHUNKED_PREFILL_SIZE=16384
+    MEM_FRACTION_STATIC=0.90
+    MODEL_ARGS=(--page-size 256)
 else
     PARALLEL_ARGS+=(
         --moe-runner-backend flashinfer_mxfp4
         --disable-flashinfer-autotune
     )
 fi
 
-MODEL_ARGS=(
-    --attention-backend compressed
-    --page-size 256
-    --disable-shared-experts-fusion
-)
-
 # AgentX concurrency counts live session trees, not individual requests.
 # Allow subagent fan-out to exceed CONC without clipping request bursts.
 MAX_RUNNING_REQUESTS=$((2 * CONC))
 CUDA_GRAPH_MAX_BS=$CONC
 [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64
+# Non-DP path keeps the concurrency-scaled decode graph; DP path set 544 above.
+[ ${#CUDA_GRAPH_ARGS[@]} -eq 0 ] && CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS")
 
 export PYTHONNOUSERSITE=1
 export TORCH_CUDA_ARCH_LIST=10.0
@@ -172,9 +223,9 @@ SGLANG_CMD=(
     --trust-remote-code
     "${PARALLEL_ARGS[@]}"
     --mem-fraction-static "$MEM_FRACTION_STATIC"
-    --swa-full-tokens-ratio 0.1
+    --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO"
     --max-running-requests "$MAX_RUNNING_REQUESTS"
-    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    "${CUDA_GRAPH_ARGS[@]}"
     --allow-auto-truncate
     --chunked-prefill-size "$CHUNKED_PREFILL_SIZE"
     --tool-call-parser deepseekv4
@@ -183,6 +234,7 @@ SGLANG_CMD=(
     --watchdog-timeout 1800
     "${MODEL_ARGS[@]}"
     "${METRICS_ARGS[@]}"
+    "${EXTRA_ARGS[@]}"
     "${CACHE_ARGS[@]}"
 )
 

@@ -14730,12 +14730,12 @@ dsv4-fp4-b200-sglang-agentic-hicache:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 8, kv-offloading: none,     conc-list: [1, 2, 3, 4, 5] }
-      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8, 10, 16, 32, 40, 44] }
+      - { tp: 8, kv-offloading: none,     conc-list: [1, 2, 4, 8, 16] }
+      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8, 16, 24] }
       # DEP without HiCache is already degraded at conc 52 (2,098 tok/s/GPU),
       # so resolve its pre-52 cliff instead of repeating the collapsed tail.
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,    conc-list: [16, 24, 32, 38, 44, 48, 50, 52], router: { name: sglang-router, version: "0.3.2" } }
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 38, 44, 50, 56, 64, 66, 68], router: { name: sglang-router, version: "0.3.2" } }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,    conc-list: [16, 32, 38, 44, 48, 52, 54, 56, 58], router: { name: sglang-router, version: "0.3.2" } }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 38, 40, 44, 48, 52, 54, 56, 60, 64, 68], router: { name: sglang-router, version: "0.3.2" } }
 
 # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve
 # covers the middle/high-interactivity range omitted by the one-decode DEP
@@ -14752,11 +14752,11 @@ dsv4-fp4-b300-sglang-agentic-hicache:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 4, kv-offloading: none, conc-list: [1, 4, 8, 16, 20, 24, 32] }
-      - { tp: 8, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 40, 48, 52, 56, 60, 64, 72] }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40, 64], router: { name: sglang-router, version: "0.3.2" } }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128], router: { name: sglang-router, version: "0.3.2" } }
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144, 196, 512], router: { name: sglang-router, version: "0.3.2" } }
+      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] }
+      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40] }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40], router: { name: sglang-router, version: "0.3.2" } }
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 128], router: { name: sglang-router, version: "0.3.2" } }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 136, 144, 152, 196], router: { name: sglang-router, version: "0.3.2" } }
 
 # DEP8 prefill uses an 8K batch because 16K OOMs in the FP4 MoE intermediate;
 # decode uses FULL_DECODE_ONLY after the controlled graph test restored decode

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
@@ -4887,3 +4887,15 @@
     - "Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201  
 
+- config-keys:
+    - dsv4-fp4-b200-sglang-agentic-hicache
+    - dsv4-fp4-b300-sglang-agentic-hicache
+  description:
+    - "B300: rework dsv4_fp4_b300_sglang.sh into three regimes selected at launch — DP-attention -> MegaMoE DeepGEMM; pure-TP (TP8/TP4) at conc<=16 -> low-latency; otherwise flashinfer_mxfp4 baseline"
+    - "B300 MegaMoE (DP-attention path): switch --moe-a2a-backend to megamoe with the DeepGEMM MegaMoE env stack (SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1, FIX_HASH_MEGA_MOE=1, USE_FAST_MASK_EP=1, FIX_MEGA_MOE_MEMORY=1, DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320, FIX_NEXTN_MEGA_MOE=1, SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0); add --enable-prefill-delayer, mem-fraction-static 0.85 (DEP8) / 0.93 (DEP4, whose heavier per-GPU weights need a higher floor to fit the KV pool), swa-full-tokens-ratio 0.075"
+    - "B300 MegaMoE rank-aware sizing: DEP8 uses chunked-prefill-size 65536 + cuda-graph-max-bs-decode 544; DEP4 uses chunked-prefill-size 32768 + cuda-graph-max-bs-decode 128 (effective per-DP chunk held at 8192)"
+    - "B300 low-latency (pure TP, conc<=16): --moe-runner-backend flashinfer_mxfp4 --disable-flashinfer-autotune --enable-deepseek-v4-fp4-indexer, mem-fraction-static 0.90, page-size 256 (SGLang DSV4 cookbook low-latency recipe; EAGLE speculative decoding intentionally omitted since it is unavailable here)"
+    - "B200: split mem-fraction-static (0.88 under DP-attention, 0.90 otherwise)"
+    - "nvidia-master.yaml: refine B200/B300 agentic-hicache conc-list search spaces"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2145
+