From d50b4d0c5acd339c879ce574e8cd23b40152e137 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 15 Jul 2026 11:58:52 -0500 Subject: [PATCH 1/2] fix(agentx): rerun B300 SGLang with OOM headroom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry the B300-only SGLang AgentX recipe and search-space changes from PR #2145 onto current main. Reduce DEP4 mem-fraction-static from 0.93 to 0.90 after run 29367944489 showed sparse-attention indexer CUDA OOMs during warmup.\n\n中文：将原 PR #2145 中仅适用于 B300 的 SGLang AgentX 配方与搜索空间变更迁移到最新 main。根据运行 29367944489 在预热阶段出现的稀疏注意力索引器 CUDA OOM，将 DEP4 的 mem-fraction-static 从 0.93 降至 0.90，为激活值和工作区预留更多 HBM。 --- .../agentic/dsv4_fp4_b300_sglang.sh | 72 ++++++++++++++++--- configs/nvidia-master.yaml | 10 +-- 2 files changed, 67 insertions(+), 15 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh index 5599ca4717..1b5a90e4bb 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh @@ -56,6 +56,8 @@ install_agentic_deps SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" +export SGLANG_DEFAULT_THINKING=1 +export SGLANG_DSV4_REASONING_EFFORT=high export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 @@ -105,7 +107,28 @@ PARALLEL_ARGS=(--tp "$TP") METRICS_ARGS=(--enable-metrics) MEM_FRACTION_STATIC=0.88 CHUNKED_PREFILL_SIZE=8192 +SWA_FULL_TOKENS_RATIO=0.1 +# Default (non-DP) attention path: compressed attention, shared-expert fusion off. +MODEL_ARGS=( + --attention-backend compressed + --page-size 256 + --disable-shared-experts-fusion +) +CUDA_GRAPH_ARGS=() +EXTRA_ARGS=() if [ "$DP_ATTENTION" = "true" ]; then + # DP-attention high-throughput agentic recipe (SGLang cookbook: DSV4-Pro FP4 + # B300, high-throughput, single-node): MegaMoE DeepGEMM MoE (DeepEP dispatch + + # fused shared experts + kernel autotune), tuned for conc128. + # Measured DEP8 conc128 vs the previous flashinfer_mxfp4 path: + # 24,466 -> 33,220 tok/s/gpu (surpasses the vLLM reference of 28,962). + export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1 + export SGLANG_OPT_FIX_HASH_MEGA_MOE=1 + export SGLANG_OPT_USE_FAST_MASK_EP=1 + export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1 + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 + export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0 PARALLEL_ARGS+=( --dp "$TP" --tokenizer-worker-num "$TP" @@ -115,11 +138,43 @@ if [ "$DP_ATTENTION" = "true" ]; then --stream-interval 20 --dist-init-addr "127.0.0.1:$((PORT + 2000))" --ep-size "$EP_SIZE" + --moe-a2a-backend megamoe + ) + MEM_FRACTION_STATIC=0.85 + SWA_FULL_TOKENS_RATIO=0.075 + # MegaMoE requires the default (non-compressed) attention path and fused shared experts. + MODEL_ARGS=() + EXTRA_ARGS=(--enable-prefill-delayer) + # chunked-prefill is divided by dp_size (=TP) under DP-attention; keep the + # effective chunk at 8192. Size the decode graph to served concurrency: + # DEP8 (dp 8) serves conc up to 512, DEP4 (dp 4) tops out near 128. + if [ "$TP" = "8" ]; then + CHUNKED_PREFILL_SIZE=65536 + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544) + else + # DEP4 (TP4/EP4) shards the model across only 4 GPUs, so per-GPU weights + # are ~2x DEP8 (233 GB loaded; profiler floor ~0.879). At 0.93, run + # 29367944489 left only 2.9-4.8 GiB free while the sparse-attention + # indexer requested 4.2-6.1 GiB, causing CUDA OOMs during warmup. Use + # 0.90 to restore about 8.6 GiB/GPU of activation/workspace headroom + # while remaining above the profiler floor. + MEM_FRACTION_STATIC=0.90 + CHUNKED_PREFILL_SIZE=32768 + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 128) + fi +elif { [ "$TP" = "8" ] || [ "$TP" = "4" ]; } && [ "${CONC:-999}" -le 16 ]; then + # TP-only low-latency (TP4 or TP8, non-DP, conc <= 16): SGLang cookbook + # DSV4-Pro FP4 B300 low-latency single-node recipe (mirrors the fixed_seq_len + # MTP TP-only path) with speculative decoding (EAGLE) removed. Default + # attention + fused shared experts + DSV4 FP4 sparse-attention indexer; + # mem-fraction 0.90. + PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 --disable-flashinfer-autotune + --enable-deepseek-v4-fp4-indexer ) - MEM_FRACTION_STATIC=0.95 - CHUNKED_PREFILL_SIZE=16384 + MEM_FRACTION_STATIC=0.90 + MODEL_ARGS=(--page-size 256) else PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 @@ -127,17 +182,13 @@ else ) fi -MODEL_ARGS=( - --attention-backend compressed - --page-size 256 - --disable-shared-experts-fusion -) - # AgentX concurrency counts live session trees, not individual requests. # Allow subagent fan-out to exceed CONC without clipping request bursts. MAX_RUNNING_REQUESTS=$((2 * CONC)) CUDA_GRAPH_MAX_BS=$CONC [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 +# Non-DP path keeps the concurrency-scaled decode graph; DP path set 544 above. +[ ${#CUDA_GRAPH_ARGS[@]} -eq 0 ] && CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") export PYTHONNOUSERSITE=1 export TORCH_CUDA_ARCH_LIST=10.0 @@ -172,9 +223,9 @@ SGLANG_CMD=( --trust-remote-code "${PARALLEL_ARGS[@]}" --mem-fraction-static "$MEM_FRACTION_STATIC" - --swa-full-tokens-ratio 0.1 + --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + "${CUDA_GRAPH_ARGS[@]}" --allow-auto-truncate --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" --tool-call-parser deepseekv4 @@ -183,6 +234,7 @@ SGLANG_CMD=( --watchdog-timeout 1800 "${MODEL_ARGS[@]}" "${METRICS_ARGS[@]}" + "${EXTRA_ARGS[@]}" "${CACHE_ARGS[@]}" ) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e35f9a2223..53b8f08b47 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -14264,11 +14264,11 @@ dsv4-fp4-b300-sglang-agentic-hicache: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 4, 8, 16, 20, 24, 32] } - - { tp: 8, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 40, 48, 52, 56, 60, 64, 72] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40, 64], router: { name: sglang-router, version: "0.3.2" } } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128], router: { name: sglang-router, version: "0.3.2" } } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144, 196, 512], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40] } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 128], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 136, 144, 152, 196], router: { name: sglang-router, version: "0.3.2" } } # DEP8 prefill uses an 8K batch because 16K OOMs in the FP4 MoE intermediate; # decode uses FULL_DECODE_ONLY after the controlled graph test restored decode From 829531201829bfa7a9563b9edf34dc3c4b63b069 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 15 Jul 2026 11:59:56 -0500 Subject: [PATCH 2/2] chore(perf): trigger B300 AgentX rerun MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Append the B300-only DeepSeek-V4 SGLang AgentX changelog entry for PR #2226.\n\n中文：为 PR #2226 追加仅针对 B300 的 DeepSeek-V4 SGLang AgentX 性能变更记录，以触发重跑。 --- perf-changelog.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4f0e4e4f12..d3ef8b3bac 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4795,3 +4795,10 @@ - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2222 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache + description: + - "Follow up on PR #2145 with the B300-only DeepSeek-V4-Pro FP4 SGLang AgentX MegaMoE DEP and low-latency TP recipes." + - "Reduce DEP4 mem-fraction-static from 0.93 to 0.90 after run 29367944489 exposed sparse-attention indexer CUDA OOMs during warmup; preserve the refined B300 concurrency search space and rerun only this configuration." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2226