From 9af6bf52ff6380dba4aa02a8f2d9f66596b43511 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Wed, 15 Jul 2026 22:13:12 -0700 Subject: [PATCH 1/2] Qwen3.5 FP4 GB300 disagg dynamo-sglang: bump image to v0.5.14-cu130 + NVFP4-V2 checkpoint --- .../gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml | 6 +++--- .../gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml | 6 +++--- .../gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml | 6 +++--- .../gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml | 6 +++--- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 8 +++++++- runners/launch_gb300-nv.sh | 2 +- 7 files changed, 22 insertions(+), 16 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml index 71a0ffb9f3..9619a3fd59 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 1P1D: TP4 Prefill + TP4 Decode +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 1P1D: TP4 Prefill + TP4 Decode # Pure tensor parallel, no expert parallel (STP) # 8k1k sa-bench concurrency sweep on GB300 # @@ -83,7 +83,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -125,7 +125,7 @@ backend: log-level: "info" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml index 00e576439a..fa43632fbc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 5P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 5P1D wide-EP # Prefill: 5 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 36 GB300 GPUs (5*4 + 4*4); 8k1k concurrency 1024/2048/3072. @@ -93,7 +93,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -128,7 +128,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml index ecab285097..691d0d72cf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 6P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 6P1D wide-EP # Prefill: 6 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 40 GB300 GPUs (6*4 + 4*4); 8k1k concurrency 5120. @@ -91,7 +91,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -126,7 +126,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml index d35f444694..44aa3934d8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml @@ -1,4 +1,4 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 7P1D wide-EP +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 7P1D wide-EP # Prefill: 7 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) # Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) # Total: 44 GB300 GPUs (7*4 + 4*4); 8k1k concurrency 5120. @@ -91,7 +91,7 @@ backend: sglang_config: prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true @@ -126,7 +126,7 @@ backend: linear-attn-decode-backend: "flashinfer" decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" model-path: "/model/" trust-remote-code: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 33e9118b7a..649dd5ceec 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11745,8 +11745,8 @@ glm5-fp4-gb200-dynamo-sglang: dp-attn: false qwen3.5-fp4-gb300-dynamo-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 + image: lmsysorg/sglang:v0.5.14-cu130 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: gb300 precision: fp4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e8e9078c2e..4875741b23 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4859,4 +4859,10 @@ description: - "Add GB300 DeepSeek-V4 dynamo-sglang agentic recipes (agg-tp4 + dep8 disagg pareto)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2157 - + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang + description: + - "Bump SGLang image to v0.5.14-cu130 for the Qwen3.5 FP4 GB300 disaggregated dynamo-sglang config" + - "Switch to the nvidia/Qwen3.5-397B-A17B-NVFP4-V2 checkpoint (model pre-staged on cluster as Qwen3.5-397B-A17B-NVFP4-V2)" + pr-link: XXX diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index d450cdb86d..16b45e8c41 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -83,7 +83,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our # Qwen3.5 sglang recipes (qwen3.5-fp4). - export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4 + export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-NVFP4-V2 export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4" elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then # SRT_SLURM_MODEL_PREFIX must match the model.path alias used in our From 38d77d67b74131d77a2ec4ffeda6f200820a1bbf Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Wed, 15 Jul 2026 22:13:30 -0700 Subject: [PATCH 2/2] Update perf-changelog pr-link for #2238 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4875741b23..bf2cc9bb10 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4865,4 +4865,4 @@ description: - "Bump SGLang image to v0.5.14-cu130 for the Qwen3.5 FP4 GB300 disaggregated dynamo-sglang config" - "Switch to the nvidia/Qwen3.5-397B-A17B-NVFP4-V2 checkpoint (model pre-staged on cluster as Qwen3.5-397B-A17B-NVFP4-V2)" - pr-link: XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2238