diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh index b036e060e..87809f54b 100644 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh @@ -20,6 +20,7 @@ if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 +export SGLANG_MAMBA_SSM_DTYPE=bfloat16 SERVER_LOG=/workspace/server.log MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh index 8081b824e..16e86e544 100755 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh @@ -20,6 +20,7 @@ hf download "$MODEL" export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 +export SGLANG_MAMBA_SSM_DTYPE=bfloat16 SERVER_LOG=/workspace/server.log MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4b1ea4aed..31582a5ce 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -415,7 +415,7 @@ qwen3.5-fp8-mi355x-sglang-disagg: - "DECODE_MTP_SIZE=0" qwen3.5-fp4-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.13-rocm720-mi35x-20260612 + image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 model: amd/Qwen3.5-397B-A17B-MXFP4 model-prefix: qwen3.5 runner: mi355x @@ -457,7 +457,7 @@ qwen3.5-fp4-mi355x-atom: - { tp: 4, conc-start: 4, conc-end: 16 } qwen3.5-fp4-mi355x-sglang-mtp: - image: lmsysorg/sglang-rocm:v0.5.13-rocm720-mi35x-20260612 + image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 model: amd/Qwen3.5-397B-A17B-MXFP4 model-prefix: qwen3.5 runner: mi355x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e05c2ca1c..f8e353e7f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4765,3 +4765,11 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-sglang-mtp + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713" + - "Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201