diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x.sh index 665f3f944..61004c0b5 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x.sh @@ -3,6 +3,17 @@ # MiniMax-M3 MXFP8 MI355X (gfx950) single-node vLLM recipe. # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 # The recipe recommends MXFP8 from TP=4 on gfx950 and requires block size 128. +# +# AITER page-16 sparse paged-attention fast path (vllm-project/vllm#47287, +# merged into the pinned nightly): maps MiniMax-M3's top-k 128-token sparse +# blocks onto AITER page-16 block tables and runs AITER Gluon paged attention +# over only the selected KV pages. This is a kernel-level speedup of the same +# sparse-attention computation (no FLOP reduction), enabled via +# VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 with fp8 KV cache +# on a TP where each rank has num_kv_heads == 1 (TP4). We deliberately do NOT +# pass the #47269 --hf-overrides use_index_cache/index_topk_freq cross-layer +# indexer-skip override: it reduces model-architecture FLOPs, which is +# disallowed by docs/PR_REVIEW_CHECKLIST.md. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -31,10 +42,39 @@ fi SERVER_LOG=/workspace/server.log export VLLM_ENGINE_READY_TIMEOUT_S=3600 export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -# MI355X mxfp8 recipe (vllm-project/recipes#581): INT6 quick all-reduce plus -# the router-append shared-experts MoE fusion (vllm-project/vllm#46545). +# MI355X mxfp8 recipe (vllm-project/recipes#581): INT4 quick all-reduce plus +# the router-append shared-experts MoE fusion (vllm-project/vllm#46545). INT4 +# quick all-reduce is applied at all concurrencies (accuracy is guarded by the +# 8k1k evals); #2003 used INT6. export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 + +# AITER page-16 sparse PA (vllm-project/vllm#47287) is a long-context, +# high-concurrency optimization: it maps MiniMax-M3's top-k 128-token sparse +# blocks onto AITER page-16 block tables. Measured on gfx950 MXFP8, it only wins +# in the 8k1k high-concurrency tail and adds overhead at short context (1k1k) or +# low batch. So enable the "high-conc fast path" (shuffled KV-cache layout for +# sparse PA + the emulation dense-linear backend, see below) only for +# isl>=8192 && conc>=64; everywhere else fall back to the #2003 path +# (non-shuffled Triton attention + native linear). Overridable via +# MM3_HIGH_CONC_FASTPATH=0/1. +if [ -z "${MM3_HIGH_CONC_FASTPATH:-}" ]; then + if [ "$ISL" -ge 8192 ] && [ "$CONC" -ge 64 ]; then + MM3_HIGH_CONC_FASTPATH=1 + else + MM3_HIGH_CONC_FASTPATH=0 + fi +fi + +if [ "$MM3_HIGH_CONC_FASTPATH" = "1" ]; then + export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 + # Quick all-reduce tuning from the MiniMax-M3 AITER recipe (vllm-project/vllm#47287): + # keep the bf16 accumulation and only quantize all-reduces above 256 KB. + export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 + export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 +else + export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0 +fi if [ "${EVAL_ONLY}" = "true" ]; then setup_eval_context @@ -61,6 +101,18 @@ export VLLM_ROCM_USE_AITER=1 # concurrency. Overridable via env. MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-32768}" +# Dense-linear backend, gated on the same high-conc fast path as sparse PA. On +# this nightly the native Triton MXFP8 linear GEMM wins in the memory-bound +# low-concurrency regime, while --linear-backend emulation (bf16 hipBLASLT) wins +# in the compute-bound high-concurrency regime (~+3-5% at 8k1k conc>=64). +# LINEAR_BACKEND overrides (a backend name to force it, or "native" to disable). +LINEAR_ARGS=() +if [ -n "${LINEAR_BACKEND:-}" ]; then + [ "$LINEAR_BACKEND" != "native" ] && LINEAR_ARGS=(--linear-backend "$LINEAR_BACKEND") +elif [ "$MM3_HIGH_CONC_FASTPATH" = "1" ]; then + LINEAR_ARGS=(--linear-backend emulation) +fi + start_gpu_monitor set -x @@ -74,7 +126,7 @@ vllm serve "$MODEL" --port "$PORT" \ --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \ --kv-cache-dtype fp8 \ --attention-backend TRITON_ATTN \ - --linear-backend emulation \ + "${LINEAR_ARGS[@]}" \ --tool-call-parser minimax_m3 \ --reasoning-parser minimax_m3 \ --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 3e3170dac..118669241 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2491,7 +2491,7 @@ dsv4-fp4-mi355x-atom-disagg: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 # MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. minimaxm3-fp8-mi355x-vllm: - image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa + image: vllm/vllm-openai-rocm:nightly-9e57de7197f234f9d9187715d96e07e007048c0f model: MiniMaxAI/MiniMax-M3-MXFP8 model-prefix: minimaxm3 runner: mi355x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fe3610186..04e573a42 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4751,6 +4751,17 @@ - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 +- config-keys: + - minimaxm3-fp8-mi355x-vllm + description: + - "Bump the MiniMax-M3 MXFP8 MI355X vLLM image to nightly-9e57de7197f234f9d9187715d96e07e007048c0f, which carries the merged AITER page-16 sparse paged-attention path (vllm-project/vllm#47287)" + - "Enable AITER page-16 sparse PA (VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1, with VLLM_ROCM_USE_AITER=1 and fp8 KV cache on TP4 where num_kv_heads == 1 per rank): AITER derives page-16 K/V views from the page-128 KV cache and routes decode/prefill through AITER Gluon paged attention over MiniMax-M3's top-k 128-token sparse blocks. Kernel-level speedup of the same computation, no FLOP reduction" + - "Concurrency-gate the fast path to the long-context high-concurrency tail (isl>=8192 && conc>=64). Sparse PA is a long-context/high-batch optimization and adds overhead at short context (1k1k) or low concurrency, so both VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT and --linear-backend emulation are enabled only in that regime; everything else (all 1k1k, and 8k1k conc<64) falls back to the prior #2003 path (non-shuffled Triton attention + native linear). Net effect vs #2003: 8k1k tput/gpu conc128 +3.3%, conc256 +2.5%, conc512 +4.7% (peak), neutral elsewhere" + - "Switch quick all-reduce quantization to INT4 for all concurrencies (#2003 used INT6); accuracy is guarded by the 8k1k evals. In the high-conc fast path also apply the AITER tuning knobs VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 and VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256" + - "Deliberately do NOT pass the #47269 --hf-overrides use_index_cache/index_topk_freq cross-layer indexer-skip override: it reduces model-architecture FLOPs, which is disallowed by docs/PR_REVIEW_CHECKLIST.md" + - "Serving flags are otherwise unchanged (--block-size 128, --language-model-only, --moe-backend aiter, --kv-cache-dtype fp8, --attention-backend TRITON_ATTN, minimax_m3 parsers); TP4 conc 1-512 sweep at 1k1k and 8k1k unchanged" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2187 + - config-keys: - dsv4-fp8-h200-vllm - dsv4-fp8-h200-vllm-mtp