From 1d24294ceff704aee6a07be49672480c3c402147 Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:31:44 +0000 Subject: [PATCH 1/3] perf(amd): tune Kimi K2.5 MXFP4 on MI355X [skip-sweep] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply the accuracy-gated serving settings, preserve the existing TP4 and TP8 matrix dimensions, extend only their concurrency upper bounds to 128, and keep the image as a draft placeholder pending the required AITER bump. 中文：应用已通过精度验证的服务配置，保留现有 TP4 和 TP8 矩阵维度，仅将并发上限扩展到 128，并暂时保留镜像占位符，等待所需的 AITER 版本升级。 --- .../fixed_seq_len/kimik2.5_fp4_mi355x.sh | 21 ++++++++++++++----- configs/amd-master.yaml | 11 +++++----- perf-changelog.yaml | 7 +++++++ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh index d4616143ac..c159af6a9f 100755 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh @@ -35,6 +35,10 @@ if [ "${EVAL_ONLY}" = "true" ]; then MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" fi +if [ "$MAX_MODEL_LEN" -lt 9472 ]; then + MAX_MODEL_LEN=9472 +fi + # If the machine runs a MEC FW older than 177, RCCL # cannot reclaim some memory. # Disable that features to avoid crashes. @@ -47,6 +51,13 @@ fi export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 +export VLLM_ROCM_USE_SKINNY_GEMM=0 +export AITER_MXFP4_INTERMEDIATE=1 +export AITER_BYPASS_TUNE_CONFIG=0 +export AITER_MOE_SORT_BACKEND=auto +export AITER_LOG_LEVEL=INFO +export OMP_NUM_THREADS=1 # Disable AITER RMSNorm for TP < 8 due to accuracy issues if [ "${TP}" -lt 8 ]; then @@ -59,9 +70,6 @@ else EP=" " fi -# following AMD andy luo's recipe -# https://x.com/linluo77/status/2017024513595301985 - # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -71,8 +79,11 @@ vllm serve $MODEL --port $PORT \ $EP \ --gpu-memory-utilization 0.90 \ --max-model-len $MAX_MODEL_LEN \ ---block-size=1 \ ---no-enable-prefix-caching \ +--kv-cache-dtype fp8 \ +--block-size 16 \ +--max-num-batched-tokens 16384 \ +--max-num-seqs 512 \ +--async-scheduling \ --trust-remote-code \ --no-enable-prefix-caching \ --mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4b1ea4aed3..2327426cce 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -770,7 +770,8 @@ kimik2.5-int4-mi300x-vllm: - { tp: 8, conc-start: 4, conc-end: 64 } kimik2.5-fp4-mi355x-vllm: - image: vllm/vllm-openai-rocm:v0.24.0 + # TODO: replace with the official vLLM ROCm image containing AITER #3832. + image: TBD model: amd/Kimi-K2.5-MXFP4 model-prefix: kimik2.5 runner: mi355x @@ -782,13 +783,13 @@ kimik2.5-fp4-mi355x-vllm: - isl: 1024 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 128 } + - { tp: 4, conc-start: 4, conc-end: 128 } - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 128 } + - { tp: 4, conc-start: 4, conc-end: 128 } kimik2.5-fp4-mi355x-vllm-agentic: image: vllm/vllm-openai-rocm:v0.22.0 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d77c35f1fd..c2151d778a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4781,3 +4781,10 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm + description: + - "Apply the accuracy-gated Kimi-K2.5 MXFP4 TP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, and the AITER BF16 GEMM path" + - "Extend the existing TP4 and TP8 concurrency sweeps from 64 to 128 without changing other search dimensions; the image remains TBD until an official vLLM ROCm image contains AITER #3832" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 From 601eefc91f79ec212f27d0dd50135b764ca58ba6 Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:55:19 -0400 Subject: [PATCH 2/3] Update kimik2.5_fp4_mi355x.sh --- benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh index c159af6a9f..e01ee4cfae 100755 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh @@ -55,7 +55,6 @@ export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 export VLLM_ROCM_USE_SKINNY_GEMM=0 export AITER_MXFP4_INTERMEDIATE=1 export AITER_BYPASS_TUNE_CONFIG=0 -export AITER_MOE_SORT_BACKEND=auto export AITER_LOG_LEVEL=INFO export OMP_NUM_THREADS=1 From 335b39a963f10da4b57dd917fb123a6eef1f79ce Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:56:13 -0400 Subject: [PATCH 3/3] Update kimik2.5_fp4_mi355x.sh --- benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh index e01ee4cfae..6dadc26876 100755 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh @@ -55,7 +55,7 @@ export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 export VLLM_ROCM_USE_SKINNY_GEMM=0 export AITER_MXFP4_INTERMEDIATE=1 export AITER_BYPASS_TUNE_CONFIG=0 -export AITER_LOG_LEVEL=INFO +export AITER_MOE_SORT_BACKEND=auto export OMP_NUM_THREADS=1 # Disable AITER RMSNorm for TP < 8 due to accuracy issues