From 4559de9653593a7e405030785f2ca0c8031f68f9 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 16:03:55 +0800
Subject: [PATCH 01/15] feat(dsv4): scaffold FP8 MI325X SGLang bring-up
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

New SKU — no prior DeepSeek-V4-Pro data on gfx942. Extrapolated per
recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro from the same-model MI355X
entry (search-space shape, deepseek_v4 flags), the same-SKU
dsr1-fp8-mi325x-sglang (gfx942 AITER infra), and the H200 dsv4 FP8 path
(--quantization deepseek_v4_fp8). gfx942 has no native FP4, so the ~960GB
mixed checkpoint runs in FP8 (~1.05TB), which only fits TP8 — MI355X TP4
bands dropped. 8x256GB (2TB) leaves ~1TB free after weights. Sweep:
TP-only low-latency (conc 1-32) + DP-attn/EP8 throughput (conc 64-2048),
1k1k + 8k1k.

Unvalidated on-cluster: needs debug-runs to confirm the mi30x image has the
DeepseekV4 class + --attention-backend dsv4 / --quantization deepseek_v4_fp8
on gfx942.

中文：搭建 MI325X 上 DeepSeek-V4-Pro FP8 单节点 SGLang 脚手架（依据
recipes.vllm.ai 及同型号 MI355X、同 SKU dsr1-mi325x 外推；FP8 仅 TP8 可
放下故去掉 TP4）。尚未集群验证。
---
 .../fixed_seq_len/dsv4_fp8_mi325x.sh          | 141 ++++++++++++++++++
 configs/amd-master.yaml                       |  34 +++++
 perf-changelog.yaml                           |   9 ++
 3 files changed, 184 insertions(+)
 create mode 100755 benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
new file mode 100755
index 000000000..336933d65
--- /dev/null
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via SGLang.
+#
+# EXTRAPOLATED bring-up recipe (unvalidated on-cluster as of 2026-07),
+# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus:
+#   * same model, adjacent SKU: dsv4_fp4_mi355x_sglang.sh (search-space
+#     shape, DP-attention + EP path, deepseek_v4 model flags, SWA/page-size)
+#   * same SKU, different model: dsr1_fp8_mi325x.sh (gfx942 infra: AITER,
+#     MEC-firmware scratch-reclaim guard)
+#   * same model, FP8 path: the H200 dsv4 recipe (--quantization
+#     deepseek_v4_fp8) — gfx942 has no native FP4, so the FP4-MoE checkpoint
+#     is run in FP8.
+#
+# Sizing: the ~960GB mixed checkpoint is ~1.05TB in FP8, so only TP8 fits
+# 8x256GB comfortably (TP4 = 1024GB is too tight for ~1.05TB). The config restricts to TP8 accordingly.
+#
+# Debug-runs must confirm before this leaves bring-up:
+#   1. The mi30x image carries the DeepseekV4 model class.
+#   2. --attention-backend dsv4 exists on gfx942 (else fall back to aiter).
+#   3. --quantization deepseek_v4_fp8 loads on ROCm (else use the AITER FP8
+#      MoE path without an explicit quant flag).
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars \
+    MODEL \
+    TP \
+    DP_ATTENTION \
+    EP_SIZE \
+    CONC \
+    ISL \
+    OSL \
+    RANDOM_RANGE_RATIO \
+    RESULT_FILENAME \
+    MAX_MODEL_LEN
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+  echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi
+
+# If the machine runs a MEC FW older than 177, RCCL cannot reclaim some
+# memory; disable that feature to avoid crashes (see dsr1_fp8_mi325x.sh).
+version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'`
+if [[ "$version" == "" || $version -lt 177 ]]; then
+  export HSA_NO_SCRATCH_RECLAIM=1
+fi
+
+# gfx942 AITER infra (dsr1_fp8_mi325x.sh) + deepseek_v4 model env (mi355x).
+export SGLANG_USE_AITER=1
+export SGLANG_AITER_MLA_PERSIST=1
+export SGLANG_DEFAULT_THINKING=1
+export SGLANG_DSV4_REASONING_EFFORT=max
+export AITER_BF16_FP8_MOE_BOUND=0
+
+SERVER_LOG=/workspace/server.log
+
+EVAL_CONTEXT_ARGS=""
+if [ "${EVAL_ONLY}" = "true" ]; then
+    setup_eval_context
+    EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN"
+fi
+
+# Start GPU monitoring (power, temperature, clocks every second)
+start_gpu_monitor
+
+# DP_ATTENTION=true runs DP-attention with expert parallel (dp-size = TP),
+# mirroring dsv4_fp4_mi355x_sglang.sh; false runs pure tensor parallel for
+# the low-latency band.
+PARALLEL_ARGS=(--tensor-parallel-size "$TP")
+CHUNKED_PREFILL_SIZE=$ISL
+if [ "${DP_ATTENTION}" = "true" ]; then
+    export SGLANG_SHARED_EXPERT_TP1=1
+    export SGLANG_DP_SHARED_EXPERT_LOCAL=1
+    export SGLANG_DP_USE_GATHERV=1
+    export SGLANG_DP_USE_REDUCE_SCATTER=1
+    export GPU_MAX_HW_QUEUES=5
+
+    CHUNKED_PREFILL_SIZE=$((ISL * TP))
+    PARALLEL_ARGS+=(
+        --dp "$TP"
+        --enable-dp-attention
+        --enable-prefill-delayer
+        --enable-two-batch-overlap
+    )
+fi
+if [ "${EP_SIZE:-1}" -gt 1 ]; then
+    PARALLEL_ARGS+=(--ep-size "$EP_SIZE")
+fi
+
+set -x
+sglang serve \
+    --model-path $MODEL --served-model-name $MODEL \
+    --host=0.0.0.0 --port $PORT \
+    "${PARALLEL_ARGS[@]}" \
+    --trust-remote-code \
+    --quantization deepseek_v4_fp8 \
+    --kv-cache-dtype fp8_e4m3 \
+    --attention-backend dsv4 \
+    --disable-radix-cache \
+    --disable-shared-experts-fusion \
+    --page-size 256 \
+    --mem-fraction-static 0.90 \
+    --swa-full-tokens-ratio 0.15 \
+    --cuda-graph-max-bs ${CONC} \
+    --max-running-requests ${CONC} \
+    --context-length $MAX_MODEL_LEN \
+    --chunked-prefill-size $CHUNKED_PREFILL_SIZE \
+    --tool-call-parser deepseekv4 \
+    --reasoning-parser deepseek-v4 \
+    --chat-template "$(dirname "$0")/../chat_templates/deepseek_v4_thinking.jinja" \
+    --watchdog-timeout 1800 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 &
+
+SERVER_PID=$!
+
+# Wait for server to be ready
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+run_benchmark_serving \
+    --model "$MODEL" \
+    --port "$PORT" \
+    --backend vllm \
+    --input-len "$ISL" \
+    --output-len "$OSL" \
+    --random-range-ratio "$RANDOM_RANGE_RATIO" \
+    --num-prompts "$((CONC * 10))" \
+    --max-concurrency "$CONC" \
+    --result-filename "$RESULT_FILENAME" \
+    --result-dir /workspace/
+
+# After throughput, run evaluation only if RUN_EVAL is true
+if [ "${RUN_EVAL}" = "true" ]; then
+    run_eval --framework lm-eval --port "$PORT"
+    append_lm_eval_summary
+fi
+
+# Stop GPU monitoring
+stop_gpu_monitor
+set +x
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index 4b1ea4aed..a283edfcb 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -119,6 +119,40 @@ dsr1-fp8-mi325x-sglang:
       search-space:
       - { tp: 8, conc-start: 4, conc-end: 64 }
 
+# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via SGLang.
+# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07).
+# Search-space shape mirrors the same-model MI355X entry (dsv4-fp4-mi355x-sglang):
+# a TP-only low-latency band + a DP-attention + EP throughput band. gfx942 has
+# no native FP4, so this runs FP8 (the ~960GB mixed checkpoint is ~1.05TB in
+# FP8) — which only fits TP8, so the MI355X TP4 bands are dropped. 8x256GB (2TB)
+# leaves ~1TB free after FP8 weights, so the throughput band could later extend
+# past conc 2048 — kept at parity with the sister MI300X entry for a first
+# bring-up. Image follows the same-model MI355X's sglang-rocm dated-tag family
+# (mi30x variant); launch script dsv4_fp8_mi325x.sh carries the deepseek_v4
+# flags + gfx942 AITER infra. debug-runs must confirm the image carries the
+# DeepseekV4 class and that --attention-backend dsv4 / --quantization
+# deepseek_v4_fp8 work on gfx942 before this merges.
+dsv4-fp8-mi325x-sglang:
+  image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: mi325x
+  precision: fp8
+  framework: sglang
+  multinode: false
+  scenarios:
+    fixed-seq-len:
+    - isl: 1024
+      osl: 1024
+      search-space:
+      - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32 }
+      - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 2048 }
+    - isl: 8192
+      osl: 1024
+      search-space:
+      - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32 }
+      - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 2048 }
+
 dsr1-fp8-mi355x-sglang:
   image: lmsysorg/sglang:v0.5.12-rocm700-mi35x
   model: deepseek-ai/DeepSeek-R1-0528
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 57d510dd1..db2b4fe46 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4750,3 +4750,12 @@
     - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928"
     - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137
+
+- config-keys:
+    - dsv4-fp8-mi325x-sglang
+  description:
+    - "Add DeepSeek-V4-Pro FP8 single-node MI325X SGLang benchmark (new SKU, previously no dsv4 data on gfx942)"
+    - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi325x-sglang for gfx942 AITER infra; FP8 path (--quantization deepseek_v4_fp8) follows the H200 dsv4 recipe since gfx942 has no native FP4"
+    - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8; 8x256GB (2TB) leaves ~1TB free after FP8 weights so the throughput band could later extend past conc 2048 (kept at parity for a first bring-up)"
+    - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2195

From 35e91f437f3469173e9292348ead8d3cf86e7620 Mon Sep 17 00:00:00 2001
From: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 03:50:17 -0700
Subject: [PATCH 02/15] Update perf-changelog.yaml

---
 perf-changelog.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 4ab209e50..1780f101e 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4752,7 +4752,6 @@
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137
 
 - config-keys:
-<<<<<<< dsv4-mi325x-fp8-bringup
     - dsv4-fp8-mi325x-sglang
   description:
     - "Add DeepSeek-V4-Pro FP8 single-node MI325X SGLang benchmark (new SKU, previously no dsv4 data on gfx942)"
@@ -4760,7 +4759,8 @@
     - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8; 8x256GB (2TB) leaves ~1TB free after FP8 weights so the throughput band could later extend past conc 2048 (kept at parity for a first bring-up)"
     - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2195
-=======
+
+- config-keys:
     - dsv4-fp4-mi355x-sglang
   description:
     - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706"
@@ -4774,4 +4774,3 @@
     - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708"
     - "Clean the export envs"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198
->>>>>>> main

From 7bfc5cd8450e1c7ab814b9ff2ffc71f782c7bd83 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 19:16:01 +0800
Subject: [PATCH 03/15] fix(dsv4): drop invalid --quantization from MI325X
 sglang script
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

sglang argparse rejected --quantization deepseek_v4_fp8 (a vLLM-only
method), crashing the canary before server start (run 29326767592).
Remove the flag: sglang reads the modelopt quant config from the
checkpoint and runs it via the AITER MoE path, matching the H200 and
MI355X dsv4 sglang recipes (which pass no --quantization).

中文：删除 MI325X sglang 脚本中无效的 --quantization deepseek_v4_fp8
（该方法仅 vLLM 支持，sglang argparse 直接报错）。改为不传 --quantization，
由 sglang 从 checkpoint 读取 modelopt 量化配置并经 AITER MoE 执行。
---
 .../single_node/fixed_seq_len/dsv4_fp8_mi325x.sh   | 14 ++++++++------
 perf-changelog.yaml                                |  3 ++-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
index 336933d65..7afa3c129 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
@@ -8,9 +8,12 @@
 #     shape, DP-attention + EP path, deepseek_v4 model flags, SWA/page-size)
 #   * same SKU, different model: dsr1_fp8_mi325x.sh (gfx942 infra: AITER,
 #     MEC-firmware scratch-reclaim guard)
-#   * same model, FP8 path: the H200 dsv4 recipe (--quantization
-#     deepseek_v4_fp8) — gfx942 has no native FP4, so the FP4-MoE checkpoint
-#     is run in FP8.
+#   * same model + framework: dsv4_fp8_h200_sglang.sh — gfx942 has no native
+#     FP4, so the FP4-MoE checkpoint is run in FP8. Like the H200 sglang
+#     recipe (and the MI355X dsv4 sglang recipe), NO --quantization flag is
+#     passed: sglang reads the modelopt quant config from the checkpoint and
+#     the AITER MoE path (SGLANG_USE_AITER=1) executes it. (--quantization
+#     deepseek_v4_fp8 is a vLLM-only method; sglang argparse rejects it.)
 #
 # Sizing: the ~960GB mixed checkpoint is ~1.05TB in FP8, so only TP8 fits
 # 8x256GB comfortably (TP4 = 1024GB is too tight for ~1.05TB). The config restricts to TP8 accordingly.
@@ -18,8 +21,8 @@
 # Debug-runs must confirm before this leaves bring-up:
 #   1. The mi30x image carries the DeepseekV4 model class.
 #   2. --attention-backend dsv4 exists on gfx942 (else fall back to aiter).
-#   3. --quantization deepseek_v4_fp8 loads on ROCm (else use the AITER FP8
-#      MoE path without an explicit quant flag).
+#   3. The modelopt FP4-MoE checkpoint loads + runs in FP8 via AITER on gfx942
+#      (else pass an explicit valid sglang quant, e.g. --quantization fp8).
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -96,7 +99,6 @@ sglang serve \
     --host=0.0.0.0 --port $PORT \
     "${PARALLEL_ARGS[@]}" \
     --trust-remote-code \
-    --quantization deepseek_v4_fp8 \
     --kv-cache-dtype fp8_e4m3 \
     --attention-backend dsv4 \
     --disable-radix-cache \
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 1780f101e..0b2f54bfc 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4755,7 +4755,8 @@
     - dsv4-fp8-mi325x-sglang
   description:
     - "Add DeepSeek-V4-Pro FP8 single-node MI325X SGLang benchmark (new SKU, previously no dsv4 data on gfx942)"
-    - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi325x-sglang for gfx942 AITER infra; FP8 path (--quantization deepseek_v4_fp8) follows the H200 dsv4 recipe since gfx942 has no native FP4"
+    - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi325x-sglang for gfx942 AITER infra; gfx942 has no native FP4 so the FP4-MoE checkpoint runs in FP8"
+    - "No --quantization flag: sglang reads the modelopt quant config from the checkpoint and the AITER MoE path executes it, matching the H200 and MI355X dsv4 sglang recipes (--quantization deepseek_v4_fp8 is vLLM-only and rejected by sglang argparse)"
     - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8; 8x256GB (2TB) leaves ~1TB free after FP8 weights so the throughput band could later extend past conc 2048 (kept at parity for a first bring-up)"
     - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2195

From 422d3a8a61da31b390cb1da45c773868ccb7c6ea Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 19:32:49 +0800
Subject: [PATCH 04/15] fix(dsv4): route MI325X MLA through unified_kv_triton
 FlashMLA
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Same fix as MI300X: the prior fix cleared argparse but the server was killed
at CUDA-graph capture (run 29328296892) because --attention-backend dsv4
compiles its FP8 MLA kernel via TileLang, whose InjectSoftwarePipeline pass
fails on gfx942. Set SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
(+ SGLANG_USE_ROCM700A=0) per dsv4-fp4-mi355x-sglang to route MLA through the
unified-KV triton path. Drop SGLANG_AITER_MLA_PERSIST; keep SGLANG_USE_AITER
for the MoE GEMMs.

中文：与 MI300X 相同的修复——设置 SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
改走统一 KV triton FlashMLA 路径，绕开在 gfx942 上编译失败的 TileLang FP8 MLA
内核。
---
 .../single_node/fixed_seq_len/dsv4_fp8_mi325x.sh  | 15 ++++++++++++---
 perf-changelog.yaml                               |  1 +
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
index 7afa3c129..c99dbbe09 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
@@ -20,7 +20,8 @@
 #
 # Debug-runs must confirm before this leaves bring-up:
 #   1. The mi30x image carries the DeepseekV4 model class.
-#   2. --attention-backend dsv4 exists on gfx942 (else fall back to aiter).
+#   2. The unified-KV triton FlashMLA path captures CUDA graphs cleanly on
+#      gfx942 (the TileLang FP8 MLA kernel does not — see the env note below).
 #   3. The modelopt FP4-MoE checkpoint loads + runs in FP8 via AITER on gfx942
 #      (else pass an explicit valid sglang quant, e.g. --quantization fp8).
 
@@ -51,9 +52,17 @@ if [[ "$version" == "" || $version -lt 177 ]]; then
   export HSA_NO_SCRATCH_RECLAIM=1
 fi
 
-# gfx942 AITER infra (dsr1_fp8_mi325x.sh) + deepseek_v4 model env (mi355x).
+# gfx942 AITER infra (dsr1_fp8_mi325x.sh, for the MoE GEMMs) + deepseek_v4
+# model env (dsv4_fp4_mi355x_sglang.sh). SGLANG_HACK_FLASHMLA_BACKEND=
+# unified_kv_triton is load-bearing: without it the dsv4 attention backend
+# compiles its FP8 MLA kernel via TileLang, whose InjectSoftwarePipeline pass
+# fails on gfx942 ("buffer access dependency ... cannot be reordered") and
+# kills the server at CUDA-graph capture. The unified-KV triton FlashMLA path
+# avoids that kernel. (SGLANG_AITER_MLA_PERSIST dropped: MLA no longer runs
+# through aiter.)
 export SGLANG_USE_AITER=1
-export SGLANG_AITER_MLA_PERSIST=1
+export SGLANG_USE_ROCM700A=0
+export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
 export SGLANG_DEFAULT_THINKING=1
 export SGLANG_DSV4_REASONING_EFFORT=max
 export AITER_BF16_FP8_MOE_BOUND=0
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 0b2f54bfc..30e856c99 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4757,6 +4757,7 @@
     - "Add DeepSeek-V4-Pro FP8 single-node MI325X SGLang benchmark (new SKU, previously no dsv4 data on gfx942)"
     - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi325x-sglang for gfx942 AITER infra; gfx942 has no native FP4 so the FP4-MoE checkpoint runs in FP8"
     - "No --quantization flag: sglang reads the modelopt quant config from the checkpoint and the AITER MoE path executes it, matching the H200 and MI355X dsv4 sglang recipes (--quantization deepseek_v4_fp8 is vLLM-only and rejected by sglang argparse)"
+    - "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton (+ SGLANG_USE_ROCM700A=0), mirroring dsv4-fp4-mi355x-sglang: the dsv4 attention backend's TileLang FP8 MLA kernel fails to compile on gfx942 (InjectSoftwarePipeline reorder check) and kills the server at CUDA-graph capture; the unified-KV triton FlashMLA path avoids it"
     - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8; 8x256GB (2TB) leaves ~1TB free after FP8 weights so the throughput band could later extend past conc 2048 (kept at parity for a first bring-up)"
     - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2195

From 7fd44e754b223f69e98083f4f8ca46460fa68556 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 19:46:59 +0800
Subject: [PATCH 05/15] fix(dsv4): use triton MoE runner for MI325X (CK
 unsupported on gfx942)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Same as MI300X: after the unified_kv_triton MLA fix, the server was killed
at CUDA-graph capture (run 29328296892 lineage) in the AITER Composable-Kernel
fused-MoE (ck_moe_stage1) — "Unsupported kernel config for moe heuristic
dispatch", no CK tuned config for dsv4's FP8 MoE shapes on gfx942. Add
--moe-runner-backend triton (JIT per-shape, no pretuned table).

中文：与 MI300X 相同——AITER CK fused-MoE 在 gfx942 上缺少 dsv4 FP8 MoE 形状
的调优配置，改用 --moe-runner-backend triton。
---
 .../single_node/fixed_seq_len/dsv4_fp8_mi325x.sh     | 12 ++++++++++--
 perf-changelog.yaml                                  |  1 +
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
index c99dbbe09..0a9fc82fe 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
@@ -18,12 +18,19 @@
 # Sizing: the ~960GB mixed checkpoint is ~1.05TB in FP8, so only TP8 fits
 # 8x256GB comfortably (TP4 = 1024GB is too tight for ~1.05TB). The config restricts to TP8 accordingly.
 #
+# MoE runner: triton, not the default AITER Composable-Kernel path. The CK
+# fused-MoE kernel (ck_moe_stage1) has no tuned config for DeepSeek-V4's FP8
+# MoE shapes on gfx942 and aborts CUDA-graph capture with "Unsupported kernel
+# config for moe heuristic dispatch"; triton JIT-generates per-shape so it
+# needs no pretuned table. (The MI355X/gfx950 recipe uses the CK default,
+# which does have gfx950 configs.)
+#
 # Debug-runs must confirm before this leaves bring-up:
 #   1. The mi30x image carries the DeepseekV4 model class.
 #   2. The unified-KV triton FlashMLA path captures CUDA graphs cleanly on
 #      gfx942 (the TileLang FP8 MLA kernel does not — see the env note below).
-#   3. The modelopt FP4-MoE checkpoint loads + runs in FP8 via AITER on gfx942
-#      (else pass an explicit valid sglang quant, e.g. --quantization fp8).
+#   3. The triton MoE runner supports the dsv4 FP8 MoE shapes on gfx942 (the
+#      AITER CK path does not); confirm throughput is acceptable vs CK.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -110,6 +117,7 @@ sglang serve \
     --trust-remote-code \
     --kv-cache-dtype fp8_e4m3 \
     --attention-backend dsv4 \
+    --moe-runner-backend triton \
     --disable-radix-cache \
     --disable-shared-experts-fusion \
     --page-size 256 \
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 30e856c99..9e0e251ea 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4758,6 +4758,7 @@
     - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi325x-sglang for gfx942 AITER infra; gfx942 has no native FP4 so the FP4-MoE checkpoint runs in FP8"
     - "No --quantization flag: sglang reads the modelopt quant config from the checkpoint and the AITER MoE path executes it, matching the H200 and MI355X dsv4 sglang recipes (--quantization deepseek_v4_fp8 is vLLM-only and rejected by sglang argparse)"
     - "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton (+ SGLANG_USE_ROCM700A=0), mirroring dsv4-fp4-mi355x-sglang: the dsv4 attention backend's TileLang FP8 MLA kernel fails to compile on gfx942 (InjectSoftwarePipeline reorder check) and kills the server at CUDA-graph capture; the unified-KV triton FlashMLA path avoids it"
+    - "--moe-runner-backend triton: the default AITER Composable-Kernel fused-MoE (ck_moe_stage1) has no tuned config for dsv4's FP8 MoE shapes on gfx942 and aborts CUDA-graph capture ('Unsupported kernel config for moe heuristic dispatch'); triton JIT-generates per-shape so it needs no pretuned table (MI355X/gfx950 keeps the CK default, which has gfx950 configs)"
     - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8; 8x256GB (2TB) leaves ~1TB free after FP8 weights so the throughput band could later extend past conc 2048 (kept at parity for a first bring-up)"
     - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2195

From ec0fdfd791b265df3d458b7a2832553a85be9924 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 20:17:00 +0800
Subject: [PATCH 06/15] fix(dsv4): switch MI325X dsv4 to vLLM FP8 (sglang
 blocked on gfx942)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Same pivot as MI300X: sglang has no gfx942 build of the dsv4 nvfp4 MoE /
TileLang-MLA kernels. vLLM runs the checkpoint in FP8 via --quantization
deepseek_v4_fp8 (dequant FP4 MoE -> FP8). Recipe mirrors dsv4-fp4-mi355x-vllm
+ gfx942 AITER infra; TP8 conc 4-512. Config key renamed
dsv4-fp8-mi325x-sglang -> dsv4-fp8-mi325x-vllm.

中文：MI325X dsv4 与 MI300X 一样由 sglang 改为 vLLM，通过
--quantization deepseek_v4_fp8 将 FP4 MoE 反量化为 FP8 运行。
---
 .../fixed_seq_len/dsv4_fp8_mi325x.sh          | 138 ++++++------------
 configs/amd-master.yaml                       |  33 ++---
 perf-changelog.yaml                           |  15 +-
 3 files changed, 61 insertions(+), 125 deletions(-)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
index 0a9fc82fe..37ec91257 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
@@ -1,36 +1,22 @@
 #!/usr/bin/env bash
+set -eo pipefail
 
-# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via SGLang.
+# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM.
 #
-# EXTRAPOLATED bring-up recipe (unvalidated on-cluster as of 2026-07),
-# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus:
-#   * same model, adjacent SKU: dsv4_fp4_mi355x_sglang.sh (search-space
-#     shape, DP-attention + EP path, deepseek_v4 model flags, SWA/page-size)
-#   * same SKU, different model: dsr1_fp8_mi325x.sh (gfx942 infra: AITER,
-#     MEC-firmware scratch-reclaim guard)
-#   * same model + framework: dsv4_fp8_h200_sglang.sh — gfx942 has no native
-#     FP4, so the FP4-MoE checkpoint is run in FP8. Like the H200 sglang
-#     recipe (and the MI355X dsv4 sglang recipe), NO --quantization flag is
-#     passed: sglang reads the modelopt quant config from the checkpoint and
-#     the AITER MoE path (SGLANG_USE_AITER=1) executes it. (--quantization
-#     deepseek_v4_fp8 is a vLLM-only method; sglang argparse rejects it.)
+# EXTRAPOLATED bring-up recipe. The sglang path was abandoned: on gfx942
+# (no native FP4) the dsv4 sglang backend's nvfp4 MoE / TileLang-MLA kernels
+# have no gfx942 equivalents (they exist only for gfx950/MI355X). vLLM instead
+# runs the checkpoint in FP8 via --quantization deepseek_v4_fp8, which
+# dequantizes the FP4 MoE experts to FP8 — the same path the H200 dsv4 vLLM
+# recipe uses (H200 is also a no-FP4 SKU). Derived from:
+#   * same model + framework + AMD family: dsv4_fp4_mi355x_vllm.sh (ROCm vLLM
+#     dsv4 structure: AITER MoE, deepseek_v4 tokenizer/parser, mp executor,
+#     FULL_AND_PIECEWISE compile)
+#   * same model, FP8 path: dsv4_fp8_h200.sh (--quantization deepseek_v4_fp8)
+#   * same SKU, different model: minimaxm3_fp8_mi325x.sh (gfx942 vLLM/AITER)
 #
-# Sizing: the ~960GB mixed checkpoint is ~1.05TB in FP8, so only TP8 fits
-# 8x256GB comfortably (TP4 = 1024GB is too tight for ~1.05TB). The config restricts to TP8 accordingly.
-#
-# MoE runner: triton, not the default AITER Composable-Kernel path. The CK
-# fused-MoE kernel (ck_moe_stage1) has no tuned config for DeepSeek-V4's FP8
-# MoE shapes on gfx942 and aborts CUDA-graph capture with "Unsupported kernel
-# config for moe heuristic dispatch"; triton JIT-generates per-shape so it
-# needs no pretuned table. (The MI355X/gfx950 recipe uses the CK default,
-# which does have gfx950 configs.)
-#
-# Debug-runs must confirm before this leaves bring-up:
-#   1. The mi30x image carries the DeepseekV4 model class.
-#   2. The unified-KV triton FlashMLA path captures CUDA graphs cleanly on
-#      gfx942 (the TileLang FP8 MLA kernel does not — see the env note below).
-#   3. The triton MoE runner supports the dsv4 FP8 MoE shapes on gfx942 (the
-#      AITER CK path does not); confirm throughput is acceptable vs CK.
+# The FP4->FP8 dequant roughly doubles the MoE footprint (~1.05 TB total),
+# which fits 8x256 GB comfortably at TP8, so the sweep is TP8-only.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -38,13 +24,12 @@ check_env_vars \
     MODEL \
     TP \
     DP_ATTENTION \
-    EP_SIZE \
     CONC \
     ISL \
     OSL \
+    MAX_MODEL_LEN \
     RANDOM_RANGE_RATIO \
-    RESULT_FILENAME \
-    MAX_MODEL_LEN
+    RESULT_FILENAME
 
 if [[ -n "$SLURM_JOB_ID" ]]; then
   echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
@@ -52,89 +37,51 @@ fi
 
 if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi
 
-# If the machine runs a MEC FW older than 177, RCCL cannot reclaim some
-# memory; disable that feature to avoid crashes (see dsr1_fp8_mi325x.sh).
-version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'`
-if [[ "$version" == "" || $version -lt 177 ]]; then
-  export HSA_NO_SCRATCH_RECLAIM=1
+if [ -n "$ROCR_VISIBLE_DEVICES" ]; then
+    export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
 fi
 
-# gfx942 AITER infra (dsr1_fp8_mi325x.sh, for the MoE GEMMs) + deepseek_v4
-# model env (dsv4_fp4_mi355x_sglang.sh). SGLANG_HACK_FLASHMLA_BACKEND=
-# unified_kv_triton is load-bearing: without it the dsv4 attention backend
-# compiles its FP8 MLA kernel via TileLang, whose InjectSoftwarePipeline pass
-# fails on gfx942 ("buffer access dependency ... cannot be reordered") and
-# kills the server at CUDA-graph capture. The unified-KV triton FlashMLA path
-# avoids that kernel. (SGLANG_AITER_MLA_PERSIST dropped: MLA no longer runs
-# through aiter.)
-export SGLANG_USE_AITER=1
-export SGLANG_USE_ROCM700A=0
-export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
-export SGLANG_DEFAULT_THINKING=1
-export SGLANG_DSV4_REASONING_EFFORT=max
-export AITER_BF16_FP8_MOE_BOUND=0
+export VLLM_ROCM_USE_AITER=1
+export VLLM_ROCM_USE_AITER_MOE=1
 
 SERVER_LOG=/workspace/server.log
 
-EVAL_CONTEXT_ARGS=""
 if [ "${EVAL_ONLY}" = "true" ]; then
     setup_eval_context
-    EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN"
+    MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN"
 fi
 
-# Start GPU monitoring (power, temperature, clocks every second)
 start_gpu_monitor
 
-# DP_ATTENTION=true runs DP-attention with expert parallel (dp-size = TP),
-# mirroring dsv4_fp4_mi355x_sglang.sh; false runs pure tensor parallel for
-# the low-latency band.
-PARALLEL_ARGS=(--tensor-parallel-size "$TP")
-CHUNKED_PREFILL_SIZE=$ISL
+PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
 if [ "${DP_ATTENTION}" = "true" ]; then
-    export SGLANG_SHARED_EXPERT_TP1=1
-    export SGLANG_DP_SHARED_EXPERT_LOCAL=1
-    export SGLANG_DP_USE_GATHERV=1
-    export SGLANG_DP_USE_REDUCE_SCATTER=1
-    export GPU_MAX_HW_QUEUES=5
-
-    CHUNKED_PREFILL_SIZE=$((ISL * TP))
-    PARALLEL_ARGS+=(
-        --dp "$TP"
-        --enable-dp-attention
-        --enable-prefill-delayer
-        --enable-two-batch-overlap
-    )
+    PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
 fi
+
+EP_ARGS=()
 if [ "${EP_SIZE:-1}" -gt 1 ]; then
-    PARALLEL_ARGS+=(--ep-size "$EP_SIZE")
+    EP_ARGS=(--enable-expert-parallel)
 fi
 
 set -x
-sglang serve \
-    --model-path $MODEL --served-model-name $MODEL \
-    --host=0.0.0.0 --port $PORT \
+vllm serve $MODEL --port $PORT \
     "${PARALLEL_ARGS[@]}" \
+    "${EP_ARGS[@]}" \
+    --quantization deepseek_v4_fp8 \
+    --async-scheduling \
+    --no-enable-prefix-caching \
+    --distributed-executor-backend mp \
+    --gpu-memory-utilization 0.9 \
+    --max-model-len "$MAX_MODEL_LEN" \
+    --kv-cache-dtype fp8 \
     --trust-remote-code \
-    --kv-cache-dtype fp8_e4m3 \
-    --attention-backend dsv4 \
-    --moe-runner-backend triton \
-    --disable-radix-cache \
-    --disable-shared-experts-fusion \
-    --page-size 256 \
-    --mem-fraction-static 0.90 \
-    --swa-full-tokens-ratio 0.15 \
-    --cuda-graph-max-bs ${CONC} \
-    --max-running-requests ${CONC} \
-    --context-length $MAX_MODEL_LEN \
-    --chunked-prefill-size $CHUNKED_PREFILL_SIZE \
-    --tool-call-parser deepseekv4 \
-    --reasoning-parser deepseek-v4 \
-    --chat-template "$(dirname "$0")/../chat_templates/deepseek_v4_thinking.jinja" \
-    --watchdog-timeout 1800 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 &
+    --moe-backend aiter \
+    --tokenizer-mode deepseek_v4 \
+    --reasoning-parser deepseek_v4 \
+    --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 &
 
 SERVER_PID=$!
 
-# Wait for server to be ready
 wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
 
 run_benchmark_serving \
@@ -147,14 +94,13 @@ run_benchmark_serving \
     --num-prompts "$((CONC * 10))" \
     --max-concurrency "$CONC" \
     --result-filename "$RESULT_FILENAME" \
-    --result-dir /workspace/
+    --result-dir /workspace/ \
+    --trust-remote-code
 
-# After throughput, run evaluation only if RUN_EVAL is true
 if [ "${RUN_EVAL}" = "true" ]; then
     run_eval --framework lm-eval --port "$PORT"
     append_lm_eval_summary
 fi
 
-# Stop GPU monitoring
 stop_gpu_monitor
 set +x
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index a283edfcb..8ed793e01 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -119,39 +119,32 @@ dsr1-fp8-mi325x-sglang:
       search-space:
       - { tp: 8, conc-start: 4, conc-end: 64 }
 
-# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via SGLang.
-# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07).
-# Search-space shape mirrors the same-model MI355X entry (dsv4-fp4-mi355x-sglang):
-# a TP-only low-latency band + a DP-attention + EP throughput band. gfx942 has
-# no native FP4, so this runs FP8 (the ~960GB mixed checkpoint is ~1.05TB in
-# FP8) — which only fits TP8, so the MI355X TP4 bands are dropped. 8x256GB (2TB)
-# leaves ~1TB free after FP8 weights, so the throughput band could later extend
-# past conc 2048 — kept at parity with the sister MI300X entry for a first
-# bring-up. Image follows the same-model MI355X's sglang-rocm dated-tag family
-# (mi30x variant); launch script dsv4_fp8_mi325x.sh carries the deepseek_v4
-# flags + gfx942 AITER infra. debug-runs must confirm the image carries the
-# DeepseekV4 class and that --attention-backend dsv4 / --quantization
-# deepseek_v4_fp8 work on gfx942 before this merges.
-dsv4-fp8-mi325x-sglang:
-  image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713
+# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM.
+# EXTRAPOLATED bring-up. Same rationale as dsv4-fp8-mi300x-vllm: sglang has no
+# gfx942 build of the dsv4 nvfp4 MoE / TileLang-MLA kernels, so vLLM runs the
+# checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8),
+# the H200 dsv4 vLLM path. Config mirrors the same-model dsv4-fp4-mi355x-vllm
+# (TP8, conc 4-512); 8x256GB (2TB) has ample headroom for the ~1.05TB FP8
+# footprint. Launch script dsv4_fp8_mi325x.sh carries the deepseek_v4 + gfx942
+# AITER flags.
+dsv4-fp8-mi325x-vllm:
+  image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: mi325x
   precision: fp8
-  framework: sglang
+  framework: vllm
   multinode: false
   scenarios:
     fixed-seq-len:
     - isl: 1024
       osl: 1024
       search-space:
-      - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32 }
-      - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 2048 }
+      - { tp: 8, conc-start: 4, conc-end: 512 }
     - isl: 8192
       osl: 1024
       search-space:
-      - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32 }
-      - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 2048 }
+      - { tp: 8, conc-start: 4, conc-end: 512 }
 
 dsr1-fp8-mi355x-sglang:
   image: lmsysorg/sglang:v0.5.12-rocm700-mi35x
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 9e0e251ea..d78c842da 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4752,15 +4752,12 @@
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137
 
 - config-keys:
-    - dsv4-fp8-mi325x-sglang
-  description:
-    - "Add DeepSeek-V4-Pro FP8 single-node MI325X SGLang benchmark (new SKU, previously no dsv4 data on gfx942)"
-    - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi325x-sglang for gfx942 AITER infra; gfx942 has no native FP4 so the FP4-MoE checkpoint runs in FP8"
-    - "No --quantization flag: sglang reads the modelopt quant config from the checkpoint and the AITER MoE path executes it, matching the H200 and MI355X dsv4 sglang recipes (--quantization deepseek_v4_fp8 is vLLM-only and rejected by sglang argparse)"
-    - "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton (+ SGLANG_USE_ROCM700A=0), mirroring dsv4-fp4-mi355x-sglang: the dsv4 attention backend's TileLang FP8 MLA kernel fails to compile on gfx942 (InjectSoftwarePipeline reorder check) and kills the server at CUDA-graph capture; the unified-KV triton FlashMLA path avoids it"
-    - "--moe-runner-backend triton: the default AITER Composable-Kernel fused-MoE (ck_moe_stage1) has no tuned config for dsv4's FP8 MoE shapes on gfx942 and aborts CUDA-graph capture ('Unsupported kernel config for moe heuristic dispatch'); triton JIT-generates per-shape so it needs no pretuned table (MI355X/gfx950 keeps the CK default, which has gfx950 configs)"
-    - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8; 8x256GB (2TB) leaves ~1TB free after FP8 weights so the throughput band could later extend past conc 2048 (kept at parity for a first bring-up)"
-    - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
+    - dsv4-fp8-mi325x-vllm
+  description:
+    - "Add DeepSeek-V4-Pro FP8 single-node MI325X vLLM benchmark (new SKU, previously no dsv4 data on gfx942)"
+    - "vLLM, not sglang: same rationale as dsv4-fp8-mi300x-vllm — sglang has no gfx942 build of the dsv4 nvfp4 MoE / TileLang-MLA kernels (gfx950/MI355X only). vLLM runs the checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), the no-FP4 H200 dsv4 vLLM path"
+    - "Recipe mirrors the same-model dsv4-fp4-mi355x-vllm (ROCm vLLM dsv4: AITER MoE, deepseek_v4 tokenizer/reasoning-parser, mp executor, FULL_AND_PIECEWISE compile) plus gfx942 AITER infra from minimaxm3-fp8-mi325x-vllm; TP8 conc 4-512, ample headroom on 8x256GB"
+    - "Image vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (the DSv4-validated ROCm vLLM build)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2195
 
 - config-keys:

From f2b2fe69f52362b91f6c3d15da58ba8fdf492382 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 21:16:55 +0800
Subject: [PATCH 07/15] =?UTF-8?q?fix(dsv4):=20MI325X=20vLLM=20=E2=80=94=20?=
 =?UTF-8?q?use=20auto=20MoE=20backend=20(AITER=5FMXFP4=5FBF16)=20on=20gfx9?=
 =?UTF-8?q?42?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Same fix as MI300X: drop --moe-backend aiter so vLLM's
select_deepseek_v4_mxfp4_moe_backend takes its ROCm+DeepseekV4 branch and
prefers AITER_MXFP4_BF16 (W4A16, dequantizes mxfp4 weights) instead of the
gfx942-unsupported AITER_MXFP4_MXFP4 (W4A4 native mxfp4).

中文：与 MI300X 相同的修复——去掉 --moe-backend aiter，让 vLLM 在
ROCm+DeepseekV4 分支选 AITER_MXFP4_BF16（反量化），避免 gfx942 不支持的
AITER_MXFP4_MXFP4。
---
 benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh | 6 +++++-
 perf-changelog.yaml                                     | 3 ++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
index 37ec91257..5ceb56edd 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
@@ -17,6 +17,11 @@ set -eo pipefail
 #
 # The FP4->FP8 dequant roughly doubles the MoE footprint (~1.05 TB total),
 # which fits 8x256 GB comfortably at TP8, so the sweep is TP8-only.
+#
+# MoE backend is left at auto (NOT --moe-backend aiter) — see dsv4_fp8_mi300x.sh:
+# on gfx942, forcing aiter selects AITER_MXFP4_MXFP4 (W4A4 native-mxfp4) which
+# the gfx942 kernel rejects; auto's ROCm+DeepseekV4 path prefers
+# AITER_MXFP4_BF16 (W4A16, dequant) with a TRITON_UNFUSED fallback.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -75,7 +80,6 @@ vllm serve $MODEL --port $PORT \
     --max-model-len "$MAX_MODEL_LEN" \
     --kv-cache-dtype fp8 \
     --trust-remote-code \
-    --moe-backend aiter \
     --tokenizer-mode deepseek_v4 \
     --reasoning-parser deepseek_v4 \
     --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 &
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index d78c842da..936314f55 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4756,7 +4756,8 @@
   description:
     - "Add DeepSeek-V4-Pro FP8 single-node MI325X vLLM benchmark (new SKU, previously no dsv4 data on gfx942)"
     - "vLLM, not sglang: same rationale as dsv4-fp8-mi300x-vllm — sglang has no gfx942 build of the dsv4 nvfp4 MoE / TileLang-MLA kernels (gfx950/MI355X only). vLLM runs the checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), the no-FP4 H200 dsv4 vLLM path"
-    - "Recipe mirrors the same-model dsv4-fp4-mi355x-vllm (ROCm vLLM dsv4: AITER MoE, deepseek_v4 tokenizer/reasoning-parser, mp executor, FULL_AND_PIECEWISE compile) plus gfx942 AITER infra from minimaxm3-fp8-mi325x-vllm; TP8 conc 4-512, ample headroom on 8x256GB"
+    - "Recipe mirrors the same-model dsv4-fp4-mi355x-vllm (ROCm vLLM dsv4: deepseek_v4 tokenizer/reasoning-parser, mp executor, FULL_AND_PIECEWISE compile) plus gfx942 AITER infra from minimaxm3-fp8-mi325x-vllm; TP8 conc 4-512, ample headroom on 8x256GB"
+    - "MoE backend left at auto (unlike MI355X's --moe-backend aiter): forcing aiter on gfx942 selects AITER_MXFP4_MXFP4 (W4A4 native-mxfp4) which the gfx942 kernel rejects; auto's ROCm+DeepseekV4 path prefers AITER_MXFP4_BF16 (W4A16, dequant) with TRITON_UNFUSED fallback (see dsv4-fp8-mi300x-vllm)"
     - "Image vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (the DSv4-validated ROCm vLLM build)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2195
 

From 56767f1fd8cf0ff3708641e95f246c55239606b7 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 01:03:28 +0800
Subject: [PATCH 08/15] feat(dsv4): add MI325X vLLM MTP spec-decode variant
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

dsv4-fp8-mi325x-vllm-mtp mirrors the normal recipe + DeepSeek-V4 built-in MTP
(--speculative-config mtp, num_speculative_tokens=2) with --dsv4 chat-template
encoding. Routes to dsv4_fp8_mi325x_mtp.sh. 256GB keeps default eval concurrency.

中文：新增 MI325X vLLM MTP 变体（--speculative-config mtp + --dsv4）。
---
 .../fixed_seq_len/dsv4_fp8_mi325x_mtp.sh      | 119 ++++++++++++++++++
 configs/amd-master.yaml                       |  23 ++++
 perf-changelog.yaml                           |   8 ++
 3 files changed, 150 insertions(+)
 create mode 100755 benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x_mtp.sh

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x_mtp.sh
new file mode 100755
index 000000000..26ee3e14f
--- /dev/null
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x_mtp.sh
@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM, MTP variant.
+#
+# MTP sibling of dsv4_fp8_mi325x.sh: adds --speculative-config
+# '{"method":"mtp","num_speculative_tokens":2}' (DeepSeek-V4 built-in MTP)
+# and --dsv4 chat-template encoding for run_benchmark_serving.
+#
+# EXTRAPOLATED bring-up recipe. The sglang path was abandoned: on gfx942
+# (no native FP4) the dsv4 sglang backend's nvfp4 MoE / TileLang-MLA kernels
+# have no gfx942 equivalents (they exist only for gfx950/MI355X). vLLM instead
+# runs the checkpoint in FP8 via --quantization deepseek_v4_fp8, which
+# dequantizes the FP4 MoE experts to FP8 — the same path the H200 dsv4 vLLM
+# recipe uses (H200 is also a no-FP4 SKU). Derived from:
+#   * same model + framework + AMD family: dsv4_fp4_mi355x_vllm.sh (ROCm vLLM
+#     dsv4 structure: AITER MoE, deepseek_v4 tokenizer/parser, mp executor,
+#     FULL_AND_PIECEWISE compile)
+#   * same model, FP8 path: dsv4_fp8_h200.sh (--quantization deepseek_v4_fp8)
+#   * same SKU, different model: minimaxm3_fp8_mi325x.sh (gfx942 vLLM/AITER)
+#
+# The FP4->FP8 dequant roughly doubles the MoE footprint (~1.05 TB total),
+# which fits 8x256 GB comfortably at TP8, so the sweep is TP8-only.
+#
+# MoE backend is left at auto (NOT --moe-backend aiter) — see dsv4_fp8_mi300x.sh:
+# on gfx942, forcing aiter selects AITER_MXFP4_MXFP4 (W4A4 native-mxfp4) which
+# the gfx942 kernel rejects; auto's ROCm+DeepseekV4 path prefers
+# AITER_MXFP4_BF16 (W4A16, dequant) with a TRITON_UNFUSED fallback.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars \
+    MODEL \
+    TP \
+    DP_ATTENTION \
+    CONC \
+    ISL \
+    OSL \
+    MAX_MODEL_LEN \
+    RANDOM_RANGE_RATIO \
+    RESULT_FILENAME
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+  echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi
+
+if [ -n "$ROCR_VISIBLE_DEVICES" ]; then
+    export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
+fi
+
+export VLLM_ROCM_USE_AITER=1
+export VLLM_ROCM_USE_AITER_MOE=1
+
+SERVER_LOG=/workspace/server.log
+
+if [ "${EVAL_ONLY}" = "true" ]; then
+    setup_eval_context
+    MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN"
+fi
+
+start_gpu_monitor
+
+PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
+if [ "${DP_ATTENTION}" = "true" ]; then
+    PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
+fi
+
+EP_ARGS=()
+if [ "${EP_SIZE:-1}" -gt 1 ]; then
+    EP_ARGS=(--enable-expert-parallel)
+fi
+
+# Use 2 speculative tokens (matches dsv4_fp4_mi355x_vllm_mtp.sh).
+NUM_SPEC_TOKENS=2
+
+set -x
+vllm serve $MODEL --port $PORT \
+    "${PARALLEL_ARGS[@]}" \
+    "${EP_ARGS[@]}" \
+    --quantization deepseek_v4_fp8 \
+    --async-scheduling \
+    --no-enable-prefix-caching \
+    --distributed-executor-backend mp \
+    --gpu-memory-utilization 0.9 \
+    --max-model-len "$MAX_MODEL_LEN" \
+    --kv-cache-dtype fp8 \
+    --trust-remote-code \
+    --tokenizer-mode deepseek_v4 \
+    --reasoning-parser deepseek_v4 \
+    --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \
+    --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 &
+
+SERVER_PID=$!
+
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+run_benchmark_serving \
+    --model "$MODEL" \
+    --port "$PORT" \
+    --backend vllm \
+    --input-len "$ISL" \
+    --output-len "$OSL" \
+    --random-range-ratio "$RANDOM_RANGE_RATIO" \
+    --num-prompts "$((CONC * 10))" \
+    --max-concurrency "$CONC" \
+    --result-filename "$RESULT_FILENAME" \
+    --result-dir /workspace/ \
+    --trust-remote-code \
+    --dsv4
+
+if [ "${RUN_EVAL}" = "true" ]; then
+    run_eval --framework lm-eval --port "$PORT"
+    append_lm_eval_summary
+fi
+
+stop_gpu_monitor
+set +x
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index 8ed793e01..2986d7d60 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -146,6 +146,29 @@ dsv4-fp8-mi325x-vllm:
       search-space:
       - { tp: 8, conc-start: 4, conc-end: 512 }
 
+# MTP variant of dsv4-fp8-mi325x-vllm. Mirrors the base recipe and adds
+# DeepSeek-V4 built-in MTP via --speculative-config (num_speculative_tokens=2),
+# routing to dsv4_fp8_mi325x_mtp.sh; benchmark uses --dsv4 chat-template
+# encoding (required for meaningful MTP acceptance).
+dsv4-fp8-mi325x-vllm-mtp:
+  image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: mi325x
+  precision: fp8
+  framework: vllm
+  multinode: false
+  scenarios:
+    fixed-seq-len:
+    - isl: 1024
+      osl: 1024
+      search-space:
+      - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp }
+    - isl: 8192
+      osl: 1024
+      search-space:
+      - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp }
+
 dsr1-fp8-mi355x-sglang:
   image: lmsysorg/sglang:v0.5.12-rocm700-mi35x
   model: deepseek-ai/DeepSeek-R1-0528
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 936314f55..e8c2bc30f 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4775,3 +4775,11 @@
     - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708"
     - "Clean the export envs"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198
+
+- config-keys:
+    - dsv4-fp8-mi325x-vllm-mtp
+  description:
+    - "Add DeepSeek-V4-Pro FP8 MI325X vLLM MTP (speculative-decoding) variant, mirroring dsv4-fp8-mi325x-vllm plus --speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":2} (DeepSeek-V4 built-in MTP)"
+    - "run_benchmark_serving uses --dsv4 chat-template encoding, required for meaningful MTP acceptance rate; routes to dsv4_fp8_mi325x_mtp.sh via the launcher's spec-decoding=mtp suffix"
+    - "256GB MI325X keeps the default eval concurrency (no KV cap needed)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2195

From 04d4f7316e50c62f8eb32f28b4ae3e2bf92f4107 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 01:25:06 +0800
Subject: [PATCH 09/15] fix(dsv4): rebase MI325X perf-changelog onto
 origin/main (append-only)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

origin/main advanced (#2191 merged); reset perf-changelog.yaml to origin/main
and re-append the dsv4-fp8-mi325x-vllm and dsv4-fp8-mi325x-vllm-mtp entries.

中文：将 MI325X 分支 perf-changelog 重置到 origin/main 并重新追加其条目。
---
 perf-changelog.yaml | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index e8c2bc30f..eec5dde49 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4752,14 +4752,20 @@
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137
 
 - config-keys:
-    - dsv4-fp8-mi325x-vllm
+    - dsv4-fp8-h200-vllm
+    - dsv4-fp8-h200-vllm-mtp
   description:
-    - "Add DeepSeek-V4-Pro FP8 single-node MI325X vLLM benchmark (new SKU, previously no dsv4 data on gfx942)"
-    - "vLLM, not sglang: same rationale as dsv4-fp8-mi300x-vllm — sglang has no gfx942 build of the dsv4 nvfp4 MoE / TileLang-MLA kernels (gfx950/MI355X only). vLLM runs the checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), the no-FP4 H200 dsv4 vLLM path"
-    - "Recipe mirrors the same-model dsv4-fp4-mi355x-vllm (ROCm vLLM dsv4: deepseek_v4 tokenizer/reasoning-parser, mp executor, FULL_AND_PIECEWISE compile) plus gfx942 AITER infra from minimaxm3-fp8-mi325x-vllm; TP8 conc 4-512, ample headroom on 8x256GB"
-    - "MoE backend left at auto (unlike MI355X's --moe-backend aiter): forcing aiter on gfx942 selects AITER_MXFP4_MXFP4 (W4A4 native-mxfp4) which the gfx942 kernel rejects; auto's ROCm+DeepseekV4 path prefers AITER_MXFP4_BF16 (W4A16, dequant) with TRITON_UNFUSED fallback (see dsv4-fp8-mi300x-vllm)"
-    - "Image vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (the DSv4-validated ROCm vLLM build)"
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2195
+    - "Bump vLLM image from v0.21.0 to v0.25.0 for DeepSeek-V4-Pro FP8 on H200, matching the B200/B300 dsv4 vLLM bump (#2169)"
+    - "Refresh stale H200 dsv4 submissions (last run 2026-05-21)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2191
+
+- config-keys:
+    - dsv4-fp8-h200-sglang
+    - dsv4-fp8-h200-sglang-mtp
+  description:
+    - "Bump the pinned lmsysorg/sglang:deepseek-v4-hopper digest from the 2026-05-02 push (7f19c6dc) to the current 2026-05-13 push (1bf5d508)"
+    - "Refresh stale H200 dsv4 SGLang submissions (last run 2026-05-04)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2191
 
 - config-keys:
     - dsv4-fp4-mi355x-sglang
@@ -4776,6 +4782,16 @@
     - "Clean the export envs"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198
 
+- config-keys:
+    - dsv4-fp8-mi325x-vllm
+  description:
+    - "Add DeepSeek-V4-Pro FP8 single-node MI325X vLLM benchmark (new SKU, previously no dsv4 data on gfx942)"
+    - "vLLM, not sglang: same rationale as dsv4-fp8-mi300x-vllm — sglang has no gfx942 build of the dsv4 nvfp4 MoE / TileLang-MLA kernels (gfx950/MI355X only). vLLM runs the checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), the no-FP4 H200 dsv4 vLLM path"
+    - "Recipe mirrors the same-model dsv4-fp4-mi355x-vllm (ROCm vLLM dsv4: deepseek_v4 tokenizer/reasoning-parser, mp executor, FULL_AND_PIECEWISE compile) plus gfx942 AITER infra from minimaxm3-fp8-mi325x-vllm; TP8 conc 4-512, ample headroom on 8x256GB"
+    - "MoE backend left at auto (unlike MI355X's --moe-backend aiter): forcing aiter on gfx942 selects AITER_MXFP4_MXFP4 (W4A4 native-mxfp4) which the gfx942 kernel rejects; auto's ROCm+DeepseekV4 path prefers AITER_MXFP4_BF16 (W4A16, dequant) with TRITON_UNFUSED fallback (see dsv4-fp8-mi300x-vllm)"
+    - "Image vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (the DSv4-validated ROCm vLLM build)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2195
+
 - config-keys:
     - dsv4-fp8-mi325x-vllm-mtp
   description:

From f454dc7839ae4a54712f460e106c90b526565f8b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 04:24:42 +0800
Subject: [PATCH 10/15] fix(dsv4): cap MI325X 8k1k conc at 256; clamp eval
 concurrency

8k1k KV fits only ~20x concurrency (9472-tok requests); conc512 is ~25x
oversubscribed -> request timeouts (MTP c512 8k1k: 55.8% failures). conc256
passed for both normal and MTP on MI300X/MI325X, so cap 8k1k conc-end at 256
(1k1k holds at 512). Also add EVAL_CONCURRENT_REQUESTS=8 (as MI300X already has)
so the gsm8k eval runs at a safe in-flight count instead of crashing the server
at high concurrency.
---
 benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh   | 7 +++++++
 .../single_node/fixed_seq_len/dsv4_fp8_mi325x_mtp.sh      | 7 +++++++
 configs/amd-master.yaml                                   | 8 ++++++--
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
index 5ceb56edd..4dafff650 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x.sh
@@ -49,6 +49,13 @@ fi
 export VLLM_ROCM_USE_AITER=1
 export VLLM_ROCM_USE_AITER_MOE=1
 
+# gsm8k eval at high concurrency (8k1k) OOM-kills the server: hundreds of
+# concurrent 9472-token requests exceed the ~20x KV budget even on 256GB MI325X
+# (c128 fit, so this was originally left at the default, but c512 crashed the
+# EngineCore mid-eval). Cap the eval to a safe in-flight count; only run_eval is
+# affected (throughput jobs use CONC directly). Matches the MI300X script.
+export EVAL_CONCURRENT_REQUESTS=8
+
 SERVER_LOG=/workspace/server.log
 
 if [ "${EVAL_ONLY}" = "true" ]; then
diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x_mtp.sh
index 26ee3e14f..7233da9dd 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x_mtp.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi325x_mtp.sh
@@ -53,6 +53,13 @@ fi
 export VLLM_ROCM_USE_AITER=1
 export VLLM_ROCM_USE_AITER_MOE=1
 
+# gsm8k eval at high concurrency (8k1k) OOM-kills the server: hundreds of
+# concurrent 9472-token requests exceed the ~20x KV budget even on 256GB MI325X
+# (c128 fit, so this was originally left at the default, but c512 crashed the
+# EngineCore mid-eval). Cap the eval to a safe in-flight count; only run_eval is
+# affected (throughput jobs use CONC directly). Matches the MI300X script.
+export EVAL_CONCURRENT_REQUESTS=8
+
 SERVER_LOG=/workspace/server.log
 
 if [ "${EVAL_ONLY}" = "true" ]; then
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index 2986d7d60..c443dc5fb 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -144,7 +144,9 @@ dsv4-fp8-mi325x-vllm:
     - isl: 8192
       osl: 1024
       search-space:
-      - { tp: 8, conc-start: 4, conc-end: 512 }
+      # 8k1k KV fits only ~20x concurrency (9472-token requests), so conc512 is
+      # ~25x oversubscribed -> request timeouts. conc256 is proven green; cap here.
+      - { tp: 8, conc-start: 4, conc-end: 256 }
 
 # MTP variant of dsv4-fp8-mi325x-vllm. Mirrors the base recipe and adds
 # DeepSeek-V4 built-in MTP via --speculative-config (num_speculative_tokens=2),
@@ -167,7 +169,9 @@ dsv4-fp8-mi325x-vllm-mtp:
     - isl: 8192
       osl: 1024
       search-space:
-      - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp }
+      # 8k1k MTP at conc512 failed (55.8% request failures, KV ~20x oversubscribed);
+      # conc256 passed on both MI300X/MI325X. Cap 8k1k MTP here (1k1k holds at 512).
+      - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp }
 
 dsr1-fp8-mi355x-sglang:
   image: lmsysorg/sglang:v0.5.12-rocm700-mi35x

From ec9b9af37d7662bb2124592e26627f145b0bb060 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 06:46:06 +0800
Subject: [PATCH 11/15] fix(dsv4): lower MI325X MTP 8k1k conc cap 256->128

MTP 8k1k at conc256 is borderline (MI300X 19.2% req failures, flaked
pass->fail across runs) because the MTP draft model shrinks KV below normal's.
conc128 passed cleanly on the memory-tightest SKU. Normal 8k1k holds at 256
(passed twice), 1k1k holds at 512.
---
 configs/amd-master.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index c443dc5fb..5a60272a7 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -169,9 +169,11 @@ dsv4-fp8-mi325x-vllm-mtp:
     - isl: 8192
       osl: 1024
       search-space:
-      # 8k1k MTP at conc512 failed (55.8% request failures, KV ~20x oversubscribed);
-      # conc256 passed on both MI300X/MI325X. Cap 8k1k MTP here (1k1k holds at 512).
-      - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp }
+      # 8k1k MTP KV is tighter than normal (draft model): conc256 is borderline
+      # (MI300X 19.2% req failures, flaked pass->fail across runs) and conc512
+      # is 55.8%. conc128 passed cleanly on the memory-tightest SKU (MI300X);
+      # cap 8k1k MTP at 128 (normal holds 256, 1k1k holds 512).
+      - { tp: 8, conc-start: 4, conc-end: 128, spec-decoding: mtp }
 
 dsr1-fp8-mi355x-sglang:
   image: lmsysorg/sglang:v0.5.12-rocm700-mi35x

From 4937d2cbbefe8f99345a2945aafca47024d95cac Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 10:09:22 +0800
Subject: [PATCH 12/15] fix(dsv4): route MI325X dsv4 to cluster:mi325x-tw
 (mi325x-amds down)

The mi325x-amds runner pool is down (canary never ran, sweep never spawned).
Switch both dsv4-fp8-mi325x-vllm and -vllm-mtp to the mi325x-tw cluster.
---
 configs/amd-master.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index 5a60272a7..56ff8da58 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -131,7 +131,8 @@ dsv4-fp8-mi325x-vllm:
   image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
-  runner: mi325x
+  # mi325x (mi325x-amds pool) runners are down; use the mi325x-tw cluster.
+  runner: cluster:mi325x-tw
   precision: fp8
   framework: vllm
   multinode: false
@@ -156,7 +157,8 @@ dsv4-fp8-mi325x-vllm-mtp:
   image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
-  runner: mi325x
+  # mi325x (mi325x-amds pool) runners are down; use the mi325x-tw cluster.
+  runner: cluster:mi325x-tw
   precision: fp8
   framework: vllm
   multinode: false

From b1b98d9669cba2f0cba1b415145ee67f35782334 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 10:20:14 +0800
Subject: [PATCH 13/15] feat(runners): add launch_mi325x-tw.sh for the
 mi325x-tw cluster
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

dsv4 MI325X was routed to cluster:mi325x-tw (mi325x-amds down), but the
per-runner launch script was missing (canary failed with 'launch_mi325x-tw.sh:
No such file'). Copy the amds launcher — same MI325X hardware + _mi325x.sh
benchmark; assumes the same /raid + compute-partition layout.
---
 runners/launch_mi325x-tw.sh | 53 +++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 runners/launch_mi325x-tw.sh

diff --git a/runners/launch_mi325x-tw.sh b/runners/launch_mi325x-tw.sh
new file mode 100644
index 000000000..a2cad5d49
--- /dev/null
+++ b/runners/launch_mi325x-tw.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export HF_HUB_CACHE_MOUNT="/raid/hf-hub-cache/"
+
+PARTITION="compute"
+SQUASH_FILE="/raid/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
+LOCK_FILE="${SQUASH_FILE}.lock"
+
+# Route spec-decoding=mtp configs to the _mtp benchmark script (parity with
+# the h200 launchers, which have carried SPEC_SUFFIX since #392).
+SPEC_SUFFIX=$([[ "${SPEC_DECODING:-}" == "mtp" ]] && printf '_mtp' || printf '')
+
+export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"
+
+set -x
+
+JOB_ID=$(set +o pipefail; salloc --partition=$PARTITION --gres=gpu:$GPU_COUNT --cpus-per-task=256 --time=480 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+')
+
+if [ -z "$JOB_ID" ]; then
+    echo "ERROR: salloc failed to allocate a job" >&2
+    exit 1
+fi
+
+export PORT=$(( 40000 + (JOB_ID % 10000) ))
+export XDG_CACHE_HOME="/tmp/xdg-cache-$JOB_ID"
+export TRITON_CACHE_DIR="/tmp/triton-cache-$JOB_ID"
+
+trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT
+
+# Use flock to serialize concurrent imports to the same squash file
+srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c "
+    set -euo pipefail
+    exec 9>\"$LOCK_FILE\"
+    flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE' >&2; exit 1; }
+    if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then
+        echo 'Squash file already exists and is valid, skipping import'
+    else
+        rm -f \"$SQUASH_FILE\"
+        enroot import -o \"$SQUASH_FILE\" docker://$IMAGE
+    fi
+"
+srun --jobid="$JOB_ID" \
+--container-image="$SQUASH_FILE" \
+--container-mounts="$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,/dev/kfd:/dev/kfd,/dev/dri:/dev/dri" \
+--container-mount-home \
+--container-writable \
+--container-remap-root \
+--container-workdir=/workspace/ \
+--no-container-entrypoint --export=ALL \
+bash benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_mi325x${SPEC_SUFFIX}.sh
+
+scancel $JOB_ID

From 0a64423f416bd0d4324f2f2f504e5a2c4250fb9b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 10:36:51 +0800
Subject: [PATCH 14/15] fix(runners): make launch_mi325x-tw.sh a docker/ROCm
 launcher (no SLURM)

The mi325x-tw runners execute on the GPU node itself with docker + ROCm and no
SLURM (prev attempt failed: 'salloc: command not found'). Run the container
directly like launch_h100-cr.sh but AMD (--device=/dev/kfd,/dev/dri,
--group-add video/render) on the node's 8 GPUs; runs the same _mi325x.sh
benchmark. First run will pull the ROCm image and hf-download the model.
---
 runners/launch_mi325x-tw.sh | 69 ++++++++++++++-----------------------
 1 file changed, 25 insertions(+), 44 deletions(-)

diff --git a/runners/launch_mi325x-tw.sh b/runners/launch_mi325x-tw.sh
index a2cad5d49..9b7f777a9 100644
--- a/runners/launch_mi325x-tw.sh
+++ b/runners/launch_mi325x-tw.sh
@@ -1,53 +1,34 @@
-#!/usr/bin/env bash
+#!/usr/bin/bash
 set -euo pipefail
 
-export HF_HUB_CACHE_MOUNT="/raid/hf-hub-cache/"
+# mi325x-tw is a NON-SLURM cluster: the runner executes on the GPU node itself
+# (docker + ROCm, no salloc/srun). So run the container directly on the node,
+# like launch_h100-cr.sh but AMD/ROCm (--device=/dev/kfd,/dev/dri instead of
+# --runtime=nvidia --gpus). dsv4 MI325X is single-node (TP8), so one node's
+# 8 GPUs suffice. Runs the same _mi325x.sh benchmark as the amds launcher.
 
-PARTITION="compute"
-SQUASH_FILE="/raid/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
-LOCK_FILE="${SQUASH_FILE}.lock"
+HF_HUB_CACHE_MOUNT="${HF_HUB_CACHE_MOUNT:-/home/gharunner/hf_hub_cache/}"
+mkdir -p "$HF_HUB_CACHE_MOUNT"
+export HF_HUB_CACHE="${HF_HUB_CACHE:-/hf_hub_cache}"
+PORT=8888
+server_name="bmk-server-${RUNNER_NAME:-mi325x-tw}"
 
-# Route spec-decoding=mtp configs to the _mtp benchmark script (parity with
-# the h200 launchers, which have carried SPEC_SUFFIX since #392).
+# Route spec-decoding=mtp configs to the _mtp benchmark script.
 SPEC_SUFFIX=$([[ "${SPEC_DECODING:-}" == "mtp" ]] && printf '_mtp' || printf '')
 
 export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"
 
 set -x
-
-JOB_ID=$(set +o pipefail; salloc --partition=$PARTITION --gres=gpu:$GPU_COUNT --cpus-per-task=256 --time=480 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+')
-
-if [ -z "$JOB_ID" ]; then
-    echo "ERROR: salloc failed to allocate a job" >&2
-    exit 1
-fi
-
-export PORT=$(( 40000 + (JOB_ID % 10000) ))
-export XDG_CACHE_HOME="/tmp/xdg-cache-$JOB_ID"
-export TRITON_CACHE_DIR="/tmp/triton-cache-$JOB_ID"
-
-trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT
-
-# Use flock to serialize concurrent imports to the same squash file
-srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c "
-    set -euo pipefail
-    exec 9>\"$LOCK_FILE\"
-    flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE' >&2; exit 1; }
-    if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then
-        echo 'Squash file already exists and is valid, skipping import'
-    else
-        rm -f \"$SQUASH_FILE\"
-        enroot import -o \"$SQUASH_FILE\" docker://$IMAGE
-    fi
-"
-srun --jobid="$JOB_ID" \
---container-image="$SQUASH_FILE" \
---container-mounts="$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,/dev/kfd:/dev/kfd,/dev/dri:/dev/dri" \
---container-mount-home \
---container-writable \
---container-remap-root \
---container-workdir=/workspace/ \
---no-container-entrypoint --export=ALL \
-bash benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_mi325x${SPEC_SUFFIX}.sh
-
-scancel $JOB_ID
+docker rm -f "$server_name" >/dev/null 2>&1 || true
+docker run --rm --network=host --name="$server_name" \
+--device=/dev/kfd --device=/dev/dri --group-add video --group-add render \
+--ipc=host --privileged --shm-size=32g --ulimit memlock=-1 --ulimit stack=67108864 \
+--security-opt seccomp=unconfined --cap-add=SYS_PTRACE \
+-v "$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE" \
+-v "$GITHUB_WORKSPACE:/workspace/" -w /workspace/ \
+-e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e PP_SIZE -e DCP_SIZE -e PCP_SIZE -e GPU_COUNT -e CONC -e MAX_MODEL_LEN -e ISL -e OSL -e RUN_EVAL -e EVAL_ONLY -e RUNNER_TYPE -e RESULT_FILENAME -e RANDOM_RANGE_RATIO -e PORT="$PORT" \
+-e PROFILE -e SGLANG_TORCH_PROFILER_DIR -e VLLM_TORCH_PROFILER_DIR -e VLLM_RPC_TIMEOUT \
+-e PYTHONPYCACHEPREFIX=/tmp/pycache/ -e CUDA_DEVICE_ORDER=PCI_BUS_ID \
+--entrypoint=/bin/bash \
+"$IMAGE" \
+benchmarks/single_node/${SCENARIO_SUBDIR}"${EXP_NAME%%_*}_${PRECISION}_mi325x${SPEC_SUFFIX}.sh"

From 0be916dfa1e53f6027d9b7da561649113a05d0ed Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 11:04:05 +0800
Subject: [PATCH 15/15] fix(runners): pass DP_ATTENTION + friends to mi325x-tw
 docker container

The amds enroot launcher used --export=ALL; the docker launcher passes an
explicit -e list that omitted DP_ATTENTION (required by _mi325x.sh's
check_env_vars) plus EP_SIZE/DP_SIZE/EVAL_MAX_MODEL_LEN/SPEC_DECODING/
NUM_SPEC_TOKENS. Canary failed 'required environment variables not set: DP_ATTENTION'.
---
 runners/launch_mi325x-tw.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runners/launch_mi325x-tw.sh b/runners/launch_mi325x-tw.sh
index 9b7f777a9..0ed4be196 100644
--- a/runners/launch_mi325x-tw.sh
+++ b/runners/launch_mi325x-tw.sh
@@ -27,6 +27,7 @@ docker run --rm --network=host --name="$server_name" \
 -v "$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE" \
 -v "$GITHUB_WORKSPACE:/workspace/" -w /workspace/ \
 -e HF_TOKEN -e HF_HUB_CACHE -e MODEL -e TP -e PP_SIZE -e DCP_SIZE -e PCP_SIZE -e GPU_COUNT -e CONC -e MAX_MODEL_LEN -e ISL -e OSL -e RUN_EVAL -e EVAL_ONLY -e RUNNER_TYPE -e RESULT_FILENAME -e RANDOM_RANGE_RATIO -e PORT="$PORT" \
+-e DP_ATTENTION -e EP_SIZE -e DP_SIZE -e EVAL_MAX_MODEL_LEN -e SPEC_DECODING -e NUM_SPEC_TOKENS \
 -e PROFILE -e SGLANG_TORCH_PROFILER_DIR -e VLLM_TORCH_PROFILER_DIR -e VLLM_RPC_TIMEOUT \
 -e PYTHONPYCACHEPREFIX=/tmp/pycache/ -e CUDA_DEVICE_ORDER=PCI_BUS_ID \
 --entrypoint=/bin/bash \