From 2cd64c38233d8cdba13b2c941b6e3eedb74ba082 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 15 Jul 2026 14:22:21 -0500 Subject: [PATCH 1/6] =?UTF-8?q?Revert=20"Revert=20"Add=20Kimi=20K2.6=20NVF?= =?UTF-8?q?P4=20B300=20EAGLE3=20AgentX=20benchmark=20/=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=20Kimi=20K2.6=20NVFP4=20B300=20EAGLE3=20AgentX=20?= =?UTF-8?q?=E5=9F=BA=E5=87=86=E6=B5=8B=E8=AF=95=20(#2158)"=20(#2227)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit bae2753a3671da5d1ed97baa2cc3b7d347de41df. --- .../agentic/kimik2.5_fp4_b300_mtp.sh | 125 ++++++++++++++++++ configs/nvidia-master.yaml | 18 +++ perf-changelog.yaml | 7 + runners/launch_b300-nv.sh | 1 + utils/process_changelog.py | 2 +- 5 files changed, 152 insertions(+), 1 deletion(-) create mode 100755 benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh new file mode 100755 index 0000000000..452fd244ae --- /dev/null +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION + + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +DRAFT_MODEL="lightseekorg/kimi-k2.6-eagle3-mla" + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi + DRAFT_MODEL_PATH="/data/models/${DRAFT_MODEL##*/}" + if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" + hf download "$DRAFT_MODEL" + DRAFT_MODEL_PATH="$DRAFT_MODEL" +fi +nvidia-smi + +resolve_trace_source +install_agentic_deps + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" + +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +DCP_SIZE="${DCP_SIZE:-1}" +DCP_ARGS=() +if [[ "$DCP_SIZE" -gt 1 ]]; then + DCP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a) + NUM_SPEC_TOKENS=3 + SYNTHETIC_ACCEPT_LEN=2.88 + SPEC_ARGS=(--speculative-config "{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL_PATH\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":$SYNTHETIC_ACCEPT_LEN,\"attention_backend\":\"TOKENSPEED_MLA\"}") + ATTN_CONFIG='{"mla_prefill_backend":"TOKENSPEED_MLA"}' + COMPILATION_CONFIG='{"pass_config":{"fuse_allreduce_rms":false}}' +else + NUM_SPEC_TOKENS=4 + SYNTHETIC_ACCEPT_LEN=3.24 + SPEC_ARGS=(--speculative-config "{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL_PATH\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":$SYNTHETIC_ACCEPT_LEN}") + ATTN_CONFIG='{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + COMPILATION_CONFIG='{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' +fi +ATTN_BACKEND_ARGS=(--attention-backend TOKENSPEED_MLA) + +OFFLOAD_ARGS=() + +if agentic_kv_offload_enabled; then + case "$KV_OFFLOAD_BACKEND" in + native) + export VLLM_USE_SIMPLE_KV_OFFLOAD=1 + CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) + OFFLOAD_ARGS=( + --disable-hybrid-kv-cache-manager + --kv-transfer-config + "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":$CPU_OFFLOAD_BYTES,\"lazy_offload\":false}}" + ) + ;; + *) echo "Error: unsupported KV_OFFLOAD_BACKEND value '$KV_OFFLOAD_BACKEND' with EAGLE3 (expected: native)" >&2; exit 1 ;; + esac +fi + + +echo "Starting vllm server..." +export PYTHONNOUSERSITE=1 + +export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --kv-cache-dtype fp8 + --trust-remote-code + --block-size 64 + --language-model-only + --gpu-memory-utilization 0.90 + --max-num-seqs "$CONC" + "${ATTN_BACKEND_ARGS[@]}" + --attention-config "$ATTN_CONFIG" + --compilation-config "$COMPILATION_CONFIG" + --max-cudagraph-capture-size 2048 + --max-num-batched-tokens 16384 + --stream-interval 10 + --enable-prefix-caching + --tensor-parallel-size "$TP" + "${SPEC_ARGS[@]}" + "${DCP_ARGS[@]}" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +build_replay_cmd "$RESULT_DIR" + +run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 73327351e1..e35f9a2223 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2989,6 +2989,24 @@ kimik2.5-fp4-b300-vllm-agentic: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } +kimik2.5-fp4-b300-vllm-agentic-mtp: + image: vllm/vllm-openai:nightly-94c0ef300180f8fd1071d9cbe7270a8348155f94 + model: Kimi-K2.6-NVFP4 + model-prefix: kimik2.5 + runner: cluster:b300-nv + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 8] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [8, 16, 32] } + - { tp: 4, ep: 1, spec-decoding: mtp, dcp-size: 4, kv-offloading: none, conc-list: [32, 64, 80, 96, 112, 128] } + - { tp: 4, ep: 1, spec-decoding: mtp, dcp-size: 4, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [64, 80, 96, 112, 128, 144, 160] } + dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 model: deepseek-ai/DeepSeek-R1-0528 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cd0320fa99..4f0e4e4f12 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4782,6 +4782,13 @@ - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 +- config-keys: + - kimik2.5-fp4-b300-vllm-agentic-mtp + description: + - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." + - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2158 + - config-keys: - kimik2.5-fp4-b300-vllm-agentic-mtp description: diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 6810ee5d85..2c0f93fc8b 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -352,6 +352,7 @@ else Kimi-K2.5 Kimi-K2.5-NVFP4 Kimi-K2.6 + Kimi-K2.6-NVFP4 MiniMax-M2.5 MiniMax-M2.5-NVFP4 MiniMax-M2.7 diff --git a/utils/process_changelog.py b/utils/process_changelog.py index 311c8c9e22..c293dc2f80 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -270,7 +270,7 @@ def main(): # Validate final results structure validated = ChangelogMatrixEntry.model_validate(final_results) - print(validated.model_dump_json(by_alias=True)) + print(validated.model_dump_json(by_alias=True, exclude_none=True)) if __name__ == "__main__": From ce4ead09673c5c85d19be5b31b13b0def74311b7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 15 Jul 2026 14:26:57 -0500 Subject: [PATCH 2/6] fix(kimik2.6-b300-mtp): use corrected EAGLE 3.1 head --- benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh | 2 +- perf-changelog.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh index 452fd244ae..85adb1c9d2 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh @@ -11,7 +11,7 @@ if [[ -n "${SLURM_JOB_ID:-}" ]]; then echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" fi -DRAFT_MODEL="lightseekorg/kimi-k2.6-eagle3-mla" +DRAFT_MODEL="lightseekorg/kimi-k2.6-eagle3.1-mla" if [[ -n "${MODEL_PATH:-}" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4f0e4e4f12..4e9428ef19 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4785,13 +4785,13 @@ - config-keys: - kimik2.5-fp4-b300-vllm-agentic-mtp description: - - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." + - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3.1-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2158 - config-keys: - kimik2.5-fp4-b300-vllm-agentic-mtp description: - - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." + - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3.1-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2222 From 38929321be864a97c13d0b5325d8c33998941ba7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 15 Jul 2026 17:19:21 -0500 Subject: [PATCH 3/6] Restore append-only changelog history --- perf-changelog.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cee5f34542..52aa49b9b2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4785,14 +4785,7 @@ - config-keys: - kimik2.5-fp4-b300-vllm-agentic-mtp description: - - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3.1-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." - - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2158 - -- config-keys: - - kimik2.5-fp4-b300-vllm-agentic-mtp - description: - - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3.1-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." + - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2222 @@ -4821,4 +4814,10 @@ - "Add GLM-5.1 FP8 GB200 disaggregated dynamo-sglang MTP config covering 8k1k, high-throughput and low-latency topologies" - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2123 - + +- config-keys: + - kimik2.5-fp4-b300-vllm-agentic-mtp + description: + - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3.1-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." + - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2228 From af29b0b266c5d6a803fd36f78f3489d50d537243 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Thu, 16 Jul 2026 00:25:46 -0400 Subject: [PATCH 4/6] fix(kimik2.6-b300-mtp): reduce GMU to 0.85 for DCP+no-offload to avoid OOM --- benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh index 85adb1c9d2..484a8f55f1 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300_mtp.sh @@ -84,6 +84,11 @@ if agentic_kv_offload_enabled; then fi +GMU=0.90 +if [[ "$DCP_SIZE" -gt 1 && "$KV_OFFLOADING" == "none" ]]; then + GMU=0.85 +fi + echo "Starting vllm server..." export PYTHONNOUSERSITE=1 @@ -98,7 +103,7 @@ VLLM_CMD=( --trust-remote-code --block-size 64 --language-model-only - --gpu-memory-utilization 0.90 + --gpu-memory-utilization "$GMU" --max-num-seqs "$CONC" "${ATTN_BACKEND_ARGS[@]}" --attention-config "$ATTN_CONFIG" From 124120507f4b8515e93d060be115520add157b08 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Thu, 16 Jul 2026 00:40:22 -0400 Subject: [PATCH 5/6] fix(perf-changelog): restore trailing whitespace --- perf-changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c8b61e7346..827b422ed2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4833,7 +4833,7 @@ - "Add GLM-5.1 FP8 GB200 disaggregated dynamo-sglang MTP config covering 8k1k, high-throughput and low-latency topologies" - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2123 - + - config-keys: - dsv4-fp8-mi325x-vllm @@ -4859,7 +4859,7 @@ description: - "Add GB300 DeepSeek-V4 dynamo-sglang agentic recipes (agg-tp4 + dep8 disagg pareto)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2157 - + - config-keys: - kimik2.5-fp4-b300-vllm-agentic-mtp description: From 38c7812956db0c08379fb06b4df3d522c190354d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 16 Jul 2026 09:24:39 -0500 Subject: [PATCH 6/6] fix(perf-changelog): restore final newline --- perf-changelog.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1dad1c7ba6..36e00e0710 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4881,4 +4881,3 @@ - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3.1-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)." - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2228 - \ No newline at end of file