diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p10d-dep4-tp4-atune.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p10d-dep4-tp4-atune.yaml new file mode 100644 index 0000000000..56e1d33493 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p10d-dep4-tp4-atune.yaml @@ -0,0 +1,89 @@ +name: kimi-k2.6-vllm-disagg-b300-1p10d-dep4-tp4-atune +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.22.0 + precision: fp4 +dynamo: + version: 1.2.0.dev20260529 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 5 + prefill_workers: 1 + decode_workers: 10 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + set_cuda_visible_devices: true + prefill_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "900" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + decode_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 32768 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "1x4x16x64x128x256x512x1024x2048x3072x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml new file mode 100644 index 0000000000..88b96e38cb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml @@ -0,0 +1,92 @@ +name: kimi-k2.6-vllm-disagg-b300-1p1d-dep4-dep8 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.22.0 + precision: fp4 +dynamo: + version: 1.2.0.dev20260529 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + set_cuda_visible_devices: true + prefill_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "900" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + decode_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 32768 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "1x4x16x64x128x256x512x1024x2048x3072x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p8d-dep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p8d-dep4-tp4.yaml new file mode 100644 index 0000000000..5be9475473 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p8d-dep4-tp4.yaml @@ -0,0 +1,89 @@ +name: kimi-k2.6-vllm-disagg-b300-1p8d-dep4-tp4 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.22.0 + precision: fp4 +dynamo: + version: 1.2.0.dev20260529 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 8 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + set_cuda_visible_devices: true + prefill_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "900" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + decode_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 32768 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "1x4x16x64x128x256x512x1024x2048x3072x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-2p5d-dep4-tp8.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-2p5d-dep4-tp8.yaml new file mode 100644 index 0000000000..8f2efe52eb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-2p5d-dep4-tp8.yaml @@ -0,0 +1,90 @@ +name: kimi-k2.6-vllm-disagg-b300-2p5d-dep4-tp8 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.22.0 + precision: fp4 +dynamo: + version: 1.2.0.dev20260529 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 5 + prefill_workers: 2 + decode_workers: 5 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + set_cuda_visible_devices: true + prefill_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "900" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + decode_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "0" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 32768 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "1x4x16x64x128x256x512x1024x2048x3072x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-3p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-3p1d-dep4-dep8.yaml new file mode 100644 index 0000000000..b77b6ca08b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-3p1d-dep4-dep8.yaml @@ -0,0 +1,92 @@ +name: kimi-k2.6-vllm-disagg-b300-3p1d-dep4-dep8 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.22.0 + precision: fp4 +dynamo: + version: 1.2.0.dev20260529 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + set_cuda_visible_devices: true + prefill_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "900" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + decode_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 32768 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "1x4x16x64x128x256x512x1024x2048x3072x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-4p1d-dep4-dep8-atune.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-4p1d-dep4-dep8-atune.yaml new file mode 100644 index 0000000000..fc01cbcfee --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-4p1d-dep4-dep8-atune.yaml @@ -0,0 +1,92 @@ +name: kimi-k2.6-vllm-disagg-b300-4p1d-dep4-dep8-atune +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.22.0 + precision: fp4 +dynamo: + version: 1.2.0.dev20260529 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 4 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + set_cuda_visible_devices: true + prefill_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "900" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + decode_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 32768 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "4x16x64x128x256x512x1024x2048x3072x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml new file mode 100644 index 0000000000..5ade70c7ac --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml @@ -0,0 +1,92 @@ +name: kimi-k2.6-vllm-disagg-b300-4p1d-dep4-dep8 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.22.0 + precision: fp4 +dynamo: + version: 1.2.0.dev20260529 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b300 + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 4 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + set_cuda_visible_devices: true + prefill_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "900" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + decode_environment: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_WATCHDOG_TIMEOUT: "1800" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "1800" + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 32768 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: "1x4x16x64x128x256x512x1024x2048x3072x4096" + req_rate: "inf" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f90c510186..b32851f060 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -14171,6 +14171,113 @@ kimik2.5-fp4-gb300-dynamo-vllm: tp: 1 ep: 24 dp-attn: true +kimik2.6-fp4-b300-dynamo-vllm: + image: vllm/vllm-openai:v0.22.0 + model: nvidia/Kimi-K2.6-NVFP4 + model-prefix: kimik2.6 + runner: b300 + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260529" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [1, 4, 16, 64, 128, 256, 512, 1024, 2048, 3072, 4096] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - conc-list: [1, 4, 16, 64, 128, 256, 512, 1024, 2048, 3072, 4096] + prefill: + num-worker: 3 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-3p1d-dep4-dep8.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - conc-list: [1, 4, 16, 64, 128, 256, 512, 1024, 2048, 3072, 4096] + prefill: + num-worker: 4 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - conc-list: [4, 16, 64, 128, 256, 512, 1024, 2048, 3072, 4096] + prefill: + num-worker: 4 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-4p1d-dep4-dep8-atune.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + - conc-list: [1, 4, 16, 64, 128, 256, 512, 1024, 2048, 3072, 4096] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p8d-dep4-tp4.yaml" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [1, 4, 16, 64, 128, 256, 512, 1024, 2048, 3072, 4096] + prefill: + num-worker: 1 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-1p10d-dep4-tp4-atune.yaml" + decode: + num-worker: 10 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [1, 4, 16, 64, 128, 256, 512, 1024, 2048, 3072, 4096] + prefill: + num-worker: 2 + tp: 1 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k2.6-fp4/8k1k/disagg-b300-2p5d-dep4-tp8.yaml" + decode: + num-worker: 5 + tp: 8 + ep: 1 + dp-attn: false minimaxm3-fp8-h100-vllm-agentic: image: vllm/vllm-openai:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 model: MiniMaxAI/MiniMax-M3-MXFP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 57d510dd15..805165b7a3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4750,3 +4750,11 @@ - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 + +- config-keys: + - kimik2.6-fp4-b300-dynamo-vllm + description: + - "Add Kimi-K2.6 NVFP4 B300 disaggregated multinode vLLM benchmarks via Dynamo" + - "Image: vllm/vllm-openai:v0.22.0" + - "7 topologies at 8k/1k: 1p1d/3p1d/4p1d dep4-dep8 (4p1d also as FlashInfer-autotune variant at conc 4-4096), 1p8d dep4-tp4, 1p10d dep4-tp4 (FlashInfer autotune), 2p5d dep4-tp8; conc 1-4096; recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4/8k1k/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2181 diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index 6810ee5d85..2fc67e3d02 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -53,8 +53,13 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" && $FRAMEWORK == "dy elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" && $FRAMEWORK == "dynamo-vllm" ]]; then export MODEL_PATH="/data/models/MiniMax-M3-MXFP8" export SRT_SLURM_MODEL_PREFIX="MiniMaxAI/MiniMax-M3-MXFP8" +elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-vllm" ]]; then + # Node-local NVMe path; only exists on compute nodes, so srtctl apply + # runs with --no-preflight for this model (see SRTCTL_APPLY_ARGS below). + export MODEL_PATH="/scratch/models/Kimi-K2.6-NVFP4" + export SRT_SLURM_MODEL_PREFIX="kimi-k2.6-nvfp4" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4 with dynamo-vllm, minimaxm2.5-fp4 with dynamo-vllm, minimaxm2.5-fp8 with dynamo-vllm, minimaxm3-fp4 with dynamo-vllm, minimaxm3-fp8 with dynamo-vllm, kimik2.6-fp4 with dynamo-vllm" exit 1 fi @@ -92,6 +97,13 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && ( $PRECIS "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/$SRTCTL_SETUP_SCRIPT" \ "configs/$SRTCTL_SETUP_SCRIPT" fi +elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" || exit 1 + # Pin srt-slurm main so upstream movement does not change behavior + git checkout b959ff7d78c396f3f47bb88a115d8a492326aafe + mkdir -p recipes/vllm/kimi-k2.6-fp4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6-fp4" recipes/vllm/kimi-k2.6-fp4 else git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 @@ -184,6 +196,13 @@ SRTCTL_APPLY_ARGS=( if [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then SRTCTL_APPLY_ARGS+=(--setup-script "$SRTCTL_SETUP_SCRIPT") fi +if [[ "$MODEL_PREFIX" == "kimik2.6" ]]; then + # MODEL_PATH lives on compute-node-local NVMe (/scratch/models), which is + # not mounted on the runner invoking srtctl, so skip the Python-level + # model-path preflight. vLLM still fails loudly at runtime if the path is + # genuinely missing on the compute node. + SRTCTL_APPLY_ARGS+=(--no-preflight) +fi SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) echo "$SRTCTL_OUTPUT"