diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen1tep16_mtp3_c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen1tep16_mtp3_c48.yaml new file mode 100644 index 000000000..9438243e2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen1tep16_mtp3_c48.yaml @@ -0,0 +1,167 @@ +# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/1ctxDEP2c128_1genTEP16c32_E2E_C32/conc32 +# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. +# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. +name: ctx1dep2_gen1tep16_mtp3_c48 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false + decode: + trust_remote_code: true + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 40 + max_num_tokens: 4096 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "48" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen4tp16_mtp3_c4.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen4tp16_mtp3_c4.yaml new file mode 100644 index 000000000..c1fda5499 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen4tp16_mtp3_c4.yaml @@ -0,0 +1,156 @@ +# Derived from config #01 for one-request-per-decode-worker minimum-TPOT measurement. +# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/1ctxDEP2c128_4genTP16c4_E2E_C16/conc16 +# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. +# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. +name: ctx1dep2_gen4tp16_mtp3_c4 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 16 + decode_workers: 4 + gpus_per_decode: 16 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false + decode: + trust_remote_code: true + tensor_parallel_size: 16 + moe_expert_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen7tep8_mtp3_c42.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen7tep8_mtp3_c42.yaml new file mode 100644 index 000000000..f32dbc95a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen7tep8_mtp3_c42.yaml @@ -0,0 +1,168 @@ +# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/1ctxDEP2c128_7genTEP8c4_E2E_C28/conc28 +# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. +# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. +name: ctx1dep2_gen7tep8_mtp3_c42 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 14 + decode_workers: 7 + gpus_per_decode: 8 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false + decode: + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 5 + max_num_tokens: 4096 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "42" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx2dep2_gen15tp4_mtp3_c90.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx2dep2_gen15tp4_mtp3_c90.yaml new file mode 100644 index 000000000..7cb1d46b1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx2dep2_gen15tp4_mtp3_c90.yaml @@ -0,0 +1,167 @@ +# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/2ctxDEP2c128_15genTP4c4_E2E_C60/conc60 +# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. +# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. +name: ctx2dep2_gen15tp4_mtp3_c90 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 15 + decode_workers: 15 + gpus_per_decode: 4 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false + decode: + trust_remote_code: true + tensor_parallel_size: 4 + moe_expert_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 5 + max_num_tokens: 4096 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "90" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen4tep8_mtp3_c192.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen4tep8_mtp3_c192.yaml new file mode 100644 index 000000000..2a088dc3b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen4tep8_mtp3_c192.yaml @@ -0,0 +1,168 @@ +# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/3ctxDEP2c128_4genTEP8c32_E2E_C128/conc128 +# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. +# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. +name: ctx3dep2_gen4tep8_mtp3_c192 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 3 + gpus_per_prefill: 2 + decode_nodes: 8 + decode_workers: 4 + gpus_per_decode: 8 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false + decode: + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 40 + max_num_tokens: 4096 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "192" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen7tep8_mtp3_c168.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen7tep8_mtp3_c168.yaml new file mode 100644 index 000000000..12b6138e6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen7tep8_mtp3_c168.yaml @@ -0,0 +1,168 @@ +# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/3ctxDEP2c128_7genTEP8c16_E2E_C112/conc112 +# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. +# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. +name: ctx3dep2_gen7tep8_mtp3_c168 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 3 + gpus_per_prefill: 2 + decode_nodes: 14 + decode_workers: 7 + gpus_per_decode: 8 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false + decode: + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 20 + max_num_tokens: 4096 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "168" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4dep2_gen1dep16_mtp3_c384.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4dep2_gen1dep16_mtp3_c384.yaml new file mode 100644 index 000000000..6a3802ef4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4dep2_gen1dep16_mtp3_c384.yaml @@ -0,0 +1,167 @@ +# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/4ctxDEP2c128_1genDEP16c256_E2E_C256/conc256 +# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. +# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. +name: ctx4dep2_gen1dep16_mtp3_c384 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 4 + gpus_per_prefill: 2 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false + decode: + trust_remote_code: true + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 20 + max_num_tokens: 4096 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "384" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx6dep2_gen5tep8_mtp3_c480.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx6dep2_gen5tep8_mtp3_c480.yaml new file mode 100644 index 000000000..14ff198a0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx6dep2_gen5tep8_mtp3_c480.yaml @@ -0,0 +1,168 @@ +# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/6ctxDEP2c128_5genTEP8c64_E2E_C320/conc320 +# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. +# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. +name: ctx6dep2_gen5tep8_mtp3_c480 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + prefill_workers: 6 + gpus_per_prefill: 2 + decode_nodes: 10 + decode_workers: 5 + gpus_per_decode: 8 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false + decode: + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 80 + max_num_tokens: 4096 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "480" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx9dep2_gen1dep16_mtp3_c1536.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx9dep2_gen1dep16_mtp3_c1536.yaml new file mode 100644 index 000000000..6e0195f2d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx9dep2_gen1dep16_mtp3_c1536.yaml @@ -0,0 +1,167 @@ +# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/9ctxDEP2c2048_1genDEP16c1024_E2E_C1024/conc1024 +# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. +# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. +name: ctx9dep2_gen1dep16_mtp3_c1536 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 5 + prefill_workers: 9 + gpus_per_prefill: 2 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + enable_chunked_prefill: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false + decode: + trust_remote_code: true + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 80 + max_num_tokens: 4096 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. + kv_transfer_timeout_ms: 600000 + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + speculative_config: + decoding_type: MTP + max_draft_len: 3 + use_mtp_vanilla: false +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1536" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 73327351e..1b5cbe0c7 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11486,6 +11486,157 @@ qwen3.5-fp4-gb300-dynamo-sglang: ep: 16 dp-attn: true +qwen3.5-fp4-gb300-dynamo-trt-mtp: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: gb300 + precision: fp4 + framework: dynamo-trt + multinode: true + disagg: true + kv-p2p-transfer: nixl + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 1P1D: 1 prefill (TP2/EP2/dp-attn), 1 decode (TP16/EP16/no-dp-attn), conc=48 + - spec-decoding: "mtp" + conc-list: [48] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen1tep16_mtp3_c48.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: false + # 1P4D: 1 prefill (TP2/EP2/dp-attn), 4 decode (TP16/EP1/no-dp-attn), conc=4 + - spec-decoding: "mtp" + conc-list: [4] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen4tp16_mtp3_c4.yaml" + decode: + num-worker: 4 + tp: 16 + ep: 1 + dp-attn: false + # 1P7D: 1 prefill (TP2/EP2/dp-attn), 7 decode (TP8/EP8/no-dp-attn), conc=42 + - spec-decoding: "mtp" + conc-list: [42] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen7tep8_mtp3_c42.yaml" + decode: + num-worker: 7 + tp: 8 + ep: 8 + dp-attn: false + # 2P15D: 2 prefill (TP2/EP2/dp-attn), 15 decode (TP4/EP1/no-dp-attn), conc=90 + - spec-decoding: "mtp" + conc-list: [90] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx2dep2_gen15tp4_mtp3_c90.yaml" + decode: + num-worker: 15 + tp: 4 + ep: 1 + dp-attn: false + # 3P4D: 3 prefill (TP2/EP2/dp-attn), 4 decode (TP8/EP8/no-dp-attn), conc=192 + - spec-decoding: "mtp" + conc-list: [192] + prefill: + num-worker: 3 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen4tep8_mtp3_c192.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + # 3P7D: 3 prefill (TP2/EP2/dp-attn), 7 decode (TP8/EP8/no-dp-attn), conc=168 + - spec-decoding: "mtp" + conc-list: [168] + prefill: + num-worker: 3 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen7tep8_mtp3_c168.yaml" + decode: + num-worker: 7 + tp: 8 + ep: 8 + dp-attn: false + # 4P1D: 4 prefill (TP2/EP2/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=384 + - spec-decoding: "mtp" + conc-list: [384] + prefill: + num-worker: 4 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4dep2_gen1dep16_mtp3_c384.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 6P5D: 6 prefill (TP2/EP2/dp-attn), 5 decode (TP8/EP8/no-dp-attn), conc=480 + - spec-decoding: "mtp" + conc-list: [480] + prefill: + num-worker: 6 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx6dep2_gen5tep8_mtp3_c480.yaml" + decode: + num-worker: 5 + tp: 8 + ep: 8 + dp-attn: false + # 9P1D: 9 prefill (TP2/EP2/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=1536 + - spec-decoding: "mtp" + conc-list: [1536] + prefill: + num-worker: 9 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx9dep2_gen1dep16_mtp3_c1536.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + glm5-fp4-gb300-dynamo-sglang: image: lmsysorg/sglang:v0.5.11-cu130 model: nvidia/GLM-5.1-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d77c35f1f..d09ef9ec4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4781,3 +4781,12 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt-mtp + description: + - "Add Qwen3.5-397B-A17B-NVFP4 GB300 disaggregated TensorRT-LLM+Dynamo MTP (3-token) benchmarks for 8k1k." + - "Image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 with Dynamo installed at runtime (version 1.3.0.dev20260708, TCP request plane)." + - "9 MTP configs: 1P1D DEP2+1D TEP16 (conc 48), 1P4D DEP2+4D TP16 (conc 4), 1P7D DEP2+7D TEP8 (conc 42), 2P15D DEP2+15D TP4 (conc 90), 3P4D DEP2+4D TEP8 (conc 192), 3P7D DEP2+7D TEP8 (conc 168), 4P1D DEP2+1D DEP16 (conc 384), 6P5D DEP2+5D TEP8 (conc 480), 9P1D DEP2+1D DEP16 (conc 1536)." + - "Runner: launch_gb300-nv.sh updated to clone NVIDIA/srt-slurm@sa-submission-q2-2026 and overlay local recipes from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/ for the dynamo-trt+qwen3.5+fp4 path; --no-preflight added for model on compute-node /scratch." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2218 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 1381bd5db..494b6500e 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -229,6 +229,13 @@ elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout sa-submission-q2-2026 +elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.29 + mkdir -p recipes/trtllm/qwen3.5/gb300-fp4/disagg + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg" \ + recipes/trtllm/qwen3.5/gb300-fp4/disagg else git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" @@ -337,7 +344,7 @@ SRTCTL_APPLY_ARGS=( -f "$CONFIG_FILE" --tags "gb300,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" ) -if [[ "$IS_AGENTIC" == "1" || "$MODEL_PREFIX" == "glm5.1" || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" ) ]]; then +if [[ "$IS_AGENTIC" == "1" || "$MODEL_PREFIX" == "glm5.1" || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" ) || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-trt" ) ]]; then SRTCTL_APPLY_ARGS+=(--no-preflight) fi if [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then