diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c136.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c136.yaml new file mode 100644 index 000000000..aac4beca8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c136.yaml @@ -0,0 +1,145 @@ +name: ctx1dep2_gen17dep4_mtp0_c136 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 + +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 17 + decode_workers: 17 + gpus_per_decode: 4 +backend: + type: trtllm + prefill_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + decode_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 1227 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + decode: + trust_remote_code: true + tensor_parallel_size: 4 + moe_expert_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 8 + max_num_tokens: 16384 + max_seq_len: 2251 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "136" + req_rate: "inf" + num_prompts_mult: 16 +infra: + etcd_nats_dedicated_node: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c272.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c272.yaml new file mode 100644 index 000000000..a595fd91d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c272.yaml @@ -0,0 +1,145 @@ +name: ctx1dep2_gen17dep4_mtp0_c272 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 + +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 17 + decode_workers: 17 + gpus_per_decode: 4 +backend: + type: trtllm + prefill_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + decode_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 1227 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + decode: + trust_remote_code: true + tensor_parallel_size: 4 + moe_expert_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 2251 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "272" + req_rate: "inf" + num_prompts_mult: 16 +infra: + etcd_nats_dedicated_node: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c68.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c68.yaml new file mode 100644 index 000000000..a67a2c738 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c68.yaml @@ -0,0 +1,145 @@ +name: ctx1dep2_gen17dep4_mtp0_c68 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 + +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 17 + decode_workers: 17 + gpus_per_decode: 4 +backend: + type: trtllm + prefill_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + decode_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 1227 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + decode: + trust_remote_code: true + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 4 + max_num_tokens: 16384 + max_seq_len: 2251 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "68" + req_rate: "inf" + num_prompts_mult: 16 +infra: + etcd_nats_dedicated_node: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen1dep16_mtp0_c1280.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen1dep16_mtp0_c1280.yaml new file mode 100644 index 000000000..890647ebe --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen1dep16_mtp0_c1280.yaml @@ -0,0 +1,152 @@ +name: ctx1dep2_gen1dep16_mtp0_c1280 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 + +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + prefill_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_REQUEST_KV_CACHE_CONCURRENT: '1' + TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM: '2' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '2' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + decode_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_REQUEST_KV_CACHE_CONCURRENT: '1' + TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM: '2' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '2' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 32768 + max_seq_len: 1227 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 2048 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + decode: + trust_remote_code: true + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 96 + max_num_tokens: 16384 + max_seq_len: 2251 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 96 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 2048 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1280" + req_rate: "inf" + num_prompts_mult: 16 +infra: + etcd_nats_dedicated_node: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen4dep16_mtp0_c16.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen4dep16_mtp0_c16.yaml new file mode 100644 index 000000000..2d4863328 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen4dep16_mtp0_c16.yaml @@ -0,0 +1,145 @@ +name: ctx1dep2_gen4dep16_mtp0_c16 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 + +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 16 + decode_workers: 4 + gpus_per_decode: 16 +backend: + type: trtllm + prefill_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + decode_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 1227 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + decode: + trust_remote_code: true + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 4 + max_num_tokens: 16384 + max_seq_len: 2251 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "16" + req_rate: "inf" + num_prompts_mult: 16 +infra: + etcd_nats_dedicated_node: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen5dep8_mtp0_c640.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen5dep8_mtp0_c640.yaml new file mode 100644 index 000000000..cca7634d7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen5dep8_mtp0_c640.yaml @@ -0,0 +1,146 @@ +name: ctx1dep2_gen5dep8_mtp0_c640 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 + +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 10 + decode_workers: 5 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + decode_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 32768 + max_seq_len: 1227 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + decode: + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 16384 + max_seq_len: 2251 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + allreduce_strategy: MNNVL +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "640" + req_rate: "inf" + num_prompts_mult: 16 +infra: + etcd_nats_dedicated_node: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx2dep2_gen3dep16_mtp0_c1536.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx2dep2_gen3dep16_mtp0_c1536.yaml new file mode 100644 index 000000000..5377f8e8e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx2dep2_gen3dep16_mtp0_c1536.yaml @@ -0,0 +1,145 @@ +name: ctx2dep2_gen3dep16_mtp0_c1536 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 + +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 12 + decode_workers: 3 + gpus_per_decode: 16 +backend: + type: trtllm + prefill_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + decode_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 32768 + max_seq_len: 1227 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + decode: + trust_remote_code: true + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 32 + max_num_tokens: 16384 + max_seq_len: 2251 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1536" + req_rate: "inf" + num_prompts_mult: 16 +infra: + etcd_nats_dedicated_node: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx5dep2_gen3dep16_mtp0_c7680.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx5dep2_gen3dep16_mtp0_c7680.yaml new file mode 100644 index 000000000..f499fd32c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx5dep2_gen3dep16_mtp0_c7680.yaml @@ -0,0 +1,158 @@ +name: ctx5dep2_gen3dep16_mtp0_c7680 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 + +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp + +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + prefill_workers: 5 + gpus_per_prefill: 2 + decode_nodes: 12 + decode_workers: 3 + gpus_per_decode: 16 +backend: + type: trtllm + prefill_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_REQUEST_KV_CACHE_CONCURRENT: '1' + TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM: '2' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '2' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + DYN_TCP_WORKER_POOL_SIZE: "17126" + DYN_TCP_WORK_QUEUE_SIZE: "68504" + decode_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_REQUEST_KV_CACHE_CONCURRENT: '1' + TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM: '2' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '2' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + DYN_TCP_WORKER_POOL_SIZE: "17126" + DYN_TCP_WORK_QUEUE_SIZE: "68504" + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 32768 + max_seq_len: 1227 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 2048 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + decode: + trust_remote_code: true + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 192 + max_num_tokens: 16384 + max_seq_len: 2251 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 192 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 2048 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "7680" + req_rate: "inf" + num_prompts_mult: 16 + reuse_http_connections: true +infra: + etcd_nats_dedicated_node: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx5dep2_gen7dep4_mtp0_c17920.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx5dep2_gen7dep4_mtp0_c17920.yaml new file mode 100644 index 000000000..e4f8fbef1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx5dep2_gen7dep4_mtp0_c17920.yaml @@ -0,0 +1,159 @@ +name: ctx5dep2_gen7dep4_mtp0_c17920 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 + +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + prefill_workers: 5 + gpus_per_prefill: 2 + decode_nodes: 7 + decode_workers: 7 + gpus_per_decode: 4 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_REQUEST_KV_CACHE_CONCURRENT: '1' + TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM: '2' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '2' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + DYN_TCP_WORKER_POOL_SIZE: "40000" + DYN_TCP_WORK_QUEUE_SIZE: "160000" + decode_environment: + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + TRTLLM_REQUEST_KV_CACHE_CONCURRENT: '1' + TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM: '2' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '2' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + PYTHONNOUSERSITE: '1' + DYN_TCP_WORKER_POOL_SIZE: "40000" + DYN_TCP_WORK_QUEUE_SIZE: "160000" + + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 32768 + max_seq_len: 1227 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 2048 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + decode: + trust_remote_code: true + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 768 + max_num_tokens: 16384 + max_seq_len: 2251 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + - 768 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 2048 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "17920" + req_rate: "inf" + num_prompts_mult: 16 + reuse_http_connections: true +infra: + etcd_nats_dedicated_node: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10dep2_gen3dep8_mtp0_c4608.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10dep2_gen3dep8_mtp0_c4608.yaml new file mode 100644 index 000000000..917db683e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10dep2_gen3dep8_mtp0_c4608.yaml @@ -0,0 +1,154 @@ +# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/10ctxDEP2c512_3genDEP8c1024_E2E_C3072.yaml +# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/10ctxDEP2c512_3genDEP8c1024_E2E_C3072__v1_sysc15360_genbs160_chunk1_ctxgdn1_gengdn0_buf8192/conc15360 +name: ctx10dep2_gen3dep8_mtp0_c4608 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 5 + prefill_workers: 10 + gpus_per_prefill: 2 + decode_nodes: 6 + decode_workers: 3 + gpus_per_decode: 8 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 256 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + enable_chunked_prefill: true + decode: + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 160 + max_num_tokens: 16384 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4608" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen13tp4_mtp0_c78.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen13tp4_mtp0_c78.yaml new file mode 100644 index 000000000..3264cdcf6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen13tp4_mtp0_c78.yaml @@ -0,0 +1,154 @@ +# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/1ctxDEP2c128_13genTP4c4_E2E_C52.yaml +# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/1ctxDEP2c128_13genTP4c4_E2E_C52__v1_sysc260_genbs5_chunk1_ctxgdn1_gengdn0_buf8192/conc260 +name: ctx1dep2_gen13tp4_mtp0_c78 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 13 + decode_workers: 13 + gpus_per_decode: 4 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + enable_chunked_prefill: true + decode: + trust_remote_code: true + tensor_parallel_size: 4 + moe_expert_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 5 + max_num_tokens: 16384 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "78" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen4tep16_mtp0_c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen4tep16_mtp0_c24.yaml new file mode 100644 index 000000000..9ea0b58b1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen4tep16_mtp0_c24.yaml @@ -0,0 +1,154 @@ +# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/1ctxDEP2c128_4genTEP16c4_E2E_C16.yaml +# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/1ctxDEP2c128_4genTEP16c4_E2E_C16__v1_sysc80_genbs5_chunk1_ctxgdn1_gengdn0_buf8192/conc80 +name: ctx1dep2_gen4tep16_mtp0_c24 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 16 + decode_workers: 4 + gpus_per_decode: 16 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + enable_chunked_prefill: true + decode: + trust_remote_code: true + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 5 + max_num_tokens: 16384 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "24" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen8tep8_mtp0_c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen8tep8_mtp0_c48.yaml new file mode 100644 index 000000000..fe33250b7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen8tep8_mtp0_c48.yaml @@ -0,0 +1,155 @@ +# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/1ctxDEP2c128_8genTEP8c4_E2E_C32.yaml +# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/1ctxDEP2c128_8genTEP8c4_E2E_C32__v1_sysc160_genbs5_chunk1_ctxgdn1_gengdn0_buf8192/conc160 +name: ctx1dep2_gen8tep8_mtp0_c48 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 16 + decode_workers: 8 + gpus_per_decode: 8 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + enable_chunked_prefill: true + decode: + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 5 + max_num_tokens: 16384 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + allreduce_strategy: MNNVL +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "48" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen15tp4_mtp0_c180.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen15tp4_mtp0_c180.yaml new file mode 100644 index 000000000..b100c7f7c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen15tp4_mtp0_c180.yaml @@ -0,0 +1,154 @@ +# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/2ctxDEP2c128_15genTP4c8_E2E_C120.yaml +# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/2ctxDEP2c128_15genTP4c8_E2E_C120__v1_sysc600_genbs10_chunk1_ctxgdn1_gengdn0_buf8192/conc600 +name: ctx2dep2_gen15tp4_mtp0_c180 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 15 + decode_workers: 15 + gpus_per_decode: 4 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + enable_chunked_prefill: true + decode: + trust_remote_code: true + tensor_parallel_size: 4 + moe_expert_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 10 + max_num_tokens: 16384 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "180" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen9tp4_mtp0_c216.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen9tp4_mtp0_c216.yaml new file mode 100644 index 000000000..595666bd0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen9tp4_mtp0_c216.yaml @@ -0,0 +1,154 @@ +# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/2ctxDEP2c512_9genTP4c16_E2E_C144.yaml +# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/2ctxDEP2c512_9genTP4c16_E2E_C144__v1_sysc720_genbs20_chunk1_ctxgdn1_gengdn0_buf8192/conc720 +name: ctx2dep2_gen9tp4_mtp0_c216 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 9 + decode_workers: 9 + gpus_per_decode: 4 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 256 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + enable_chunked_prefill: true + decode: + trust_remote_code: true + tensor_parallel_size: 4 + moe_expert_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 20 + max_num_tokens: 16384 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "216" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx3dep2_gen1dep4_mtp0_c1536.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx3dep2_gen1dep4_mtp0_c1536.yaml new file mode 100644 index 000000000..2beea78e7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx3dep2_gen1dep4_mtp0_c1536.yaml @@ -0,0 +1,155 @@ +# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/3ctxDEP2c512_1genDEP4c1024_E2E_C1024.yaml +# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/3ctxDEP2c512_1genDEP4c1024_E2E_C1024__v1_sysc5120_genbs320_chunk1_ctxgdn1_gengdn0_buf8192/conc5120 +# 1.5x performance reference only (not config-identical: GEN BS256, buffer7168, GEN GDN1): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/ablation_3xDEP2_1xDEP4_max_tpt/results/sys_conc1536__ctx_gdn1_chunked1__ctx_nt32768__gen_bs256__cgdefault__s1r1__buf7168__allranklog1/conc1536 +name: ctx3dep2_gen1dep4_mtp0_c1536 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 3 + gpus_per_prefill: 2 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 4 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 256 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + enable_chunked_prefill: true + decode: + trust_remote_code: true + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 320 + max_num_tokens: 16384 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1536" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7dep2_gen12tep4_mtp0_c1152.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7dep2_gen12tep4_mtp0_c1152.yaml new file mode 100644 index 000000000..47cc96339 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7dep2_gen12tep4_mtp0_c1152.yaml @@ -0,0 +1,157 @@ +# 9th point (C768) for exp3 8k1k mtp0 srt-slurm, matching configs/01-08 (variant-1 transforms at 1.5x sysc). +# Derived from baseline point: results/e2e/7ctxDEP2c512_12genTEP4c64_E2E_C768 (conc768). +# Variant transforms vs baseline: sysc x1.5 (768->1152), CTX chunked_prefill 0->1, FlashInfer GDN prefill 0->1 (env), +# GEN max_batch_size x1.25 (64->80), KV cache_transceiver buffer 16384->8192, TLLM_PROFILE_LOG_RANKS all, adaptive num_prompts/num_warmups. +# UNLIKE 01-08: spread_workers=False -> prefill packs 2/node (pf_nodes=4), total 16 nodes fits a single NVL72 domain (segment=16). +name: ctx7dep2_gen12tep4_mtp0_c1152 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + prefill_workers: 7 + gpus_per_prefill: 2 + decode_nodes: 12 + decode_workers: 12 + gpus_per_decode: 4 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 256 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + enable_chunked_prefill: true + decode: + trust_remote_code: true + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 80 + max_num_tokens: 16384 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1152" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx8dep2_gen9tep4_mtp0_c1728.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx8dep2_gen9tep4_mtp0_c1728.yaml new file mode 100644 index 000000000..8d19209e9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx8dep2_gen9tep4_mtp0_c1728.yaml @@ -0,0 +1,154 @@ +# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/8ctxDEP2c512_9genTEP4c128_E2E_C1152.yaml +# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/8ctxDEP2c512_9genTEP4c128_E2E_C1152__v1_sysc5760_genbs160_chunk1_ctxgdn1_gengdn0_buf8192/conc5760 +name: ctx8dep2_gen9tep4_mtp0_c1728 +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + frameworks: + tensorrt_llm: 1.3.0rc18 +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + prefill_workers: 8 + gpus_per_prefill: 2 + decode_nodes: 9 + decode_workers: 9 + gpus_per_decode: 4 + het_jobs: false +backend: + type: trtllm + prefill_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + decode_environment: + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TRTLLM_ENABLE_PDL: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + HF_HUB_OFFLINE: '1' + PYTHONNOUSERSITE: '1' + TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + trtllm_config: + prefill: + trust_remote_code: true + max_batch_size: 256 + max_num_tokens: 32768 + max_seq_len: 8395 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + cuda_graph_config: null + disable_overlap_scheduler: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED + enable_chunked_prefill: true + decode: + trust_remote_code: true + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 160 + max_num_tokens: 16384 + max_seq_len: 9419 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 32 + - 64 + - 128 + - 256 + - 384 + - 512 + print_iter_log: true + enable_iter_perf_stats: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: FIRST_COME_FIRST_SERVED +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1728" + req_rate: "inf" + num_prompts_mult: 16 +dynamo: + install: true + version: 1.3.0.dev20260708 + request_plane: tcp +infra: + etcd_nats_dedicated_node: false +profiling: + type: none +health_check: + max_attempts: 720 + interval_seconds: 10 +enable_config_dump: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 73327351e..0f39fb55e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11486,6 +11486,277 @@ qwen3.5-fp4-gb300-dynamo-sglang: ep: 16 dp-attn: true +qwen3.5-fp4-gb300-dynamo-trt: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: gb300 + precision: fp4 + framework: dynamo-trt + multinode: true + disagg: true + kv-p2p-transfer: nixl + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + # 1P1D-DEP2: 1 prefill (TP2/EP2/dp-attn), 17 decode (TP4/EP4/no-dp-attn), conc=68 + - conc-list: [68] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c68.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 4 + dp-attn: false + # 1P1D-DEP2: 1 prefill (TP2/EP2/dp-attn), 17 decode (TP4/EP1/no-dp-attn), conc=136 + - conc-list: [136] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c136.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + # 1P1D-DEP2: 1 prefill (TP2/EP2/dp-attn), 17 decode (TP4/EP1/no-dp-attn), conc=272 + - conc-list: [272] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen17dep4_mtp0_c272.yaml" + decode: + num-worker: 17 + tp: 4 + ep: 1 + dp-attn: false + # 1P1D: 1 prefill (TP2/EP2/dp-attn), 4 decode (TP16/EP16/no-dp-attn), conc=16 + - conc-list: [16] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen4dep16_mtp0_c16.yaml" + decode: + num-worker: 4 + tp: 16 + ep: 16 + dp-attn: false + # 1P5D: 1 prefill (TP2/EP2/dp-attn), 5 decode (TP8/EP8/no-dp-attn), conc=640 + - conc-list: [640] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen5dep8_mtp0_c640.yaml" + decode: + num-worker: 5 + tp: 8 + ep: 8 + dp-attn: false + # 1P1D: 1 prefill (TP2/EP2/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=1280 + - conc-list: [1280] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx1dep2_gen1dep16_mtp0_c1280.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 2P3D: 2 prefill (TP2/EP2/dp-attn), 3 decode (TP16/EP16/dp-attn), conc=1536 + - conc-list: [1536] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx2dep2_gen3dep16_mtp0_c1536.yaml" + decode: + num-worker: 3 + tp: 16 + ep: 16 + dp-attn: true + # 5P3D: 5 prefill (TP2/EP2/dp-attn), 3 decode (TP16/EP16/dp-attn), conc=7680 + - conc-list: [7680] + prefill: + num-worker: 5 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx5dep2_gen3dep16_mtp0_c7680.yaml" + decode: + num-worker: 3 + tp: 16 + ep: 16 + dp-attn: true + # 5P7D: 5 prefill (TP2/EP2/dp-attn), 7 decode (TP4/EP4/dp-attn), conc=17920 + - conc-list: [17920] + prefill: + num-worker: 5 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/1k1k/stp/ctx5dep2_gen7dep4_mtp0_c17920.yaml" + decode: + num-worker: 7 + tp: 4 + ep: 4 + dp-attn: true + - isl: 8192 + osl: 1024 + search-space: + # 1P13D: 1 prefill (TP2/EP2/dp-attn), 13 decode (TP4/EP1/no-dp-attn), conc=78 + - conc-list: [78] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen13tp4_mtp0_c78.yaml" + decode: + num-worker: 13 + tp: 4 + ep: 1 + dp-attn: false + # 1P4D: 1 prefill (TP2/EP2/dp-attn), 4 decode (TP16/EP16/no-dp-attn), conc=24 + - conc-list: [24] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen4tep16_mtp0_c24.yaml" + decode: + num-worker: 4 + tp: 16 + ep: 16 + dp-attn: false + # 1P8D: 1 prefill (TP2/EP2/dp-attn), 8 decode (TP8/EP8/no-dp-attn), conc=48 + - conc-list: [48] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen8tep8_mtp0_c48.yaml" + decode: + num-worker: 8 + tp: 8 + ep: 8 + dp-attn: false + # 2P15D: 2 prefill (TP2/EP2/dp-attn), 15 decode (TP4/EP1/no-dp-attn), conc=180 + - conc-list: [180] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen15tp4_mtp0_c180.yaml" + decode: + num-worker: 15 + tp: 4 + ep: 1 + dp-attn: false + # 2P9D: 2 prefill (TP2/EP2/dp-attn), 9 decode (TP4/EP1/no-dp-attn), conc=216 + - conc-list: [216] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen9tp4_mtp0_c216.yaml" + decode: + num-worker: 9 + tp: 4 + ep: 1 + dp-attn: false + # 3P1D: 3 prefill (TP2/EP2/dp-attn), 1 decode (TP4/EP4/dp-attn), conc=1536 + - conc-list: [1536] + prefill: + num-worker: 3 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx3dep2_gen1dep4_mtp0_c1536.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + # 7P12D: 7 prefill (TP2/EP2/dp-attn), 12 decode (TP4/EP4/no-dp-attn), conc=1152 + - conc-list: [1152] + prefill: + num-worker: 7 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7dep2_gen12tep4_mtp0_c1152.yaml" + decode: + num-worker: 12 + tp: 4 + ep: 4 + dp-attn: false + # 8P9D: 8 prefill (TP2/EP2/dp-attn), 9 decode (TP4/EP4/no-dp-attn), conc=1728 + - conc-list: [1728] + prefill: + num-worker: 8 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx8dep2_gen9tep4_mtp0_c1728.yaml" + decode: + num-worker: 9 + tp: 4 + ep: 4 + dp-attn: false + # 10P3D: 10 prefill (TP2/EP2/dp-attn), 3 decode (TP8/EP8/dp-attn), conc=4608 + - conc-list: [4608] + prefill: + num-worker: 10 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10dep2_gen3dep8_mtp0_c4608.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true + glm5-fp4-gb300-dynamo-sglang: image: lmsysorg/sglang:v0.5.11-cu130 model: nvidia/GLM-5.1-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d77c35f1f..9611a2d99 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4781,3 +4781,13 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt + description: + - "Add Qwen3.5-397B-A17B-NVFP4 GB300 disaggregated multinode TensorRT-LLM+Dynamo benchmarks (STP, no MTP) for 1k1k and 8k1k." + - "Image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 with Dynamo installed at runtime (version 1.3.0.dev20260708, TCP request plane)." + - "1k1k: 9 STP configs spanning 1P1D DEP2+17D TP4 (conc 68/136/272), 1P4D DEP2+4D TEP16 (conc 16), 1P5D DEP2+5D TEP8 (conc 640), 1P1D DEP2+1D DEP16 (conc 1280), 2P3D DEP2+3D DEP16 (conc 1536), 5P3D DEP2+3D DEP16 (conc 7680), 5P7D DEP2+7D DEP4 (conc 17920)." + - "8k1k: 9 STP configs spanning 1P13D DEP2+13D TP4 (conc 78), 1P4D DEP2+4D TEP16 (conc 24), 1P8D DEP2+8D TEP8 (conc 48), 2P15D DEP2+15D TP4 (conc 180/216), 3P1D DEP2+1D DEP4 (conc 1536), 7P12D DEP2+12D TEP4 (conc 1152), 8P9D DEP2+9D TEP4 (conc 1728), 10P3D DEP2+3D DEP8 (conc 4608)." + - "Runner: launch_gb300-nv.sh updated to clone NVIDIA/srt-slurm@sa-submission-q2-2026 and overlay local recipes from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/ for the dynamo-trt+qwen3.5+fp4 path." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2206 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 1381bd5db..494b6500e 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -229,6 +229,13 @@ elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout sa-submission-q2-2026 +elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.29 + mkdir -p recipes/trtllm/qwen3.5/gb300-fp4/disagg + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg" \ + recipes/trtllm/qwen3.5/gb300-fp4/disagg else git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" @@ -337,7 +344,7 @@ SRTCTL_APPLY_ARGS=( -f "$CONFIG_FILE" --tags "gb300,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" ) -if [[ "$IS_AGENTIC" == "1" || "$MODEL_PREFIX" == "glm5.1" || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" ) ]]; then +if [[ "$IS_AGENTIC" == "1" || "$MODEL_PREFIX" == "glm5.1" || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" ) || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-trt" ) ]]; then SRTCTL_APPLY_ARGS+=(--no-preflight) fi if [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then