From 0191fa2f16c47d4b3066775385dac9cdf6d24b96 Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Tue, 14 Jul 2026 12:37:35 -0700 Subject: [PATCH] feat(bench): add GB300 TRT disaggregated offline sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the DSV4 generation-only harness with 12 configurations, exact-full-batch GEN-log recovery, and upstream gb300-nv runner settings. Include DEP32 MTP3 c512 and exclude only DEP16 MTP1 c8192. 中文：新增 DSV4 gen-only 工具链，共包含 12 项配置，并支持从 GEN 日志恢复完整批次指标，同时采用上游 gb300-nv 运行器配置。保留 DEP32 MTP3 c512，仅排除 DEP16 MTP1 c8192。 --- .github/configs/nvidia-master.yaml | 124 +++ .../multi_node/offline/dsv4_fp4_gb300_trt.sh | 96 ++ .../offline/trtllm_gen_only/.gitignore | 12 + .../benchmark/configs/.gitkeep | 0 .../ctx1_gen1_dep32_concurrency1024_mtp3.yaml | 115 +++ .../ctx1_gen1_dep32_concurrency128_mtp3.yaml | 115 +++ .../ctx1_gen1_dep32_concurrency2048_mtp3.yaml | 115 +++ .../ctx1_gen1_dep32_concurrency256_mtp3.yaml | 115 +++ .../ctx1_gen1_dep32_concurrency512_mtp3.yaml | 115 +++ .../ctx1_gen1_dep32_concurrency64_mtp3.yaml | 115 +++ .../ctx1_gen1_tep4_concurrency1_mtp3.yaml | 114 +++ .../ctx1_gen1_tep4_concurrency2_mtp3.yaml | 114 +++ .../ctx1_gen1_tep4_concurrency4_mtp3.yaml | 114 +++ .../ctx1_gen1_tep8_concurrency1_mtp3.yaml | 115 +++ .../ctx4_gen1_dep16_concurrency4096_mtp1.yaml | 117 +++ .../ctx4_gen1_dep32_concurrency4096_mtp1.yaml | 117 +++ .../benchmark/scripts/disaggr_torch.slurm | 259 ++++++ .../benchmark/scripts/get_env.py | 33 + .../benchmark/scripts/run_benchmark.sh | 192 ++++ .../benchmark/scripts/start_server.sh | 8 + .../benchmark/scripts/start_worker.sh | 50 ++ .../benchmark/scripts/submit.py | 837 ++++++++++++++++++ .../benchmark/scripts/wait_server.sh | 33 + .../trtllm_gen_only/benchmark/submit_all.sh | 36 + .../trtllm_gen_only/dataset/gen_dataset.sh | 50 ++ .../dataset/random_generator.py | 479 ++++++++++ .../process/get_gen_only_perf.py | 207 +++++ .../trtllm_gen_only/process/process.sh | 44 + runners/launch_gb300-nv.sh | 12 + .../bench_offline/test_trt_disagg_gen_only.py | 541 +++++++++++ utils/bench_offline/trt_disagg_gen_only.py | 498 +++++++++++ utils/process_result.py | 28 +- utils/test_process_result.py | 47 +- 33 files changed, 4957 insertions(+), 10 deletions(-) create mode 100755 benchmarks/multi_node/offline/dsv4_fp4_gb300_trt.sh create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/.gitignore create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/.gitkeep create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency1024_mtp3.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency128_mtp3.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency2048_mtp3.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency256_mtp3.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency512_mtp3.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency64_mtp3.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency1_mtp3.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency2_mtp3.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency4_mtp3.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep8_concurrency1_mtp3.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep16_concurrency4096_mtp1.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep32_concurrency4096_mtp1.yaml create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/disaggr_torch.slurm create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/get_env.py create mode 100755 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/run_benchmark.sh create mode 100755 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/start_server.sh create mode 100755 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/start_worker.sh create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/submit.py create mode 100755 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/wait_server.sh create mode 100755 benchmarks/multi_node/offline/trtllm_gen_only/benchmark/submit_all.sh create mode 100755 benchmarks/multi_node/offline/trtllm_gen_only/dataset/gen_dataset.sh create mode 100644 benchmarks/multi_node/offline/trtllm_gen_only/dataset/random_generator.py create mode 100755 benchmarks/multi_node/offline/trtllm_gen_only/process/get_gen_only_perf.py create mode 100755 benchmarks/multi_node/offline/trtllm_gen_only/process/process.sh create mode 100644 utils/bench_offline/test_trt_disagg_gen_only.py create mode 100644 utils/bench_offline/trt_disagg_gen_only.py diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 4bf731b510..204b5b7e5b 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -3615,6 +3615,130 @@ dsv4-fp4-gb300-vllm-offline-16chip: search-space: - { tp: 16, ep: 16, dp-attn: true, conc-start: 32, conc-end: 512, spec-decoding: offline } +# GB300 TensorRT-LLM disaggregated generation-worker benchmark. Each row +# selects one bundled frontier config; spec-decoding=offline routes it through +# native trtllm-serve. +dsv4-fp4-gb300-trt-offline: + image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc15.post1 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: gb300-nv + precision: fp4 + framework: trt + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 256 + search-space: + - spec-decoding: offline + conc-list: [1] + prefill: &dsv4-trt-offline-ctx1 + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency1_mtp3.yaml" + decode: &dsv4-trt-offline-tep4 + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: offline + conc-list: [2] + prefill: + <<: *dsv4-trt-offline-ctx1 + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency2_mtp3.yaml" + decode: *dsv4-trt-offline-tep4 + - spec-decoding: offline + conc-list: [4] + prefill: + <<: *dsv4-trt-offline-ctx1 + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency4_mtp3.yaml" + decode: *dsv4-trt-offline-tep4 + - spec-decoding: offline + conc-list: [1] + prefill: + <<: *dsv4-trt-offline-ctx1 + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep8_concurrency1_mtp3.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: offline + conc-list: [64] + prefill: + <<: *dsv4-trt-offline-ctx1 + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency64_mtp3.yaml" + decode: &dsv4-trt-offline-dep32 + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: offline + conc-list: [128] + prefill: + <<: *dsv4-trt-offline-ctx1 + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency128_mtp3.yaml" + decode: *dsv4-trt-offline-dep32 + - spec-decoding: offline + conc-list: [256] + prefill: + <<: *dsv4-trt-offline-ctx1 + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency256_mtp3.yaml" + decode: *dsv4-trt-offline-dep32 + - spec-decoding: offline + conc-list: [512] + prefill: + <<: *dsv4-trt-offline-ctx1 + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency512_mtp3.yaml" + decode: *dsv4-trt-offline-dep32 + - spec-decoding: offline + conc-list: [1024] + prefill: + <<: *dsv4-trt-offline-ctx1 + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency1024_mtp3.yaml" + decode: *dsv4-trt-offline-dep32 + - spec-decoding: offline + conc-list: [2048] + prefill: + <<: *dsv4-trt-offline-ctx1 + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency2048_mtp3.yaml" + decode: *dsv4-trt-offline-dep32 + - spec-decoding: offline + conc-list: [4096] + prefill: &dsv4-trt-offline-ctx4 + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep16_concurrency4096_mtp1.yaml" + decode: &dsv4-trt-offline-dep16 + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: offline + conc-list: [4096] + prefill: + <<: *dsv4-trt-offline-ctx4 + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep32_concurrency4096_mtp1.yaml" + decode: *dsv4-trt-offline-dep32 + dsv4-fp4-b300-sglang-offline: image: lmsysorg/sglang:nightly-dev-cu13-20260610-f332e526 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/benchmarks/multi_node/offline/dsv4_fp4_gb300_trt.sh b/benchmarks/multi_node/offline/dsv4_fp4_gb300_trt.sh new file mode 100755 index 0000000000..8418be5c7c --- /dev/null +++ b/benchmarks/multi_node/offline/dsv4_fp4_gb300_trt.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Adapt one bundled DSV4 generation-only case to the InferenceX multi-node contract. + +set -euo pipefail + +required=( + CONFIG_FILE RESULT_FILENAME GITHUB_WORKSPACE RUNNER_NAME MODEL + CONC_LIST ISL OSL PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP + PREFILL_DP_ATTN DECODE_NUM_WORKERS DECODE_TP DECODE_EP + DECODE_DP_ATTN TRT_GEN_ONLY_SQUASH_FILE MODEL_PATH +) +for name in "${required[@]}"; do + if [[ -z "${!name:-}" ]]; then + echo "ERROR: required environment variable ${name} is unset" >&2 + exit 1 + fi +done + +if [[ $(wc -w <<<"${CONC_LIST}") -ne 1 ]]; then + echo "ERROR: each offline matrix row must select exactly one concurrency: ${CONC_LIST}" >&2 + exit 1 +fi +CONCURRENCY=${CONC_LIST} +SOURCE_CONFIG="${GITHUB_WORKSPACE}/${CONFIG_FILE}" +if [[ ! -f "${SOURCE_CONFIG}" ]]; then + echo "ERROR: source config does not exist: ${SOURCE_CONFIG}" >&2 + exit 1 +fi + +CASE_NAME=$(basename "${SOURCE_CONFIG}" .yaml) +RUN_KEY=$(printf '%s' "${RESULT_FILENAME}-${CASE_NAME}" | sha1sum | cut -c1-12) +RUN_ROOT="/data/home/sa-shared/gharunners/trt-gen-only/${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}-${RUN_KEY}" +LOG_DIR="${RUN_ROOT}/logs" +EFFECTIVE_CONFIG="${RUN_ROOT}/effective_config.yaml" +METADATA_FILE="${RUN_ROOT}/case_metadata.json" +DATASET_ROOT="/data/home/sa-shared/gharunners/datasets/dsv4-trt-offline" +ADAPTER="${GITHUB_WORKSPACE}/utils/bench_offline/trt_disagg_gen_only.py" +SUBMIT="${GITHUB_WORKSPACE}/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/submit.py" +CONTAINER_MOUNT="/data/:/data/,/scratch/:/scratch/,${GITHUB_WORKSPACE}:/workspace" + +mkdir -p "${RUN_ROOT}" "${LOG_DIR}" "${DATASET_ROOT}" + +archive_logs() { + if [[ -d "${RUN_ROOT}" ]]; then + tar czf "${GITHUB_WORKSPACE}/multinode_server_logs.tar.gz" \ + -C "${RUN_ROOT}" . 2>/dev/null || true + fi +} +trap archive_logs EXIT + +python3 "${ADAPTER}" prepare \ + --source-config "${SOURCE_CONFIG}" \ + --output-config "${EFFECTIVE_CONFIG}" \ + --metadata-output "${METADATA_FILE}" \ + --partition "${SLURM_PARTITION:-batch_1}" \ + --account "${SLURM_ACCOUNT:-benchmark}" \ + --job-time "${SLURM_JOB_TIME:-03:00:00}" \ + --job-name "${RUNNER_NAME}" \ + --container-image "${TRT_GEN_ONLY_SQUASH_FILE}" \ + --container-mount "${CONTAINER_MOUNT}" \ + --model-path "${MODEL_PATH}" \ + --dataset-root "${DATASET_ROOT}" \ + --log-dir "${LOG_DIR}" \ + --decode-steps "${OSL}" \ + --concurrency "${CONCURRENCY}" \ + --prefill-num-workers "${PREFILL_NUM_WORKERS}" \ + --prefill-tp "${PREFILL_TP}" \ + --prefill-ep "${PREFILL_EP}" \ + --prefill-dp-attn "${PREFILL_DP_ATTN}" \ + --decode-num-workers "${DECODE_NUM_WORKERS}" \ + --decode-tp "${DECODE_TP}" \ + --decode-ep "${DECODE_EP}" \ + --decode-dp-attn "${DECODE_DP_ATTN}" + +export TRT_GEN_ONLY_DATASET_GENERATOR="/workspace/benchmarks/multi_node/offline/trtllm_gen_only/dataset/gen_dataset.sh" +python3 "${SUBMIT}" \ + --config "${EFFECTIVE_CONFIG}" \ + --log-dir "${LOG_DIR}" \ + --wait + +CTX_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP)) +GEN_GPUS=$((DECODE_NUM_WORKERS * DECODE_TP)) +TOTAL_GPUS=$((CTX_GPUS + GEN_GPUS)) +RAW_BASENAME="${RESULT_FILENAME}_${CASE_NAME}_conc${CONCURRENCY}_gpus_${TOTAL_GPUS}_ctx_${CTX_GPUS}_gen_${GEN_GPUS}" +RAW_RESULT="${GITHUB_WORKSPACE}/${RAW_BASENAME}.json" + +python3 "${ADAPTER}" result \ + --config "${EFFECTIVE_CONFIG}" \ + --log-dir "${LOG_DIR}" \ + --output "${RAW_RESULT}" \ + --source-config "${CASE_NAME}.yaml" \ + --model-id "${MODEL}" \ + --decode-steps "${OSL}" + +echo "Offline disaggregated result: ${RAW_RESULT}" diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/.gitignore b/benchmarks/multi_node/offline/trtllm_gen_only/.gitignore new file mode 100644 index 0000000000..9f2349fd71 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/.gitignore @@ -0,0 +1,12 @@ +# Generated datasets +dataset/data/ + +# Python +__pycache__/ +*.pyc + +# Editor / OS +.DS_Store + +# Benchmark outputs (submit.py writes here when configs have no log_dir) +benchmark/scripts/logs/ diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/.gitkeep b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency1024_mtp3.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency1024_mtp3.yaml new file mode 100644 index 0000000000..157c3fc401 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency1024_mtp3.yaml @@ -0,0 +1,115 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx1_gen1_dep32_concurrency1024_mtp3 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '1024' + input_length: 8192 + output_length: 623 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 3 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 3 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency128_mtp3.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency128_mtp3.yaml new file mode 100644 index 0000000000..0052c9a717 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency128_mtp3.yaml @@ -0,0 +1,115 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx1_gen1_dep32_concurrency128_mtp3 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '128' + input_length: 8192 + output_length: 623 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 3 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 3 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency2048_mtp3.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency2048_mtp3.yaml new file mode 100644 index 0000000000..20ea5a6858 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency2048_mtp3.yaml @@ -0,0 +1,115 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx1_gen1_dep32_concurrency2048_mtp3 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '2048' + input_length: 8192 + output_length: 623 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 3 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 3 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency256_mtp3.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency256_mtp3.yaml new file mode 100644 index 0000000000..a9de00de02 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency256_mtp3.yaml @@ -0,0 +1,115 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx1_gen1_dep32_concurrency256_mtp3 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '256' + input_length: 8192 + output_length: 623 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 3 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 3 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency512_mtp3.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency512_mtp3.yaml new file mode 100644 index 0000000000..3cec99193e --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency512_mtp3.yaml @@ -0,0 +1,115 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx1_gen1_dep32_concurrency512_mtp3 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '512' + input_length: 8192 + output_length: 623 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 3 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 3 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency64_mtp3.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency64_mtp3.yaml new file mode 100644 index 0000000000..65c5bba7fe --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency64_mtp3.yaml @@ -0,0 +1,115 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx1_gen1_dep32_concurrency64_mtp3 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '64' + input_length: 8192 + output_length: 623 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 3 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 3 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency1_mtp3.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency1_mtp3.yaml new file mode 100644 index 0000000000..e64bae14a8 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency1_mtp3.yaml @@ -0,0 +1,114 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx1_gen1_tep4_concurrency1_mtp3 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '1' + input_length: 8192 + output_length: 623 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 3 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 3 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency2_mtp3.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency2_mtp3.yaml new file mode 100644 index 0000000000..a0e1a28f29 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency2_mtp3.yaml @@ -0,0 +1,114 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx1_gen1_tep4_concurrency2_mtp3 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '2' + input_length: 8192 + output_length: 623 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 3 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 3 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency4_mtp3.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency4_mtp3.yaml new file mode 100644 index 0000000000..0a21ad473e --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency4_mtp3.yaml @@ -0,0 +1,114 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx1_gen1_tep4_concurrency4_mtp3 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '4' + input_length: 8192 + output_length: 623 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 3 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 3 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep8_concurrency1_mtp3.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep8_concurrency1_mtp3.yaml new file mode 100644 index 0000000000..06c9015085 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep8_concurrency1_mtp3.yaml @@ -0,0 +1,115 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx1_gen1_tep8_concurrency1_mtp3 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '1' + input_length: 8192 + output_length: 623 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 3 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 3 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep16_concurrency4096_mtp1.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep16_concurrency4096_mtp1.yaml new file mode 100644 index 0000000000..02e8387ed6 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep16_concurrency4096_mtp1.yaml @@ -0,0 +1,117 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx4_gen1_dep16_concurrency4096_mtp1 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '4096' + input_length: 8192 + output_length: 435 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-435-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 4 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 512 + max_num_tokens: 1024 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 1 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 1 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep32_concurrency4096_mtp1.yaml b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep32_concurrency4096_mtp1.yaml new file mode 100644 index 0000000000..04bfb77c1d --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep32_concurrency4096_mtp1.yaml @@ -0,0 +1,117 @@ +slurm: + script_file: disaggr_torch.slurm + partition: batch_1 + account: benchmark + job_time: 03:00:00 + job_name: dsv4-offline-ctx4_gen1_dep32_concurrency4096_mtp1 + extra_args: --gres=gpu:4 + set_segment: true + numa_bind: true +benchmark: + mode: gen_only + use_nv_sa_benchmark: false + multi_round: 1 + benchmark_ratio: 0.8 + streaming: false + concurrency_list: '4096' + input_length: 8192 + output_length: 435 + dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-435-16384-ratio-1_for_serve.json +hardware: + gpus_per_node: 4 + num_ctx_servers: 4 + num_gen_servers: 1 +environment: + container_mount: /data/:/data/,/scratch/:/scratch/ + container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh + model_path: /scratch/models/DeepSeek-V4-Pro + trtllm_repo: '' + build_wheel: false + cuda_architectures: '' + trtllm_wheel_path: '' + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 + TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 + PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 + ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' + gen_worker_env_var: '' +profiling: + nsys_on: false + ctx_profile_range: 10-30 + gen_profile_range: 200-250 +accuracy: + enable_accuracy_test: false + tasks: + gsm8k: + model: local-completions + model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384 + extra_kwargs: + trust_remote_code: true +worker_config: + gen: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 512 + max_num_tokens: 1024 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + max_draft_len: 1 + ctx: + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: null + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + cache_transceiver_config: + max_tokens_in_buffer: 8192 + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + speculative_config: + decoding_type: MTP + max_draft_len: 1 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/disaggr_torch.slurm b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/disaggr_torch.slurm new file mode 100644 index 0000000000..4e8c1209dd --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/disaggr_torch.slurm @@ -0,0 +1,259 @@ +#!/bin/bash +set -euo pipefail + +# Parse named arguments +while [[ $# -gt 0 ]]; do + case $1 in + # Benchmark Configuration + --benchmark-mode) benchmark_mode="$2"; shift 2 ;; + + # Environment and paths + --trtllm-repo) trtllm_repo="$2"; shift 2 ;; + --work-dir) work_dir="$2"; shift 2 ;; + --full-logdir) full_logdir="$2"; shift 2 ;; + --container-name) container_name="$2"; shift 2 ;; + --container-mount) container_mount="$2"; shift 2 ;; + --container-image) container_image="$2"; shift 2 ;; + --build-wheel) build_wheel="$2"; shift 2 ;; + --cuda-architectures) cuda_architectures="$2"; shift 2 ;; + --trtllm-wheel-path) trtllm_wheel_path="$2"; shift 2 ;; + --dataset-file) dataset_file="$2"; shift 2 ;; + --dataset-generator) dataset_generator="$2"; shift 2 ;; + --model-path) model_path="$2"; shift 2 ;; + --dataset-input-length) dataset_input_length="$2"; shift 2 ;; + --dataset-output-length) dataset_output_length="$2"; shift 2 ;; + *) + echo "Unknown argument: $1" + exit 1 + ;; + esac +done + +# Print all parsed arguments +echo "Parsed arguments:" +echo +echo "Benchmark Configuration:" +echo " benchmark_mode: ${benchmark_mode}" +echo +echo "Environment Configuration:" +echo " trtllm_repo: ${trtllm_repo}" +echo " work_dir: ${work_dir}" +echo " full_logdir: ${full_logdir}" +echo " container_mount: ${container_mount}" +echo " container_image: ${container_image}" +echo " build_wheel: ${build_wheel}" +echo " cuda_architectures: ${cuda_architectures}" +echo " trtllm_wheel_path: ${trtllm_wheel_path}" +echo " dataset_file: ${dataset_file}" +echo " dataset_generator: ${dataset_generator}" +echo " model_path: ${model_path}" +echo " dataset shape: ${dataset_input_length}/${dataset_output_length}" + +# Set TRTLLM_DISAGG_BENCHMARK_GEN_ONLY=1 for gen_only_no_context mode +if [ "${benchmark_mode}" = "gen_only_no_context" ]; then + export TRTLLM_DISAGG_BENCHMARK_GEN_ONLY=1 + echo "Setting TRTLLM_DISAGG_BENCHMARK_GEN_ONLY=1 for gen_only_no_context mode" +fi + +# Function to cleanup on failure +cleanup_on_failure() { + echo "Error: $1" + exit 1 +} + +cleanup_background_steps() { + local child + for child in $(jobs -pr); do + kill "$child" 2>/dev/null || true + done + wait 2>/dev/null || true +} +trap cleanup_background_steps EXIT + +replace_placeholder() { + file_path="$1" + all_nodes_str="$2" + new_file_path="$3" + # A compute node can get ENODATA reading a file the login node just wrote, until its + # client-side cache revalidates (cross-client coherence lag on some parallel FS, e.g. + # aga /scratch — can exceed a minute). Refresh metadata + retry so the job doesn't abort + # under `set -e`. + i=0 + until { ls -la "$file_path" >/dev/null 2>&1; cat "$file_path" > "$new_file_path" 2>/dev/null; } && [ -s "$new_file_path" ]; do + i=$((i + 1)) + if [ "$i" -ge 120 ]; then + echo "ERROR: could not read ${file_path} after ${i} attempts" >&2 + return 1 + fi + [ $((i % 10)) -eq 0 ] && echo "waiting for ${file_path} to become readable (attempt ${i})..." + sleep 3 + done + IFS=',' read -r -a node_array <<< "$all_nodes_str" + for i in "${!node_array[@]}"; do + current_val="${node_array[$i]}" + placeholder="" + + # Use sed to replace the placeholder with the value in-place + sed -i "s|$placeholder|$current_val|g" "${new_file_path}" + echo "Replaced $placeholder with $current_val in ${new_file_path}" + done +} + +env > ${full_logdir}/environment.txt + +# Start container +echo "Starting container..." +if ! srun -l --container-image=${container_image} \ + --container-name=${container_name} \ + --container-mounts=${container_mount} \ + --mpi=pmix \ + echo "Container up." &> ${full_logdir}/1_container_launch.log; then + cleanup_on_failure "Failed to start container. Check ${full_logdir}/1_container_launch.log" +fi + +# Install TensorRT-LLM +if [ -n "${trtllm_wheel_path}" ]; then + # Install from pre-built wheel if path is provided + echo "Installing TensorRT-LLM from wheel: ${trtllm_wheel_path}..." + if ! srun --container-name=${container_name} \ + --container-mounts=${container_mount} --no-container-mount-home \ + --mpi=pmix --overlap -N $SLURM_NNODES --ntasks-per-node=1 \ + bash -c "pip install ${trtllm_wheel_path}[devel]" \ + &> ${full_logdir}/2_install.log; then + cleanup_on_failure "TensorRT-LLM wheel installation failed. Check ${full_logdir}/2_install.log for details" + fi + echo "TensorRT-LLM wheel installation completed successfully" +elif [ -d "${trtllm_repo}" ]; then + # Build and install from repository if no wheel path provided + echo "Installing TensorRT-LLM from ${trtllm_repo}..." + TRT_LLM_GIT_COMMIT=$(git -C ${trtllm_repo} rev-parse --short HEAD 2>/dev/null || echo "unknown") + echo "TRT_LLM_GIT_COMMIT: ${TRT_LLM_GIT_COMMIT}" + + if [ "${build_wheel}" = "true" ]; then + echo "Building TensorRT-LLM wheel on one node..." + build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --use_ccache --clean" + if [ -n "${cuda_architectures:-}" ]; then + build_command="${build_command} --cuda_architectures \"${cuda_architectures}\"" + fi + if ! srun --container-name=${container_name} \ + --container-mounts=${container_mount} \ + --mpi=pmix --overlap -N 1 --ntasks-per-node=1 \ + bash -c "cd ${trtllm_repo} && ${build_command}" \ + &> ${full_logdir}/2_build.log; then + cleanup_on_failure "TensorRT-LLM build failed. Check ${full_logdir}/2_build.log for details" + fi + echo "TensorRT-LLM build completed successfully" + fi + + echo "Installing TensorRT-LLM..." + if ! srun --container-name=${container_name} \ + --container-mounts=${container_mount} --no-container-mount-home \ + --mpi=pmix --overlap -N $SLURM_NNODES --ntasks-per-node=1 \ + bash -c "cd ${trtllm_repo} && pip install -e .[devel]" \ + &> ${full_logdir}/2_install.log; then + cleanup_on_failure "TensorRT-LLM installation failed. Check ${full_logdir}/2_install.log for details" + fi + echo "TensorRT-LLM installation completed successfully" +else + echo "trtllm_wheel_path and trtllm_repo are not provided, will use the installed TensorRT-LLM from the container" + # get_env file is in the same directory as this script + get_env_file=${work_dir}/get_env.py + if ! srun --container-name=${container_name} \ + --container-mounts=${container_mount} --no-container-mount-home \ + --mpi=pmix --overlap -N 1 --ntasks-per-node=1 \ + bash -c "python ${get_env_file} -e ${full_logdir}/env_vars.json" \ + &> ${full_logdir}/2_get_env.log; then + cleanup_on_failure "Failed to get TensorRT-LLM environment variables. Check ${full_logdir}/2_get_env.log for details" + fi + echo "TensorRT-LLM environment variables saved to ${full_logdir}/env_vars.json" +fi + +# Build the exact fixed-length Think-Max dataset once on shared storage. The +# lock and atomic rename let concurrently dispatched matrix jobs share it. +dataset_dir=$(dirname "${dataset_file}") +dataset_lock="${dataset_file}.lock" +mkdir -p "${dataset_dir}" +if ! srun --container-name=${container_name} \ + --container-mounts=${container_mount} --no-container-mount-home \ + --mpi=pmix --overlap -N 1 -n 1 \ + bash -s -- "${dataset_file}" "${dataset_lock}" \ + "${dataset_generator}" "${model_path}" \ + "${dataset_input_length}" "${dataset_output_length}" <<'DATASET_BASH' +set -euo pipefail +dataset_file=$1 +lock_file=$2 +generator=$3 +model_path=$4 +input_length=$5 +output_length=$6 +expected_lines=16384 +exec 9>"${lock_file}" +flock -w 7200 9 +if [[ -s "${dataset_file}" ]] && [[ $(wc -l < "${dataset_file}") -eq ${expected_lines} ]]; then + echo "Dataset cache is valid: ${dataset_file}" + exit 0 +fi +dataset_dir=$(dirname "${dataset_file}") +tmp_dir=$(mktemp -d "${dataset_dir}/.dsv4-dataset.XXXXXX") +trap 'rm -rf "${tmp_dir}"' EXIT +MODEL_PATH="${model_path}" OUT_DIR="${tmp_dir}" \ +ISL="${input_length}" OSL="${output_length}" NUM_PROMPTS="${expected_lines}" \ +RANDOM_RATIO=1 NUM_WORKERS=100 bash "${generator}" +generated="${tmp_dir}/DeepSeek-V4-${input_length}-${output_length}-${expected_lines}-ratio-1_for_serve.json" +[[ -s "${generated}" ]] +[[ $(wc -l < "${generated}") -eq ${expected_lines} ]] +mv "${generated}" "${dataset_file}" +echo "Published dataset cache: ${dataset_file}" +DATASET_BASH +then + cleanup_on_failure "Dataset preparation failed: ${dataset_file}" +fi + +# NOTE: the upstream lm_eval api_models.py patch step was removed — this repo runs gen-only +# SOL benchmarks (no accuracy eval), so lm_eval is never imported and the container doesn't +# ship it. The patch only added a failing srun + noisy ModuleNotFoundError to every job. + +# Get node lists and replace the placeholder with the actual node names +echo "SLURM_NODELIST: ${SLURM_NODELIST}" +all_nodes=($(scontrol show hostname $SLURM_NODELIST | sort)) +all_nodes_str=$(IFS=','; echo "${all_nodes[*]}") +echo "all_nodes_str: ${all_nodes_str}" + +start_server_cmds_base_file=${full_logdir}/start_server_cmds_base.sh +start_server_cmds_file=${full_logdir}/start_server_cmds.sh +replace_placeholder "${start_server_cmds_base_file}" "${all_nodes_str}" "${start_server_cmds_file}" +server_config_base_file=${full_logdir}/server_config_base.yaml +server_config_file=${full_logdir}/server_config.yaml +replace_placeholder "${server_config_base_file}" "${all_nodes_str}" "${server_config_file}" +client_cmds_base_file=${full_logdir}/client_cmds_base.sh +client_cmds_file=${full_logdir}/client_cmds.sh +replace_placeholder "${client_cmds_base_file}" "${all_nodes_str}" "${client_cmds_file}" + +# start the servers (skip ctx workers if TRTLLM_DISAGG_BENCHMARK_GEN_ONLY is set). +echo "Starting worker commands from ${start_server_cmds_file}..." +cat ${start_server_cmds_file} | while read cmd; do + # Skip ctx worker commands if in gen-only mode + # CTX appears as argument to start_worker.sh and in log filename + if [ "${TRTLLM_DISAGG_BENCHMARK_GEN_ONLY:-0}" = "1" ] && [[ "$cmd" == *"start_worker.sh CTX"* ]]; then + echo "Skipping ctx worker command (TRTLLM_DISAGG_BENCHMARK_GEN_ONLY is set): ${cmd}" + continue + fi + echo "Executing command: ${cmd}" + eval "${cmd}" +done +echo "Server is ready!" + +# Start client commands +echo "Starting client commands from ${client_cmds_file}..." +while read -r cmd <&3; do + echo "Starting client command: ${cmd}" + eval "${cmd}" + if [ $? -ne 0 ]; then + cleanup_on_failure "Command failed: ${cmd}." + fi +done 3< "${client_cmds_file}" + +echo "Job completed successfully, total runtime: $SECONDS seconds" + +# Background server steps are terminated by the EXIT trap. Exiting normally +# preserves a successful Slurm state for `sbatch --wait` and GitHub Actions. diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/get_env.py b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/get_env.py new file mode 100644 index 0000000000..0120c9f0f1 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/get_env.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +import argparse +import json +import os + + +def main(): + parser = argparse.ArgumentParser( + description="Get TensorRT-LLM environment variables and save to JSON" + ) + parser.add_argument("-e", "--env-file", required=True, help="Environment file path") + args = parser.parse_args() + + # read env file, append new envs to it + with open(args.env_file, "r") as f: + env_data = json.load(f) + + # Get environment variables + new_env_data = { + "TRT_LLM_GIT_COMMIT": os.environ.get("TRT_LLM_GIT_COMMIT", ""), + "TRT_LLM_VERSION": os.environ.get("TRT_LLM_VERSION", ""), + } + print(f"Environment variables: {new_env_data}") + env_data.update(new_env_data) + # Save to environment file + with open(args.env_file, "w") as f: + json.dump(env_data, f, indent=2) + + print(f"Environment variables saved to {args.env_file}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/run_benchmark.sh b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/run_benchmark.sh new file mode 100755 index 0000000000..c7180d70a9 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/run_benchmark.sh @@ -0,0 +1,192 @@ +#!/bin/bash + +# Add error handling +set -e +set -u +trap 'echo "Error occurred at line $LINENO"; exit 1' ERR + +# Add parameter validation +if [ "$#" -lt 10 ]; then + echo "Error: Missing required arguments, got $# arguments, args: $@" + echo "Usage: $0 model_name dataset_file multi_round num_gen_servers concurrency_list streaming log_path hostname port ucx_warmup_requests" + exit 1 +fi + +model_name=$1 +dataset_file=$2 +multi_round=$3 +num_gen_servers=$4 +concurrency_list=$5 +streaming=$6 +log_path=$7 +hostname=$8 +port=$9 +ucx_warmup_requests=${10} + +# check process id is not 0 +if [[ ${SLURM_PROCID} != "0" ]]; then + echo "Process id is ${SLURM_PROCID} for loadgen, exiting" + exit 0 +fi + +do_get_logs(){ + local input_file=$1 + local output_file=$2 + local mode=$3 + local start_line=$4 + # check mode is ctx or gen + if [ "${mode}" = "ctx" ]; then + sed -n "${start_line},\$p" ${input_file} | grep -a "'num_generation_tokens': 0" > ${output_file} || true + elif [ "${mode}" = "gen" ]; then + sed -n "${start_line},\$p" ${input_file} | grep -a "'num_ctx_requests': 0, 'num_ctx_tokens': 0" > ${output_file} || true + else + echo "Invalid mode: ${mode}" + return 1 + fi + return 0 +} + +do_process_all_logs(){ + local input_folder=$1 + local output_folder=$2 + local mode=$3 + if [ "${mode}" != "line" ] && [ "${mode}" != "log" ] && [ "${mode}" != "clean" ]; then + echo "Invalid mode: ${mode}" + exit 1 + fi + local ctx_log + local ctx_num + local gen_log + local gen_num + local line_count + local start_line + for ctx_log in ${input_folder}/3_output_CTX_*.log; do + if [ -f "${ctx_log}" ]; then + ctx_num=$(basename "${ctx_log}" | sed 's/3_output_CTX_\([0-9]*\)\.log/\1/') + if [ "${mode}" = "line" ]; then + line_count=$(wc -l < ${ctx_log}) + echo ${line_count} > ${output_folder}/ctx_only_line_${ctx_num}.txt + elif [ "${mode}" = "log" ]; then + if [ ! -f "${output_folder}/ctx_only_line_${ctx_num}.txt" ]; then + start_line=1 + else + # wc -l records the last warmup line number. Timed parsing + # must begin on the following line, never on that row. + start_line=$(cat "${output_folder}/ctx_only_line_${ctx_num}.txt") + start_line=$((start_line + 1)) + rm -f ${output_folder}/ctx_only_line_${ctx_num}.txt + fi + do_get_logs ${ctx_log} ${output_folder}/ctx_only_${ctx_num}.txt "ctx" ${start_line} + elif [ "${mode}" = "clean" ]; then + rm -f ${ctx_log} + fi + fi + done + # process all the gen log files in the input folder + for gen_log in ${input_folder}/3_output_GEN_*.log; do + if [ -f "${gen_log}" ]; then + gen_num=$(basename "${gen_log}" | sed 's/3_output_GEN_\([0-9]*\)\.log/\1/') + if [ "${mode}" = "line" ]; then + line_count=$(wc -l < ${gen_log}) + echo ${line_count} > ${output_folder}/gen_only_line_${gen_num}.txt + elif [ "${mode}" = "log" ]; then + if [ ! -f "${output_folder}/gen_only_line_${gen_num}.txt" ]; then + start_line=1 + else + # Keep the entire warmup cohort outside the timed iterlog. + start_line=$(cat "${output_folder}/gen_only_line_${gen_num}.txt") + start_line=$((start_line + 1)) + rm -f ${output_folder}/gen_only_line_${gen_num}.txt + fi + do_get_logs ${gen_log} ${output_folder}/gen_only_${gen_num}.txt "gen" ${start_line} + elif [ "${mode}" = "clean" ]; then + rm -f ${gen_log} + fi + fi + done + if [ "${mode}" = "clean" ]; then + if [ -d "${tmp_start_logs}" ]; then + mkdir -p ${log_path}/start_logs + cp ${tmp_start_logs}/3_output_CTX_*.log ${log_path}/start_logs/ 2>/dev/null || true + cp ${tmp_start_logs}/3_output_GEN_*.log ${log_path}/start_logs/ 2>/dev/null || true + rm -rf ${tmp_start_logs} + fi + fi +} + +tmp_start_logs=/tmp/${SLURM_JOB_ID}/start_logs +mkdir -p ${tmp_start_logs} +cp ${log_path}/3_output_CTX_*.log ${tmp_start_logs}/ 2>/dev/null || true +cp ${log_path}/3_output_GEN_*.log ${tmp_start_logs}/ 2>/dev/null || true + +# warmup requests for ucx connections +if [ "${ucx_warmup_requests}" -gt 0 ]; then + echo "warming up ucx connections with small requests... ${ucx_warmup_requests}" + python -m tensorrt_llm.serve.scripts.benchmark_serving \ + --model ${model_name} \ + --dataset-name random \ + --random-ids \ + --random-input-len 100 \ + --random-output-len 10 \ + --num-prompts ${ucx_warmup_requests} \ + --host ${hostname} \ + --port ${port} \ + --ignore-eos \ + --non-streaming + echo "UCX warmup done" +fi + +echo "Hostname: ${hostname}, Port: ${port}" +echo "Starting benchmark..." +for concurrency in ${concurrency_list}; do + concurrency=$((concurrency * num_gen_servers)) + num_prompts=$((concurrency * multi_round)) + warmup_dir=${log_path}/warmup_concurrency_${concurrency} + mkdir -p "${warmup_dir}" + echo "Warming up one full cohort at concurrency ${concurrency}" + python -m tensorrt_llm.serve.scripts.benchmark_serving \ + --model ${model_name} \ + --backend openai \ + --host ${hostname} \ + --port ${port} \ + --dataset-name "trtllm_custom" \ + --dataset-path ${dataset_file} \ + --num-prompts ${concurrency} \ + --max-concurrency ${concurrency} \ + --trust-remote-code \ + --ignore-eos \ + --no-test-input \ + --save-result \ + --result-dir "${warmup_dir}" \ + --result-filename "result.json" \ + --percentile-metrics "ttft,tpot,itl,e2el" \ + $(if [ "${streaming}" = "false" ]; then echo "--non-streaming"; fi) + echo "Warmup cohort completed; timed log offsets begin after this point" + sleep 2 + + echo "Timing concurrency ${concurrency} with ${num_prompts} prompts" + mkdir -p ${log_path}/concurrency_${concurrency} + # Snapshot line offsets only after the complete warmup cohort. The + # extracted gen_only log therefore contains timed iterations exclusively. + do_process_all_logs ${log_path}/ ${log_path}/concurrency_${concurrency} "line" + python -m tensorrt_llm.serve.scripts.benchmark_serving \ + --model ${model_name} \ + --backend openai \ + --host ${hostname} \ + --port ${port} \ + --dataset-name "trtllm_custom" \ + --dataset-path ${dataset_file} \ + --num-prompts ${num_prompts} \ + --max-concurrency ${concurrency} \ + --trust-remote-code \ + --ignore-eos \ + --no-test-input \ + --save-result \ + --result-dir "${log_path}/concurrency_${concurrency}" \ + --result-filename "result.json" \ + --percentile-metrics "ttft,tpot,itl,e2el" \ + $(if [ "${streaming}" = "false" ]; then echo "--non-streaming"; fi) + echo "Timed benchmark with concurrency ${concurrency} done" + do_process_all_logs ${log_path}/ ${log_path}/concurrency_${concurrency} "log" +done +# do_process_all_logs ${log_path}/ ${log_path}/concurrency_${concurrency} "clean" diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/start_server.sh b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/start_server.sh new file mode 100755 index 0000000000..ff8e5aa902 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/start_server.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -u +set -e +set -x + +config_file=$1 + +trtllm-serve disaggregated -c ${config_file} -t 7200 -r 7200 diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/start_worker.sh b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/start_worker.sh new file mode 100755 index 0000000000..791d5a40af --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/start_worker.sh @@ -0,0 +1,50 @@ +#! /bin/bash +set -u +set -e +set -x + +role=${1} +instance_id=${2} +model_path=${3} +port=${4} +numa_bind=${5} +log_dir=${6} +enable_nsys=${7} +config_file=${8} +cuda_devices=${9} + +# Set CUDA_VISIBLE_DEVICES from script argument (srun --export cannot +# reliably pass comma-separated values inside shared containers). +export CUDA_VISIBLE_DEVICES=${cuda_devices} + +# GB300/GB200 NVL72 (4 GPU/node) has no InfiniBand RC among the intra-node GPUs, so UCX must +# not probe 'rc' or the NIXL KV-cache transfer backend fails to initialize +# ("Failed to create NIXL backend: UCX"). Pin the NVLink/host transports. +export UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp + +echo "SLURM_PROCID: ${SLURM_PROCID}, hostname: $(hostname), instance_id: ${instance_id}" +echo "CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES}" + +if [ "${numa_bind}" = "true" ]; then + numa_bind_cmd="numactl -m 0,1" + echo "numactl -m 0,1 - Only allocate memory from nodes on GB200/GB300 NVL72" +else + numa_bind_cmd="" + echo "Not binding memory. If on GB200/GB300 NVL72, use \"numactl -m 0,1\" to only allocate memory from nodes." +fi + +echo "config_file: ${config_file}" + +nsys_prefix="" +if [ "${enable_nsys}" != "true" ]; then + echo "nsys is not enabled, start normal flow" +else + nsys_file=${log_dir}/nsys_worker_proc_${role}_${instance_id}_${SLURM_PROCID} + echo "nsys is enabled on ${role} GPUs, TLLM_PROFILE_START_STOP=${TLLM_PROFILE_START_STOP}" + nsys_prefix="nsys profile -o ${nsys_file} -f true -t cuda,nvtx,python-gil -c cudaProfilerApi --cuda-graph-trace node --capture-range-end=stop --gpu-metrics-devices=none" +fi + +${nsys_prefix} ${numa_bind_cmd} trtllm-llmapi-launch \ + trtllm-serve ${model_path} \ + --host $(hostname) --port ${port} \ + --config ${config_file} diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/submit.py b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/submit.py new file mode 100644 index 0000000000..67eb2d7e71 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/submit.py @@ -0,0 +1,837 @@ +#!/usr/bin/env python3 + +import argparse +import glob +import json +import math +import os +import shutil +import subprocess +import sys +import traceback +from datetime import datetime +from typing import Any, Dict, List + +import yaml + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Submit disaggregated benchmark job') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-c', + '--config', + type=str, + help='Path to the configuration YAML file') + group.add_argument('-d', + '--dir', + type=str, + help='Directory containing YAML configuration files') + parser.add_argument('--log-dir', + type=str, + default=None, + help='Log directory') + parser.add_argument('--dry-run', + action='store_true', + help='Dry run the Python part, test purpose only') + parser.add_argument('--wait', + action='store_true', + help='Wait for the submitted Slurm job and propagate its status') + return parser.parse_args() + + +def load_config(config_path): + with open(config_path, 'r') as f: + return yaml.safe_load(f) + + +def save_worker_config(worker_config, output_path): + """Save worker config to a separate YAML file.""" + os.makedirs(os.path.dirname(output_path), exist_ok=True) + with open(output_path, 'w') as f: + yaml.dump(worker_config, f, default_flow_style=False) + + +def calculate_nodes(world_size, num_servers, gpus_per_node): + """Calculate required nodes based on world size and server count.""" + return math.ceil(world_size * num_servers / gpus_per_node) + + +def allocate_gpus( + total_nodes: int, + gpus_per_node: int, + num_gen_servers: int, + num_ctx_servers: int, + gen_world_size: int, + ctx_world_size: int, + base_port: int = 8000, +) -> List[Dict[str, Any]]: + allocations = {} + hostnames = [f"" for i in range(total_nodes)] + + global_gpu_cursor = 0 + + def get_gpu_location(gpus_per_node: int): + node_id = global_gpu_cursor // gpus_per_node + local_gpu_id = global_gpu_cursor % gpus_per_node + return node_id, local_gpu_id + + def assign_server(server_allocation: Dict[str, Any], world_size: int, + gpus_per_node: int): + nonlocal global_gpu_cursor + for _ in range(world_size): + node_id, gpu_id = get_gpu_location(gpus_per_node) + hostname = hostnames[node_id] + if hostname not in server_allocation["nodes"]: + server_allocation["nodes"][hostname] = [] + server_allocation["nodes"][hostname].append(gpu_id) + global_gpu_cursor += 1 + + port = base_port + + def assign_servers( + server_allocations: Dict[str, Any], + server_type: str, + num_servers: int, + world_size: int, + gpus_per_node: int, + ): + nonlocal port + if server_type not in server_allocations: + server_allocations[server_type] = {} + for i in range(num_servers): + server_allocation = { + "port": port, + "nodes": {}, + } + assign_server(server_allocation, world_size, gpus_per_node) + server_allocations[server_type][i] = server_allocation + port += 1 + + # Keep the allocation order aligned with disagg_utils, which builds + # server_configs as ctx_cfgs + gen_cfgs and assigns rank offsets in that + # same order during split_world_comm(). + assign_servers(allocations, "CTX", num_ctx_servers, ctx_world_size, + gpus_per_node) + assign_servers(allocations, "GEN", num_gen_servers, gen_world_size, + gpus_per_node) + + return allocations + + +def convert_allocations_to_server_config(allocations, + server_port=8333, + router_config=None): + generation_servers = {} + context_servers = {} + server_hostname = None + + for server_type in allocations.keys(): + num_servers = len(allocations[server_type]) + urls = [] + for server_id in allocations[server_type].keys(): + instance = allocations[server_type][server_id] + urls.append( + f"{list(instance['nodes'].keys())[0]}:{instance['port']}") + + server_config_entry = {'num_instances': num_servers, 'urls': urls} + if router_config: + server_config_entry['router'] = router_config.copy() + + if server_type == "GEN": + generation_servers = server_config_entry + server_hostname = urls[0].split(':')[0] + if allocations[server_type][server_id]['port'] == server_port: + server_port += 1 # Avoid port conflict + elif server_type == "CTX": + context_servers = server_config_entry + + server_config = { + 'backend': 'pytorch', + 'hostname': server_hostname, + 'port': server_port, + 'context_servers': context_servers, + 'generation_servers': generation_servers + } + return server_config + + +def upsert_env_config(env_config, config_key, key_name, value_str): + """Upsert env var into env_config key. + + Replaces existing entry for the same key name, or prepends if not present. + """ + parts = [ + part for part in env_config.get(config_key, '').split() + if not part.startswith(f"{key_name}=") + ] + env_config[config_key] = " ".join([value_str, *parts]).strip() + + +def convert_envs_to_str(env_vars: Dict[str, str]) -> str: + return ','.join([f"{key}='{value}'" for key, value in env_vars.items()]) + + +def replace_env_in_file(log_dir, file_path, env_var): + with open(file_path, 'r', encoding='utf-8') as f: + config_content = f.read() + + for env_name, env_value in env_var.items(): + file_content = config_content.replace(env_name, env_value) + + tmp_dir = os.path.join(log_dir, "lm_eval_configs") + os.makedirs(tmp_dir, exist_ok=True) + tmp_file = os.path.join(tmp_dir, os.path.basename(file_path)) + + # Write modified config to temp file + with open(tmp_file, 'w', encoding='utf-8') as f: + f.write(file_content) + + # Check if has custom utils.py in the same directory + # Needed for GPQA task + custom_utils_path = os.path.join(os.path.dirname(file_path), 'utils.py') + if os.path.exists(custom_utils_path): + # copy utils.py to temp directory + shutil.copy(custom_utils_path, tmp_dir) + + # Return temp directory + return tmp_dir + + +def build_worker_environment(worker_config, env_config, role, benchmark_mode, + nsys_on, profile_range, concurrency): + """Build complete environment dictionary for worker processes. + + Args: + worker_config: Worker configuration dict + env_config: Environment configuration dict + role: Server role ("CTX" or "GEN") + benchmark_mode: Benchmark mode string + nsys_on: Whether nsys profiling is enabled + profile_range: Profile range string (e.g., "10-30") + concurrency: Concurrency level + + Returns: + Dictionary of environment variables + + Note: + CUDA_VISIBLE_DEVICES is passed as an argument to start_worker.sh, + not via srun --export (which cannot reliably pass comma-separated + values inside shared containers). + """ + env = {} + + # 1. Add mode-based env vars to env_config + if benchmark_mode == "gen_only_no_context": + upsert_env_config(env_config, 'worker_env_var', + 'TRTLLM_DISAGG_BENCHMARK_GEN_ONLY', + 'TRTLLM_DISAGG_BENCHMARK_GEN_ONLY=1') + if benchmark_mode == "gen_only": + upsert_env_config(env_config, 'worker_env_var', + 'TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP', + 'TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1') + if role == "GEN": + gen_config = worker_config.get('gen', {}) + concurrency_int = int(concurrency) + max_batch_size = int( + gen_config.get('max_batch_size', concurrency_int)) + enable_attention_dp = gen_config.get('enable_attention_dp', False) + tp_size = int(gen_config.get('tensor_parallel_size', 1)) + max_capacity = ((max_batch_size * tp_size) + if enable_attention_dp else max_batch_size) + queue_size = min(max_capacity, concurrency_int) + if queue_size < concurrency_int: + print(f"[WARNING] TLLM_BENCHMARK_REQ_QUEUES_SIZE capped to " + f"{queue_size} (max_batch_size={max_batch_size} x " + f"tp_size={tp_size} with " + f"attention_dp={enable_attention_dp}) " + f"which is less than concurrency={concurrency}. " + f"Fill loop would hang if set to {concurrency}.") + upsert_env_config(env_config, 'gen_worker_env_var', + 'TLLM_BENCHMARK_REQ_QUEUES_SIZE', + f'TLLM_BENCHMARK_REQ_QUEUES_SIZE={queue_size}') + + # 2. Add profiling env vars to env_config (conditional) + if nsys_on: + upsert_env_config(env_config, 'worker_env_var', + 'TLLM_PROFILE_RECORD_GC', 'TLLM_PROFILE_RECORD_GC=1') + upsert_env_config(env_config, 'worker_env_var', 'TLLM_NVTX_DEBUG', + 'TLLM_NVTX_DEBUG=1') + upsert_env_config(env_config, 'worker_env_var', + 'NSYS_MPI_STORE_TEAMS_PER_RANK', + 'NSYS_MPI_STORE_TEAMS_PER_RANK=1') + if role == "CTX": + upsert_env_config(env_config, 'ctx_worker_env_var', + 'TLLM_PROFILE_START_STOP', + f'TLLM_PROFILE_START_STOP={profile_range}') + elif role == "GEN": + upsert_env_config(env_config, 'gen_worker_env_var', + 'TLLM_PROFILE_START_STOP', + f'TLLM_PROFILE_START_STOP={profile_range}') + + # 3. Parse user-defined worker env vars from config + # (now includes mode-based and profiling vars from steps 1-2) + worker_env_var = env_config.get('worker_env_var', '') + for var_string in worker_env_var.split(): + if '=' in var_string: + key, val = var_string.split('=', 1) + env[key] = val + + # 4. Add role-specific env vars (CTX or GEN) + # (now includes role-specific mode/profiling vars from steps 1-2) + role_env_vars = { + "CTX": env_config.get('ctx_worker_env_var', ''), + "GEN": env_config.get('gen_worker_env_var', '') + } + role_specific_env_var = role_env_vars.get(role, '') + for var_string in role_specific_env_var.split(): + if '=' in var_string: + key, val = var_string.split('=', 1) + env[key] = val + + return env + + +def build_server_environment(env_config, benchmark_mode): + """Build complete environment dictionary for server process. + + Args: + env_config: Environment configuration dict + benchmark_mode: Benchmark mode string + + Returns: + Dictionary of environment variables + """ + env = {} + + # Add mode-based env vars to env_config + if benchmark_mode == "gen_only_no_context": + upsert_env_config(env_config, 'server_env_var', + 'TRTLLM_DISAGG_BENCHMARK_GEN_ONLY', + 'TRTLLM_DISAGG_BENCHMARK_GEN_ONLY=1') + + # Parse user-defined server env vars (now includes mode-based vars) + server_env_var = env_config.get('server_env_var', '') + for var_string in server_env_var.split(): + if '=' in var_string: + key, val = var_string.split('=', 1) + env[key] = val + + return env + + +def format_export_string(env_dict): + """Convert environment dictionary to srun --export format. + + Args: + env_dict: Dictionary of environment variables + + Returns: + String formatted for srun --export flag (e.g., "KEY1=val1,KEY2=val2") + Returns "NONE" if no variables specified. + + Note: + Values containing commas are quoted to avoid conflicts with srun's delimiter. + """ + if not env_dict: + return "NONE" + + export_list = [] + for k, v in env_dict.items(): + # srun cannot handle values that contain commas + if ',' in v: + export_list.append(f"'{k}={v}'") + else: + export_list.append(f"{k}={v}") + return ",".join(export_list) + + +def save_env_file(env_file, server_env_var, worker_env_var, ctx_worker_env_var, + gen_worker_env_var): + + def get_env_var_str(env_var_str): + env_data = {} + for env_var in env_var_str.split(): + if '=' in env_var: + key, value = env_var.split('=', 1) + env_data[key] = value + return env_data + + env_data = {} + env_data['server_env_var'] = get_env_var_str(server_env_var) + env_data['worker_env_var'] = get_env_var_str(worker_env_var) + env_data['ctx_worker_env_var'] = get_env_var_str(ctx_worker_env_var) + env_data['gen_worker_env_var'] = get_env_var_str(gen_worker_env_var) + with open(env_file, 'w') as f: + json.dump(env_data, f, indent=2) + print(f"Environment variables saved to {env_file}") + + +def submit_job(config, log_dir, dry_run, wait): + # Extract configurations + slurm_config = config['slurm'] + slurm_config.setdefault('extra_args', '') + slurm_config.setdefault('set_segment', True) + + hw_config = config['hardware'] + env_config = config['environment'] + worker_config = config['worker_config'] + benchmark_config = config['benchmark'] + + if 'work_dir' in env_config and os.path.isdir(env_config['work_dir']): + script_dir = env_config['work_dir'] + else: + script_dir = os.path.dirname(os.path.abspath(__file__)) + + # Set default accuracy configuration for backward compatibility + if 'accuracy' not in config: + config['accuracy'] = { + 'enable_accuracy_test': + False, + 'model': + 'local-completions', + 'tasks': + 'gsm8k', + 'model_args_extra': + 'num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=1200,max_gen_toks=256,max_length=4096' + } + + # Set default environment configuration for backward compatibility + env_config.setdefault('trtllm_repo', '') + env_config.setdefault('build_wheel', False) + env_config.setdefault('cuda_architectures', '') + env_config.setdefault('trtllm_wheel_path', '') + env_config.setdefault('worker_env_var', '') + env_config.setdefault('server_env_var', '') + + profiling_config = config.get('profiling', {}) + profiling_config.setdefault('nsys_on', False) + profiling_config.setdefault('ctx_profile_range', '10-30') + profiling_config.setdefault('gen_profile_range', '200-250') + + # Get number of servers from config + ctx_num = hw_config['num_ctx_servers'] + gen_num = hw_config['num_gen_servers'] + gpus_per_node = hw_config['gpus_per_node'] + + # Calculate nodes based on world sizes + ctx_tp_size = worker_config['ctx'].get('tensor_parallel_size', 1) + ctx_cp_size = worker_config['ctx'].get('context_parallel_size', 1) + ctx_pp_size = worker_config['ctx'].get('pipeline_parallel_size', 1) + ctx_world_size = ctx_tp_size * ctx_cp_size * ctx_pp_size + ctx_nodes = calculate_nodes(ctx_world_size, ctx_num, gpus_per_node) + + gen_tp_size = worker_config['gen'].get('tensor_parallel_size', 1) + gen_cp_size = worker_config['gen'].get('context_parallel_size', 1) + gen_pp_size = worker_config['gen'].get('pipeline_parallel_size', 1) + gen_world_size = gen_tp_size * gen_cp_size * gen_pp_size + gen_nodes = calculate_nodes(gen_world_size, gen_num, gpus_per_node) + ucx_warmup_requests = 2 * ctx_world_size * \ + gen_world_size if benchmark_config['mode'] == "e2e" else 0 + + total_nodes = ctx_nodes + gen_nodes + total_tasks = total_nodes * gpus_per_node + + # Generate log directory path based on configuration + isl = benchmark_config['input_length'] + osl = benchmark_config['output_length'] + gen_batch_size = worker_config['gen']['max_batch_size'] + gen_enable_attention_dp = worker_config['gen']['enable_attention_dp'] + + # Get eplb num_slots for gen worker + load_balancer_config = worker_config['gen'].get('moe_config', {}).get( + 'load_balancer', {}) + if isinstance(load_balancer_config, str): + with open(load_balancer_config, 'r') as f: + load_balancer_config = yaml.safe_load(f) + eplb_num_slots = load_balancer_config.get('num_slots', 0) + + # Get mtp_size from gen config's speculative_config. Leave it unset when + # max_draft_len is omitted so auto-MTP runs are not mislabeled as mtp0. + mtp_size = worker_config['gen'].get('speculative_config', + {}).get('max_draft_len') + + # Create base log directory path + if 'log_dir' in env_config and env_config['log_dir']: + log_dir = env_config['log_dir'] + if log_dir is None: + log_base = os.path.join(script_dir, "logs") + + date_prefix = datetime.now().strftime("%Y%m%d-%H%M%S") + log_base = os.path.join(log_base, f"{date_prefix}/{isl}-{osl}") + + mtp_suffix = "" if mtp_size is None else f"_mtp{mtp_size}" + # Include concurrency so single-concurrency configs that share (tp, batch, mtp) don't + # collide on the same auto-generated log dir when environment.log_dir is unset. + conc_suffix = f"_conc{str(benchmark_config['concurrency_list']).replace(',', '-')}" + + # Determine directory suffix based on attention_dp + if gen_enable_attention_dp: + dir_suffix = f"disagg_ctx{ctx_num}_gen{gen_num}_dep{gen_tp_size}_batch{gen_batch_size}{conc_suffix}_eplb{eplb_num_slots}{mtp_suffix}" + else: + dir_suffix = f"disagg_ctx{ctx_num}_gen{gen_num}_tep{gen_tp_size}_batch{gen_batch_size}{conc_suffix}_eplb{eplb_num_slots}{mtp_suffix}" + + # Create full log directory path + log_dir = os.path.join(log_base, dir_suffix) + + # if trtllm_config.yaml exists, don't remove the directory, remove other files in the directory except trtllm_config.yaml + # also don't remove concurrency_* folders + if os.path.exists(log_dir): + if not os.path.exists(os.path.join(log_dir, 'trtllm_config.yaml')): + print(f"[WARNING] Removing existing log directory: {log_dir}") + shutil.rmtree(log_dir) + else: + print( + f"[WARNING] trtllm_config.yaml exists, not removing the directory: {log_dir}" + ) + for file in os.listdir(log_dir): + if file != 'trtllm_config.yaml' and not file.startswith( + 'concurrency_'): + if os.path.isdir(os.path.join(log_dir, file)): + shutil.rmtree(os.path.join(log_dir, file)) + else: + os.remove(os.path.join(log_dir, file)) + os.makedirs(log_dir, exist_ok=True) + print(f"Log will be saved to: {log_dir}") + + # Setup config file paths and save worker configs + ctx_config_path = os.path.join(log_dir, 'ctx_config.yaml') + gen_config_path = os.path.join(log_dir, 'gen_config.yaml') + save_worker_config(worker_config['ctx'], ctx_config_path) + save_worker_config(worker_config['gen'], gen_config_path) + + # Prepare allocation template + allocations = allocate_gpus( + total_nodes=total_nodes, + gpus_per_node=gpus_per_node, + num_gen_servers=gen_num, + num_ctx_servers=ctx_num, + gen_world_size=gen_world_size, + ctx_world_size=ctx_world_size, + ) + with open(os.path.join(log_dir, "allocations.json"), "w") as f: + json.dump(allocations, f, indent=2) + + # Generate disagg server config + router_config = config.get('router_config', None) + server_config = convert_allocations_to_server_config( + allocations, router_config=router_config) + # Merge server_config_extra into disagg server config + if 'server_config_extra' in config: + server_config.update(config['server_config_extra']) + with open(os.path.join(log_dir, "server_config_base.yaml"), "w") as f: + yaml.dump(server_config, f) + disagg_server_hostname = server_config['hostname'] + disagg_server_port = server_config['port'] + + default_dataset_generator = os.path.abspath( + os.path.join(script_dir, '../../dataset/gen_dataset.sh')) + dataset_generator = os.environ.get( + 'TRT_GEN_ONLY_DATASET_GENERATOR', default_dataset_generator) + + container_name = "disaggr-test" + start_server_cmds = [] + container_mount_str = env_config['container_mount'] + container_mount_str += f",{script_dir}:{script_dir}" + mount_destinations = [ + entry.split(':', 1)[1] + for entry in container_mount_str.split(',') + if ':' in entry + ] + generator_is_mounted = any( + os.path.commonpath([ + os.path.abspath(dataset_generator), + os.path.abspath(destination), + ]) == os.path.abspath(destination) + for destination in mount_destinations + ) + if not generator_is_mounted and os.path.isfile(dataset_generator): + generator_dir = os.path.dirname(dataset_generator) + container_mount_str += f",{generator_dir}:{generator_dir}" + + # Pre-define server-type-specific configurations + server_configs = { + "GEN": { + "world_size": gen_world_size, + "profile_range": profiling_config['gen_profile_range'], + "config_path": gen_config_path + }, + "CTX": { + "world_size": ctx_world_size, + "profile_range": profiling_config['ctx_profile_range'], + "config_path": ctx_config_path + } + } + + for server_type in allocations.keys(): + server_cfg = server_configs[server_type] + + for server_id in allocations[server_type].keys(): + allocation = allocations[server_type][server_id] + gpu_ids = list(allocation["nodes"].values())[0] + + cuda_devices = ','.join(map(str, gpu_ids)) + worker_env = build_worker_environment( + worker_config=worker_config, + env_config=env_config, + role=server_type, + benchmark_mode=benchmark_config['mode'], + nsys_on=profiling_config['nsys_on'], + profile_range=server_cfg['profile_range'], + concurrency=benchmark_config['concurrency_list'].split(',')[0], + ) + export_str = format_export_string(worker_env) + + cmd = [ + "srun -l", + f"--nodelist {','.join(allocation['nodes'].keys())}", + f"-N {len(allocation['nodes'])}", + f"--ntasks {server_cfg['world_size']}", + f"--ntasks-per-node {gpus_per_node}", + f"--export=\"{export_str}\"", + f"--container-image {env_config['container_image']}", + f"--container-name {container_name}", + f"--container-mounts {container_mount_str}", + "--no-container-mount-home --mpi=pmix --overlap", + f"bash {os.path.join(script_dir, 'start_worker.sh')}", + server_type, + str(server_id), + env_config['model_path'], + str(allocation["port"]), + str(slurm_config['numa_bind']).lower(), + log_dir, + str(profiling_config['nsys_on']).lower(), + server_cfg['config_path'], + cuda_devices, + f"&> {log_dir}/3_output_{server_type}_{server_id}.log &", + ] + start_server_cmds.append(" ".join(cmd)) + + # Generate start server commands (use script_dir for start_server.sh) + server_env = build_server_environment(env_config, benchmark_config['mode']) + export_str = format_export_string(server_env) + + cmd = [ + "srun -l", + f"--nodelist {disagg_server_hostname}", + f"--container-name={container_name}", + f"--export=\"{export_str}\"", + f"--container-image={env_config['container_image']}", + f"--container-mounts={container_mount_str}", + f"--no-container-mount-home --mpi=pmix --overlap -N 1 -n 1", + f"bash {os.path.join(script_dir, 'start_server.sh')} {os.path.join(log_dir, 'server_config.yaml')}", + f"&> {log_dir}/4_output_server.log &", + ] + start_server_cmds.append(" ".join(cmd)) + + # Read env_config after worker/server env build so env_vars.json includes runtime-added vars + save_env_file( + os.path.join(log_dir, "env_vars.json"), + env_config.get('server_env_var', ''), + env_config.get('worker_env_var', ''), + env_config.get('ctx_worker_env_var', ''), + env_config.get('gen_worker_env_var', ''), + ) + + # Generate wait server command (use script_dir for wait_server.sh) + cmd = [ + "srun -l", + f"--container-name={container_name}", + f"--container-mounts={container_mount_str}", + f"--mpi=pmix --overlap -N 1 -n 1", + f"bash {os.path.join(script_dir, 'wait_server.sh')} {disagg_server_hostname} {disagg_server_port}", + f"&> {log_dir}/5_wait_server.log", + ] + start_server_cmds.append(" ".join(cmd)) + + with open(os.path.join(log_dir, "start_server_cmds_base.sh"), "w") as f: + f.write("\n".join(start_server_cmds) + "\n") + + # Generate client commands (use script_dir for benchmark scripts) + client_cmds = [] + client_slurm_prefix = [ + f"srun -l --container-name={container_name}", + f"--container-mounts={container_mount_str}", + f"--mpi=pmix --overlap -N 1 -n 1", + ] + # Append benchmark commands + if benchmark_config.get('enable_benchmark', True): + env_var = config['benchmark'].get('env_var', {}) + benchmark_prefix = client_slurm_prefix + [ + f"--export \"{convert_envs_to_str(env_var)}\"" + ] + if benchmark_config.get('use_aiperf', False): + benchmark_cmd = [ + f"bash {os.path.join(script_dir, 'run_benchmark_aiperf.sh')}", + f"'{env_config['model_path']}' '{benchmark_config['dataset_file']}' {benchmark_config['multi_round']} {gen_num} '{benchmark_config['concurrency_list']}' {benchmark_config['streaming']} '{log_dir}' {disagg_server_hostname} {disagg_server_port} {ucx_warmup_requests}", + f"&> {log_dir}/6_bench.log" + ] + client_cmds.append(" ".join(benchmark_prefix + benchmark_cmd)) + elif benchmark_config['use_nv_sa_benchmark']: + if benchmark_config['mode'] == "gen_only": + print( + f"[ERROR] SA benchmark client script is not supported for gen_only mode" + ) + sys.exit(1) + benchmark_cmd = [ + f"bash {os.path.join(script_dir, 'run_benchmark_nv_sa.sh')}", + f"'{env_config['model_path']}' {isl} {osl} {benchmark_config['benchmark_ratio']} {benchmark_config['multi_round']} {gen_num} '{benchmark_config['concurrency_list']}' {benchmark_config['streaming']} '{log_dir}' {disagg_server_hostname} {disagg_server_port} {ucx_warmup_requests}", + f"&> {log_dir}/6_bench.log" + ] + client_cmds.append(" ".join(benchmark_prefix + benchmark_cmd)) + else: + benchmark_cmd = [ + f"bash {os.path.join(script_dir, 'run_benchmark.sh')}", + f"'{env_config['model_path']}' '{benchmark_config['dataset_file']}' {benchmark_config['multi_round']} {gen_num} '{benchmark_config['concurrency_list']}' {benchmark_config['streaming']} '{log_dir}' {disagg_server_hostname} {disagg_server_port} {ucx_warmup_requests}", + f"&> {log_dir}/6_bench.log" + ] + client_cmds.append(" ".join(benchmark_prefix + benchmark_cmd)) + + # Append accuracy test commands + if config['accuracy']['enable_accuracy_test']: + env_var = config['accuracy'].get('env_var', {}) + accuracy_prefix = client_slurm_prefix + [ + f"--export \"{convert_envs_to_str(env_var)}\"" + ] + for task in config['accuracy']['tasks']: + extra_kwargs = config['accuracy']['tasks'][task].get( + 'extra_kwargs', {}) + extra_kwargs_str = "" + for key, value in extra_kwargs.items(): + if isinstance(value, bool): + if value: + extra_kwargs_str += f" --{key}" + elif key == "custom_config": + extra_kwargs_str += f" --include_path={replace_env_in_file(log_dir, value, env_var)}" + else: + extra_kwargs_str += f" --{key}='{value}'" + end_point_map = { + 'local-completions': 'v1/completions', + 'local-chat-completions': 'v1/chat/completions', + } + model = config['accuracy']['tasks'][task]['model'] + accuracy_cmd = [ + 'lm_eval', '--model', model, '--tasks', task, '--model_args', + f"model={env_config['model_path']},base_url=http://{disagg_server_hostname}:{disagg_server_port}/{end_point_map[model]},{config['accuracy']['tasks'][task]['model_args_extra']}", + '--log_samples', '--output_path', + f'{log_dir}/accuracy_eval_{task}', extra_kwargs_str, + f"&> {log_dir}/7_accuracy_eval_{task}.log" + ] + client_cmds.append(" ".join(accuracy_prefix + accuracy_cmd)) + + # record ${SLURM_JOB_NODELIST} to ${log_dir}/8_done_job_id.txt + done_cmd = [ + "echo", "${SLURM_JOB_NODELIST}", ">", + f"{log_dir}/8_done_${{SLURM_JOB_ID}}.txt" + ] + client_cmds.append(" ".join(done_cmd)) + + with open(os.path.join(log_dir, "client_cmds_base.sh"), "w") as f: + f.write("\n".join(client_cmds) + "\n") + + # Resolve slurm script_file path + # If it's a relative path, make it relative to script_dir + slurm_script_file = slurm_config['script_file'] + if not os.path.isabs(slurm_script_file): + slurm_script_file = os.path.join(script_dir, slurm_script_file) + + # Verify the script file exists + if not os.path.exists(slurm_script_file): + print(f"[ERROR] SLURM script file not found: {slurm_script_file}", + file=sys.stderr) + sys.exit(1) + + # Prepare sbatch command + # yapf: disable + cmd = [ + 'sbatch', + *(['--wait'] if wait else []), + f'--partition={slurm_config["partition"]}', + f'--account={slurm_config["account"]}', + f'--time={slurm_config["job_time"]}', + f'--job-name={slurm_config["job_name"]}', + f'--nodes={total_nodes}', + f'--ntasks={total_tasks}', + f'--ntasks-per-node={hw_config["gpus_per_node"]}', + *([] if not slurm_config['set_segment'] + else [f'--segment={total_nodes}']), + f'--output={log_dir}/slurm-%j.out', + f'--error={log_dir}/slurm-%j.err', + *([arg for arg in slurm_config['extra_args'].split() if arg]), + slurm_script_file, + + # Benchmark Configuration + '--benchmark-mode', benchmark_config['mode'], + + # Environment and paths + '--trtllm-repo', env_config['trtllm_repo'], + '--work-dir', script_dir, + '--full-logdir', log_dir, + '--container-name', container_name, + '--container-mount', container_mount_str, + '--container-image', env_config['container_image'], + '--build-wheel', str(env_config['build_wheel']).lower(), + '--cuda-architectures', env_config['cuda_architectures'], + '--trtllm-wheel-path', env_config['trtllm_wheel_path'], + '--dataset-file', benchmark_config['dataset_file'], + '--dataset-generator', dataset_generator, + '--model-path', env_config['model_path'], + '--dataset-input-length', str(benchmark_config['input_length']), + '--dataset-output-length', str(benchmark_config['output_length']), + ] + # yapf: enable + + if dry_run: + print( + "[WARNING] Dry run mode, will not submit the job. This should be used for test purpose only." + ) + print("sbatch command:") + print(" ".join(cmd)) + return + else: + # Submit the job + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + print(f"Error submitting job: {e}", file=sys.stderr) + sys.exit(1) + + +def main(): + args = parse_args() + + # Determine which mode to use + if args.config: + # Single config file mode + config_files = [args.config] + else: + # Directory mode - find all YAML files + yaml_pattern = os.path.join(args.dir, '*.yaml') + config_files = sorted(glob.glob(yaml_pattern)) + + if not config_files: + print(f"No YAML files found in directory: {args.dir}", + file=sys.stderr) + sys.exit(1) + + print(f"Found {len(config_files)} YAML file(s) in {args.dir}") + + # Process each config file + failures = [] + for config_file in config_files: + print(f"Processing: {config_file}") + try: + config = load_config(config_file) + submit_job(config, args.log_dir, args.dry_run, args.wait) + print(f"Successfully submitted job for: {config_file}\n") + except Exception as e: + traceback.print_exc() + print(f"Error processing {config_file}: {e}", file=sys.stderr) + failures.append(config_file) + if failures: + print(f"Failed configs: {', '.join(failures)}", file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/wait_server.sh b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/wait_server.sh new file mode 100755 index 0000000000..2c6049a1da --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/wait_server.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -euo pipefail + +# Parse arguments +hostname=$1 +port=$2 + +# Constants for health check +readonly TIMEOUT=1800 # 30 minutes +readonly HEALTH_CHECK_INTERVAL=10 +readonly STATUS_UPDATE_INTERVAL=30 + + +# Wait for server to be healthy +echo "Waiting for server ${hostname}:${port} to be healthy..." +start_time=$(date +%s) +while ! curl -s -o /dev/null -w "%{http_code}" "http://${hostname}:${port}/health" > /dev/null 2>&1; do + current_time=$(date +%s) + elapsed=$((current_time - start_time)) + + if [ $elapsed -ge $TIMEOUT ]; then + echo "Error: Server not healthy after ${TIMEOUT} seconds" + exit 1 + fi + + if [ $((elapsed % STATUS_UPDATE_INTERVAL)) -eq 0 ] && [ $elapsed -gt 0 ]; then + echo "Waiting for server to be healthy... (${elapsed}s elapsed)" + fi + + sleep $HEALTH_CHECK_INTERVAL +done + +echo "Server is healthy and ready to accept requests!" diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/submit_all.sh b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/submit_all.sh new file mode 100755 index 0000000000..458f445886 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/submit_all.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Submit every gen-only config in configs/ as a separate SLURM job. +# configs/ holds the per-case trtllm_config.yaml (the Pareto MTP points copied from the cmh +# reference run, with EPLB stripped / perfect-router + ratio-1 applied). +# +# Usage: +# bash submit_all.sh # submit all configs in configs/ +# bash submit_all.sh --dry-run # print sbatch commands without submitting +# bash submit_all.sh configs/ctx1_gen1_dep32_concurrency128_mtp3.yaml # submit a single config +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SUBMIT="$HERE/scripts/submit.py" +CONFIG_DIR="$HERE/configs" + +DRY="" +TARGET="$CONFIG_DIR" +for arg in "$@"; do + case "$arg" in + --dry-run) DRY="--dry-run" ;; + *.yaml) TARGET="$arg" ;; + esac +done + +if [[ ! -d "$CONFIG_DIR" ]] || ! ls "$CONFIG_DIR"/*.yaml >/dev/null 2>&1; then + echo "ERROR: no configs in $CONFIG_DIR. Copy the per-case trtllm_config.yaml there first." >&2 + exit 1 +fi + +if [[ "$TARGET" == *.yaml ]]; then + echo "Submitting single config: $TARGET" + python3 "$SUBMIT" -c "$TARGET" $DRY +else + echo "Submitting all configs in: $CONFIG_DIR" + python3 "$SUBMIT" -d "$CONFIG_DIR" $DRY +fi diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/dataset/gen_dataset.sh b/benchmarks/multi_node/offline/trtllm_gen_only/dataset/gen_dataset.sh new file mode 100755 index 0000000000..78de54674b --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/dataset/gen_dataset.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Generate the random synthetic dataset for the DeepSeek-V4-Pro gen-only SOL sweep. +# isl=8192 osl=1024 random_ratio=1 num_prompts=16384 tokenizer=deepseek_v4 +# random_ratio=1 pins every prompt to exactly ISL/OSL (no length variance), for a clean +# fixed-length gen SOL. (The reference sweep_config.yaml used random_ratio=0.8, num_samples=200000.) +# num_prompts must be >= the largest benchmarked concurrency (8192 here); 16384 gives warmup margin. +# +# IMPORTANT: --custom_tokenizer deepseek_v4 imports tensorrt_llm.tokenizer.deepseek_v4, so this +# must run in an environment that has TensorRT-LLM installed (e.g. inside the trtllm container, +# or a venv/conda env with the trtllm wheel). transformers/numpy/tqdm are also required. +# +# Usage: +# MODEL_PATH=/path/to/DeepSeek-V4-Pro OUT_DIR=/path/to/dataset bash gen_dataset.sh +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ---- Parameters (override via env) ---- +MODEL_PATH="${MODEL_PATH:-/DeepSeek-V4-Pro}" # tokenizer/model dir (fp4 checkpoint) +OUT_DIR="${OUT_DIR:-$HERE/data}" +ISL="${ISL:-8192}" +OSL="${OSL:-1024}" +NUM_PROMPTS="${NUM_PROMPTS:-16384}" # must be >= max benchmarked concurrency (8192) +RANDOM_RATIO="${RANDOM_RATIO:-1}" +NUM_WORKERS="${NUM_WORKERS:-100}" + +if [[ "$MODEL_PATH" == *""* ]]; then + echo "ERROR: set MODEL_PATH to your DeepSeek-V4-Pro checkpoint directory." >&2 + exit 1 +fi + +mkdir -p "$OUT_DIR" +# Filename pattern: DeepSeek-V4----ratio-_for_serve.json +OUT_PREFIX="$OUT_DIR/DeepSeek-V4-${ISL}-${OSL}-${NUM_PROMPTS}-ratio-${RANDOM_RATIO}" + +echo "Generating dataset -> ${OUT_PREFIX}_for_serve.json" +python3 "$HERE/random_generator.py" \ + --num_prompts "$NUM_PROMPTS" \ + --num_tokens "$ISL" \ + --max_tokens "$OSL" \ + --random_ratio "$RANDOM_RATIO" \ + --tokenizer_name "$MODEL_PATH" \ + --custom_tokenizer deepseek_v4 \ + --output_file_serve "${OUT_PREFIX}_for_serve.json" \ + --num_workers "$NUM_WORKERS" \ + --use_parallel + +echo +echo "Done. Ensure 'dataset_file' in the benchmark/configs/*.yaml points here:" +echo " ${OUT_PREFIX}_for_serve.json" diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/dataset/random_generator.py b/benchmarks/multi_node/offline/trtllm_gen_only/dataset/random_generator.py new file mode 100644 index 0000000000..2b33eae404 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/dataset/random_generator.py @@ -0,0 +1,479 @@ +import random +import json +from typing import List, Optional, Tuple, Dict +from pathlib import Path +import numpy as np +import tqdm +import logging +import argparse +from transformers import AutoTokenizer, AutoConfig +from multiprocessing import Pool, cpu_count +import time + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def _build_chat_template_kwargs(custom_tokenizer): + """Build apply_chat_template kwargs. deepseek_v4 always runs in Think Max mode.""" + if custom_tokenizer == "deepseek_v4": + return {"enable_thinking": True, "reasoning_effort": "max"} + return {} + + +random.seed(42) +np.random.seed(42) + + +def get_chat_template_token_number(tokenizer, chat_template_kwargs=None): + chat_template_kwargs = chat_template_kwargs or {} + message = tokenizer.apply_chat_template( + [{"role": "user", "content": "a"}], + add_generation_prompt=True, + tokenize=False, + **chat_template_kwargs, + ) + encode_ids = tokenizer.encode(message, add_special_tokens=False) + return len(encode_ids) - 1 + + +def _create_random_text_and_tokens(tokenizer, vocab_size, input_len, output_len, num_prompts, random_ratio, chat_template_kwargs=None): + """Helper function to create random text and calculate token count.""" + chat_template_kwargs = chat_template_kwargs or {} + chat_template_token_number = get_chat_template_token_number(tokenizer, chat_template_kwargs) + print(f"chat_template_token_number: {chat_template_token_number}") + input_len = input_len - chat_template_token_number + + input_lens = np.random.randint( + int(input_len * random_ratio) if input_len > 1 else 1, + input_len + 1, + size=num_prompts, + ) + output_lens = np.random.randint( + int(output_len * random_ratio) if output_len > 1 else 1, + output_len + 1, + size=num_prompts, + ) + offsets = np.random.randint(0, vocab_size, size=num_prompts) + + prompts = [] + + # origin_ids = [] + # for i in range(num_prompts): + # origin_id = [(offsets[i] + i + j) % + # tokenizer.vocab_size for j in range(int(input_lens[i]*1.5))] + # origin_ids.append(origin_id) + # print(f"length of origin_ids: {len(origin_ids)}") + + # origin_texts = tokenizer.batch_decode(origin_ids) + # print(f"length of origin_texts: {len(origin_texts)}") + # re_encoded_sequences = tokenizer.batch_encode_plus( + # origin_texts, add_special_tokens=False) + # print(f"length of re_encoded_sequences: {len(re_encoded_sequences)}") + # print(f"length of input_ids: {len(re_encoded_sequences['input_ids'])}") + + # re_encoded_ids = [] + # for i in range(num_prompts): + # re_encoded_ids.append( + # # 使用 'input_ids' 键 + # re_encoded_sequences['input_ids'][i][:input_lens[i]]) + # print(f"length of re_encoded_ids: {len(re_encoded_ids)}") + + # re_encoded_texts = tokenizer.batch_decode(re_encoded_ids) + # print(f"length of re_encoded_texts: {len(re_encoded_texts)}") + + # for i in range(num_prompts): + # prompt = tokenizer.apply_chat_template( + # [{"role": "user", "content": re_encoded_texts[i]}], + # add_generation_prompt=True, + # tokenize=False, + # ) + # input_lens[i] += chat_template_token_number + # prompts.append(prompt) + + for i in range(num_prompts): + origin_text = tokenizer.decode([(offsets[i] + i + j) % + vocab_size for j in range(int(input_lens[i]*1.5))]) + re_encoded_sequence = tokenizer.encode(origin_text, add_special_tokens=False)[ + :(input_lens[i]) + ] + prompt_text = tokenizer.decode(re_encoded_sequence) + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt_text}], + add_generation_prompt=True, + tokenize=False, + **chat_template_kwargs, + ) + input_lens[i] += chat_template_token_number + prompts.append(prompt) + + return prompts, input_lens, output_lens + + +def _random_generate_worker(args): + """Worker function for parallel processing""" + (worker_id, num_prompts, num_tokens, max_tokens, tokenizer_name, vocab_size, + random_ratio, custom_tokenizer) = args + + # Initialize tokenizer in worker process + if custom_tokenizer and custom_tokenizer == "deepseek_v32": + from tensorrt_llm.tokenizer.deepseek_v32 import DeepseekV32Tokenizer + tokenizer = DeepseekV32Tokenizer.from_pretrained( + tokenizer_name, trust_remote_code=True) + elif custom_tokenizer and custom_tokenizer == "deepseek_v4": + from tensorrt_llm.tokenizer.deepseek_v4 import DeepseekV4Tokenizer + tokenizer = DeepseekV4Tokenizer.from_pretrained( + tokenizer_name, trust_remote_code=True) + else: + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_name, trust_remote_code=True) + + chat_template_kwargs = _build_chat_template_kwargs(custom_tokenizer) + prompts, input_lens, output_lens = _create_random_text_and_tokens( + tokenizer, vocab_size, num_tokens, max_tokens, num_prompts, random_ratio, + chat_template_kwargs) + results = [] + for i in range(num_prompts): + prompt = prompts[i] + input_len = input_lens[i] + output_len = output_lens[i] + prompt_data = { + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": prompt}, + ], + "num_tokens": int(input_len), # Convert numpy.int64 to Python int + "expected_tokens": num_tokens, # Record expected token count + "max_tokens": int(output_len), # Convert numpy.int64 to Python int + } + results.append(prompt_data) + return results + + +class RandomPromptGenerator: + """ + Random prompt generator using random token IDs + Based on the structure of OptimizedPromptGenerator + """ + + def __init__(self, tokenizer_name: str = "deepseek-ai/DeepSeek-R1", custom_tokenizer: str = None, vocab_from_config: bool = False): + """Initialize with tokenizer""" + logger.info(f"Loading tokenizer: {tokenizer_name}") + if custom_tokenizer and custom_tokenizer == "deepseek_v32": + from tensorrt_llm.tokenizer.deepseek_v32 import DeepseekV32Tokenizer + self.tokenizer = DeepseekV32Tokenizer.from_pretrained( + tokenizer_name, trust_remote_code=True) + elif custom_tokenizer and custom_tokenizer == "deepseek_v4": + from tensorrt_llm.tokenizer.deepseek_v4 import DeepseekV4Tokenizer + self.tokenizer = DeepseekV4Tokenizer.from_pretrained( + tokenizer_name, trust_remote_code=True) + else: + self.tokenizer = AutoTokenizer.from_pretrained( + tokenizer_name, trust_remote_code=True) + if vocab_from_config: + self.config = AutoConfig.from_pretrained( + tokenizer_name, trust_remote_code=True) + self.vocab_size = self.config.vocab_size + else: + try: + self.vocab_size = self.tokenizer.vocab_size + except (AttributeError, NotImplementedError): + # Custom tokenizers wrap an inner HF tokenizer + self.vocab_size = self.tokenizer.tokenizer.vocab_size + self.tokenizer_name = tokenizer_name + self.custom_tokenizer = custom_tokenizer + self.chat_template_kwargs = _build_chat_template_kwargs(custom_tokenizer) + logger.info(f"Tokenizer loaded. Vocabulary size: {self.vocab_size}") + + def generate_prompts_batch( + self, + num_prompts: int, + num_tokens: int = 1024, + max_tokens: int = 1024, + random_ratio: float = 1.0 + ) -> List[Dict]: + """Generate multiple random prompts in batch (single process)""" + logger.info( + f"Generating {num_prompts} random prompts with target {num_tokens} tokens each (single process)") + + # 修改变量名，避免与函数参数冲突 + prompt_texts, input_lens, output_lens = _create_random_text_and_tokens( + self.tokenizer, self.vocab_size, num_tokens, max_tokens, num_prompts, random_ratio, + self.chat_template_kwargs) + + results = [] # 使用 results 而不是 prompts + for i in tqdm.tqdm(range(num_prompts), desc="Generating random prompts"): + prompt_text = prompt_texts[i] # 使用 prompt_texts + input_len = input_lens[i] + output_len = output_lens[i] + + prompt_data = { + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": prompt_text}, # 使用 prompt_text + ], + "num_tokens": int(input_len), + "expected_tokens": num_tokens, + "max_tokens": int(output_len), + } + results.append(prompt_data) # 使用 results + return results + + def generate_prompts_parallel( + self, + num_prompts: int, + num_tokens: int = 1024, + max_tokens: int = 1024, + num_workers: Optional[int] = None, + random_ratio: float = 1.0 + ) -> List[Dict]: + """ + Generate multiple random prompts using parallel processing + """ + if num_workers is None: + num_workers = min(cpu_count(), 8) # Limit to 8 workers max + + logger.info( + f"Generating {num_prompts} random prompts with target {num_tokens} tokens each") + logger.info(f"Using {num_workers} workers for parallel processing") + + # Prepare work distribution + prompts_per_worker = num_prompts // num_workers + remaining = num_prompts % num_workers + + worker_args = [] + for i in range(num_workers): + num_prompts_for_worker = prompts_per_worker + \ + (1 if i < remaining else 0) + worker_args.append( + (i, num_prompts_for_worker, num_tokens, max_tokens, self.tokenizer_name, + self.vocab_size, random_ratio, self.custom_tokenizer)) + + # Execute parallel processing + all_prompts = [] + with Pool(num_workers) as pool: + results = list(tqdm.tqdm( + pool.imap(_random_generate_worker, worker_args), + total=num_workers, + desc="Generating prompts", + unit="worker", + postfix={"total_prompts": num_prompts} + )) + + for worker_results in results: + all_prompts.extend(worker_results) + return all_prompts + + def dump_to_file( + self, + prompts: List[Dict], + output_file: str, + format_type: str = "serve" + ): + """ + Dump prompts to file in specified format + Similar to the dump methods in prompt_generator.py + """ + logger.info( + f"Writing {len(prompts)} prompts to {output_file} ({format_type} format)") + + if format_type == "serve": + self._dump_serve_format(prompts, output_file) + elif format_type == "bench": + self._dump_bench_format(prompts, output_file) + else: + raise ValueError(f"Unsupported format: {format_type}") + + def _dump_serve_format(self, prompts: List[Dict], output_file: str): + """Dump in serve format (optimized batch writing)""" + # Build all line content in batch, then write at once + lines = [] + for prompt in prompts: + lines.append(json.dumps({"input": prompt}) + "\n") + + # Write all content at once + with open(output_file, "w", encoding="utf-8", buffering=8192) as outfile: + outfile.writelines(lines) + + def _dump_bench_format(self, prompts: List[Dict], output_file: str): + """Dump in bench format (optimized batch writing)""" + # Build all line content in batch + lines = [] + for idx, prompt in enumerate(prompts): + user_content = "" + for msg in prompt["messages"]: + if msg["role"] == "user": + user_content = msg["content"] + break + + bench_item = { + "task_id": idx, + "prompt": user_content, + # Or consider adding max_tokens parameter + "output_tokens": prompt.get("max_tokens", 2048) + } + lines.append(json.dumps( + bench_item, ensure_ascii=False, separators=(',', ':'))) + + # Write all content at once, keeping consistent line breaks with prompt_generator + with open(output_file, "w", encoding="utf-8") as outfile: + outfile.write('\n'.join(lines)) + if lines: + outfile.write('\n') + + def verify(self, prompts: List[Dict], target_tokens: int) -> Dict[str, float]: + """Verify the generated prompts (similar to verify method in prompt_generator.py)""" + actual_token_lengths = [prompt["num_tokens"] for prompt in prompts] + expected_token_lengths = [prompt.get( + "expected_tokens", target_tokens) for prompt in prompts] + + # Calculate actual token length statistics + actual_stats = { + "mean": float(np.mean(actual_token_lengths)), + "std": float(np.std(actual_token_lengths)), + "min": float(np.min(actual_token_lengths)), + "max": float(np.max(actual_token_lengths)), + "count": len(actual_token_lengths), + } + + # Calculate differences from target token length + differences = [actual - expected for actual, + expected in zip(actual_token_lengths, expected_token_lengths)] + difference_stats = { + "mean_diff": float(np.mean(differences)), + "std_diff": float(np.std(differences)), + "min_diff": float(np.min(differences)), + "max_diff": float(np.max(differences)), + } + + logger.info(f"Generated {actual_stats['count']} random prompts") + logger.info(" Actual token length stats:") + logger.info( + f" Mean: {actual_stats['mean']:.2f} ± {actual_stats['std']:.2f}") + logger.info( + f" Range: {actual_stats['min']:.0f} - {actual_stats['max']:.0f}") + logger.info(f" Token length differences (actual - expected):") + logger.info( + f" Mean: {difference_stats['mean_diff']:.2f} ± {difference_stats['std_diff']:.2f}") + logger.info( + f" Range: {difference_stats['min_diff']:.0f} - {difference_stats['max_diff']:.0f}") + logger.info(f"Target length: {target_tokens}") + + return {**actual_stats, **difference_stats} + + +def main(): + """Main function with command line interface""" + parser = argparse.ArgumentParser( + description="Generate random prompts using random token IDs.") + parser.add_argument( + "--num_tokens", + type=int, + default=1024, + help="Number of tokens per prompt (default: 1024).", + ) + parser.add_argument( + "--num_prompts", + type=int, + default=100, + help="Number of prompts to generate (default: 100).", + ) + parser.add_argument( + "--tokenizer_name", + type=str, + default="deepseek-ai/DeepSeek-R1", + help="Tokenizer name or path from HuggingFace.", + ) + parser.add_argument( + "--output_file_serve", + type=str, + default=None, + help="Path to output file in serve format." + ) + parser.add_argument( + "--output_file_bench", + type=str, + default=None, + help="Path to output file in bench format." + ) + parser.add_argument( + "--max_tokens", + type=int, + default=1024, + help="Maximum number of tokens for each request output (default: 2048).", + ) + parser.add_argument( + "--num_workers", + type=int, + default=None, + help="Number of parallel workers (default: auto-detect, max 8).", + ) + parser.add_argument( + "--use_parallel", + action="store_true", + help="Use parallel processing (default: False, use single process).", + ) + # add random ratio + parser.add_argument( + "--random_ratio", + type=float, + default=1.0, + help="Random ratio for input and output tokens (default: 1.0).", + ) + parser.add_argument( + "--custom_tokenizer", + type=str, + default=None, + help="Custom tokenizer name supported by trtllm.", + choices=["deepseek_v32", "deepseek_v4"], + ) + parser.add_argument( + "--vocab_from_config", + action="store_true", + help="Use the vocab size from the config. If not provided, will use the vocab size from the tokenizer.", + ) + + args = parser.parse_args() + + # Initialize generator + generator = RandomPromptGenerator( + tokenizer_name=args.tokenizer_name, custom_tokenizer=args.custom_tokenizer, + vocab_from_config=args.vocab_from_config) + + start_time = time.time() + + # Generate prompts + if args.use_parallel: + prompts = generator.generate_prompts_parallel( + num_prompts=args.num_prompts, + num_tokens=args.num_tokens, + max_tokens=args.max_tokens, + num_workers=args.num_workers, + random_ratio=args.random_ratio + ) + else: + prompts = generator.generate_prompts_batch( + num_prompts=args.num_prompts, + num_tokens=args.num_tokens, + max_tokens=args.max_tokens, + random_ratio=args.random_ratio + ) + + # Write to files if specified + if args.output_file_serve: + generator.dump_to_file(prompts, args.output_file_serve, "serve") + + if args.output_file_bench: + generator.dump_to_file(prompts, args.output_file_bench, "bench") + + # Verify results + stats = generator.verify(prompts, args.num_tokens) + + end_time = time.time() + logger.info(f"Total execution time: {end_time - start_time:.2f} seconds") + logger.info( + f"Average time per prompt: {(end_time - start_time) / args.num_prompts:.4f} seconds") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/process/get_gen_only_perf.py b/benchmarks/multi_node/offline/trtllm_gen_only/process/get_gen_only_perf.py new file mode 100755 index 0000000000..c5b6677a3a --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/process/get_gen_only_perf.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +""" +Extract gen-only performance metrics from trtllm iterlog files (no ctx.json needed). + +Reports: tps/user, output_tps/gen_gpu, output_tput, avg_itertime + +Scans /*/concurrency_*/gen_only*.txt. For the tekit disagg benchmark that means +pointing -i at the ISL-OSL directory that contains the per-config run dirs, e.g.: + //8192-1024/ +whose children look like disagg_ctx1_gen1_dep16_batch128_eplb384_mtp3/concurrency_256/gen_only_0.txt + +tps/user is corrected by the MTP acceptance rate: tps_per_user = accept_rate / avg_itertime. +The acceptance rate defaults to the DeepSeek-V4-Pro MTP rates {1: 1.7, 3: 2.44}; override with +--accept-rate. + +Usage: + python get_gen_only_perf.py -i + python get_gen_only_perf.py -i --accept-rate "1:1.7,3:2.44" +""" + +import argparse +import glob +import os +import re +import pandas as pd + +# Default DeepSeek-V4-Pro MTP acceptance rates (accepted tokens per gen iteration). +# tps/user = accept_rate / avg_itertime. Override with --accept-rate. +DEFAULT_ACCEPT_RATE = {1: 1.7, 3: 2.44} + + +def process_gen_iterlog(dir_prefix, accept_rate=None): + """Process gen-only iterlog files and extract performance metrics.""" + if accept_rate is None: + accept_rate = {} + + # Recursive so -i can point at the base log dir (which holds one subtree per config), a single + # config dir, or an older nested layout. + pattern = os.path.join(dir_prefix, "**", "concurrency_*", "gen_only*.txt") + files = glob.glob(pattern, recursive=True) + if not files: + print(f"No files found matching {pattern}") + return [] + + print(f"Found {len(files)} iterlog files") + summary = [] + + for file in sorted(files): + # Concurrency comes from the concurrency_/ subdir, which is authoritative for both + # the tekit layout (.../disagg_ctx1_gen1_dep16_batch128_eplb384_mtp3/concurrency_256/) + # and the internal layout (.../ctx1_gen1_dep16_concurrency256_eplb384_mtp3/concurrency_256/). + # tp/dp-mode/eplb/mtp are read individually so we don't depend on the exact token order. + tp_match = re.search(r'(tep|dep)(\d+)', file) + conc_match = re.search(r'concurrency_(\d+)', file) + if not tp_match or not conc_match: + continue + + dp_tep = tp_match.group(1) + rank_num = int(tp_match.group(2)) + concurrency = int(conc_match.group(1)) + eplb_match = re.search(r'eplb(\d+)', file) + eplb_num = int(eplb_match.group(1)) if eplb_match else 0 + mtp_match = re.search(r'_mtp(\d+)', file) + mtp_num = int(mtp_match.group(1)) if mtp_match else 0 + + # Extract ctx/gen instance counts from the config dir name + config_dir = file.rsplit('/', 2)[0] + config_name = os.path.basename(config_dir) + ctx_match = re.search(r'ctx(\d+)_gen(\d+)', config_name) + ctx_inst = int(ctx_match.group(1)) if ctx_match else 1 + gen_inst = int(ctx_match.group(2)) if ctx_match else 1 + + # Parse iterlog + try: + with open(file, 'r', encoding='utf-8', errors='ignore') as f: + content = f.read() + except Exception as e: + print(f"Error reading {file}: {e}") + continue + + log_pattern = ( + r'iter = (\d+), global_rank = (\d+), rank = (\d+), ' + r'currank_total_requests = (\d+)/(\d+), ' + r'host_step_time = ([\d.]+)ms, prev_device_step_time = ([\d.]+)ms, ' + r'timestamp = ([^,]+), num_scheduled_requests: (\d+), ' + r'states = \{\'num_ctx_requests\': (\d+), \'num_ctx_tokens\': (\d+), ' + r'\'num_generation_tokens\': (\d+)\}' + ) + matches = re.findall(log_pattern, content) + if not matches: + continue + + rows = [] + for m in matches: + rows.append({ + 'iter': int(m[0]), + 'global_rank': int(m[1]), + 'elapsed_time': float(m[6]) / 1000, # prev_device_step_time + 'num_scheduled_requests': int(m[8]), + 'num_ctx_tokens': int(m[10]), + 'num_generation_tokens': int(m[11]), + }) + + df = pd.DataFrame(rows) + + # Filter: gen-only iters (no ctx tokens) + df = df[df['num_ctx_tokens'] == 0] + df = df.groupby(['iter', 'global_rank']).last().reset_index() + + # Trim warmup/cooldown + df = df.iloc[50:-10] + + # Filter to steady-state iters + if dp_tep == 'tep': + df = df[df['num_scheduled_requests'] == concurrency] + df = df[df['num_generation_tokens'] == concurrency * (mtp_num + 1)] + else: # dep + per_rank = concurrency // rank_num + df = df[df['num_scheduled_requests'] == per_rank] + df = df[df['num_generation_tokens'] == per_rank * (mtp_num + 1)] + + if df.empty: + continue + + # Filter outliers (median ±20%) + median_t = df['elapsed_time'].median() + df = df[(df['elapsed_time'] >= median_t * 0.8) & + (df['elapsed_time'] <= median_t * 1.2)] + if df.empty: + continue + + # Calculate metrics + avg_itertime = df['elapsed_time'].mean() + ar = accept_rate.get(mtp_num, 1.0) if mtp_num > 0 else 1.0 + tps_per_user = ar / avg_itertime if avg_itertime > 0 else 0 + output_tput = tps_per_user * concurrency + output_tps_per_gen_gpu = output_tput / rank_num + + name = f"ctx{ctx_inst}_gen{gen_inst}_{dp_tep}{rank_num}_c{concurrency}_mtp{mtp_num}" + summary.append({ + 'config': name, + 'concurrency': concurrency, + 'mtp': mtp_num, + 'tp': rank_num, + 'adp': dp_tep == 'dep', + 'eplb': eplb_num, + 'tps_per_user': round(tps_per_user, 2), + 'output_tps_per_gen_gpu': round(output_tps_per_gen_gpu, 2), + 'output_tput': round(output_tput, 2), + 'avg_itertime_ms': round(avg_itertime * 1000, 3), + 'num_iters': len(df), + }) + + return summary + + +def main(): + parser = argparse.ArgumentParser( + description='Extract gen-only performance metrics from iterlog files (no ctx.json needed)') + parser.add_argument('-i', '--input-dir', required=True, + help='Gen-only benchmark directory') + parser.add_argument('--accept-rate', type=str, default=None, + help='MTP accept rate as "mtp:rate,..." (default: "1:1.7,3:2.44")') + parser.add_argument('-o', '--output', type=str, default=None, + help='Output CSV file (default: /gen_only_perf.csv)') + args = parser.parse_args() + + if not os.path.isdir(args.input_dir): + print(f"Error: {args.input_dir} does not exist") + return + + # Parse accept rate (default to the DeepSeek-V4-Pro MTP rates if not given) + if args.accept_rate: + accept_rate = {} + for pair in args.accept_rate.split(','): + k, v = pair.split(':') + accept_rate[int(k)] = float(v) + else: + accept_rate = dict(DEFAULT_ACCEPT_RATE) + print(f"Using MTP accept rates: {accept_rate}") + + summary = process_gen_iterlog(args.input_dir, accept_rate) + if not summary: + print("No data extracted") + return + + df = pd.DataFrame(summary) + df = df.sort_values('tps_per_user', ascending=False) + + # Save CSV + output_file = args.output or os.path.join(args.input_dir, 'gen_only_perf.csv') + df.to_csv(output_file, index=False) + print(f"\nSaved to {output_file}") + + # Print table + print(f"\n{'Config':<55} {'Conc':>6} {'TPS/User':>10} {'OutTPS/GPU':>12} {'OutTput':>12} {'IterTime':>10} {'Iters':>6}") + print("-" * 115) + for _, r in df.iterrows(): + print(f"{r['config']:<55} {r['concurrency']:>6} {r['tps_per_user']:>10.2f} " + f"{r['output_tps_per_gen_gpu']:>12.2f} {r['output_tput']:>12.2f} " + f"{r['avg_itertime_ms']:>9.3f}ms {r['num_iters']:>6}") + + print(f"\nTotal: {len(summary)} configs processed") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/multi_node/offline/trtllm_gen_only/process/process.sh b/benchmarks/multi_node/offline/trtllm_gen_only/process/process.sh new file mode 100755 index 0000000000..01adb8fa58 --- /dev/null +++ b/benchmarks/multi_node/offline/trtllm_gen_only/process/process.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Post-process gen-only benchmark outputs into a per-(config, concurrency) performance table. +# +# Point -i at the base log_dir (env.yaml log_dir) that holds the per-config / trees; the +# tool recurses to find every concurrency_*/gen_only*.txt beneath it. Pointing at a single +# / dir works too. +# +# tps/user is corrected by the MTP acceptance rate. Defaults to the DeepSeek-V4-Pro MTP rates +# (mtp1=1.7, mtp3=2.44); override via ACCEPT_RATE ("mtp:rate,...") if you measure different ones +# (accepted-draft-token stats appear in the gen worker logs / gen_only_*.txt, or a matching E2E run). +# +# Requires: python3 with pandas. +# +# Usage: +# IN= bash process.sh # uses default accept rates +# IN= ACCEPT_RATE="1:1.7,3:2.44" bash process.sh +# bash process.sh -i --accept-rate "1:1.7,3:2.44" +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="$HERE/get_gen_only_perf.py" + +IN="${IN:-}" +ACCEPT_RATE="${ACCEPT_RATE:-}" +EXTRA=() + +# Allow CLI flags to pass straight through to the python tool. +while [[ $# -gt 0 ]]; do + case "$1" in + -i|--input-dir) IN="$2"; shift 2 ;; + --accept-rate) ACCEPT_RATE="$2"; shift 2 ;; + *) EXTRA+=("$1"); shift ;; + esac +done + +if [[ -z "$IN" ]]; then + echo "ERROR: set the input dir with -i (or IN=...)." >&2 + exit 1 +fi + +ARGS=(-i "$IN") +[[ -n "$ACCEPT_RATE" ]] && ARGS+=(--accept-rate "$ACCEPT_RATE") +# ${arr[@]:-} guards against 'unbound variable' for empty arrays under `set -u` (bash 3.2). +python3 "$SCRIPT" "${ARGS[@]}" ${EXTRA[@]:+"${EXTRA[@]}"} diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 5ef6c8ff49..b52ea83051 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -108,6 +108,18 @@ export EVAL_ONLY="${EVAL_ONLY:-false}" export ISL="$ISL" export OSL="$OSL" +# Native TRT-LLM disaggregated generation-worker benchmark. The end-to-end +# workflow exports CONFIG_FILE from the selected master-config row. +# The wrapper renders cluster-local paths and invokes the bundled generation-only harness. +if [[ "$IS_MULTINODE" == "true" && + "$SPEC_DECODING" == "offline" && + "$FRAMEWORK" == "trt" && + "$MODEL_PREFIX" == "dsv4" ]]; then + export TRT_GEN_ONLY_SQUASH_FILE="$SQUASH_FILE" + bash "benchmarks/multi_node/offline/dsv4_fp4_gb300_trt.sh" + exit $? +fi + # --------------------------------------------------------------------------- # Single-node path (multinode: false configs, e.g. the offline decode-step DSV4 # offline bench). Mirrors the launch_gb200-nv.sh single-node branch: diff --git a/utils/bench_offline/test_trt_disagg_gen_only.py b/utils/bench_offline/test_trt_disagg_gen_only.py new file mode 100644 index 0000000000..0d08787aad --- /dev/null +++ b/utils/bench_offline/test_trt_disagg_gen_only.py @@ -0,0 +1,541 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +_THIS_DIR = Path(__file__).resolve().parent +if str(_THIS_DIR) not in sys.path: + sys.path.insert(0, str(_THIS_DIR)) + +from trt_disagg_gen_only import ( + build_result, + build_result_from_logs, + case_metadata, + parse_gen_iterlog, + render_effective_config, + timed_max_tokens, + validate_case_against_workflow, +) + + +REPO_ROOT = Path(__file__).resolve().parents[2] +CONFIG_ROOT = ( + REPO_ROOT + / "benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs" +) +REQUIRED_CONFIGS = { + "ctx1_gen1_dep32_concurrency512_mtp3.yaml", +} +UPSTREAM_RUNNER_ROOT = Path("/data/home/sa-shared/gharunners") +UPSTREAM_DATASET_ROOT = UPSTREAM_RUNNER_ROOT / "datasets/dsv4-trt-offline" +UPSTREAM_CONTAINER_IMAGE = ( + f"{UPSTREAM_RUNNER_ROOT}/squash/" + "nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh" +) + + +def _load_yaml(path: Path) -> dict: + return yaml.safe_load(path.read_text()) + + +def _iterlog_line( + *, + iteration: int, + rank: int, + scheduled: int, + generation_tokens: int, + device_ms: float, + ctx_tokens: int = 0, +) -> str: + return ( + f"iter = {iteration}, global_rank = {rank}, rank = {rank}, " + f"currank_total_requests = {scheduled}/{scheduled}, " + f"host_step_time = {device_ms + 1.0:.3f}ms, " + f"prev_device_step_time = {device_ms:.3f}ms, " + "timestamp = 2026-07-07 00:00:00, " + f"num_scheduled_requests: {scheduled}, " + "states = {'num_ctx_requests': 0, " + f"'num_ctx_tokens': {ctx_tokens}, " + f"'num_generation_tokens': {generation_tokens}}}" + ) + + +def test_offline_token_budgets_use_fixed_acceptance_lengths(): + assert timed_max_tokens(decode_steps=256, mtp=3) == 623 + assert timed_max_tokens(decode_steps=256, mtp=1) == 435 + + +def test_tep_parser_keeps_only_exact_full_batch_generation_rows(tmp_path): + path = tmp_path / "gen_only_0.txt" + path.write_text( + "\n".join( + [ + _iterlog_line( + iteration=1, + rank=0, + scheduled=2, + generation_tokens=8, + device_ms=10.0, + ), + _iterlog_line( + iteration=2, + rank=0, + scheduled=1, + generation_tokens=4, + device_ms=99.0, + ), + _iterlog_line( + iteration=3, + rank=0, + scheduled=2, + generation_tokens=8, + device_ms=12.0, + ctx_tokens=8192, + ), + ] + ) + ) + + summary = parse_gen_iterlog( + path, + concurrency=2, + gen_tp=4, + mtp=3, + attention_dp=False, + ) + + assert summary.raw_exact_batch_samples == 1 + assert summary.retained_samples == 1 + assert summary.mean_device_step_ms == pytest.approx(10.0) + + +def test_dep_parser_uses_per_rank_batch_and_deduplicates_iterations(tmp_path): + path = tmp_path / "gen_only_0.txt" + path.write_text( + "\n".join( + [ + _iterlog_line( + iteration=7, + rank=0, + scheduled=2, + generation_tokens=8, + device_ms=20.0, + ), + _iterlog_line( + iteration=7, + rank=0, + scheduled=2, + generation_tokens=8, + device_ms=22.0, + ), + _iterlog_line( + iteration=8, + rank=0, + scheduled=1, + generation_tokens=4, + device_ms=90.0, + ), + ] + ) + ) + + summary = parse_gen_iterlog( + path, + concurrency=64, + gen_tp=32, + mtp=3, + attention_dp=True, + ) + + assert summary.raw_exact_batch_samples == 1 + assert summary.retained_samples == 1 + assert summary.mean_device_step_ms == pytest.approx(22.0) + + +def test_result_keeps_steps_primary_and_acceptance_secondary(): + result = build_result( + model_id="deepseek-ai/DeepSeek-V4-Pro", + source_config="ctx1_gen1_tep4_concurrency2_mtp3.yaml", + concurrency=2, + ctx_gpus=4, + gen_gpus=4, + gen_tp=4, + mtp=3, + mean_device_step_ms=10.0, + median_device_step_ms=10.0, + p90_device_step_ms=10.0, + p99_device_step_ms=10.0, + raw_samples=256, + retained_samples=250, + decode_steps=256, + ) + + assert result["engine_mode"] == "offline" + assert result["measurement_boundary"] == "gen_iteration" + assert result["tpot_unit"] == "decode_step" + assert result["total_token_throughput"] == pytest.approx(200.0) + assert result["output_throughput"] == pytest.approx(200.0) + assert result["decode_step_throughput_per_gen_gpu"] == pytest.approx(50.0) + assert result["assumed_tokens_per_step"] == pytest.approx(2.44) + assert result["token_equivalent_output_throughput"] == pytest.approx(488.0) + assert result["token_equivalent_output_throughput_per_gen_gpu"] == pytest.approx(122.0) + assert result["timed_max_tokens"] == 623 + + +def test_master_config_contains_twelve_selected_source_cases(tmp_path): + master = _load_yaml(REPO_ROOT / ".github/configs/nvidia-master.yaml") + config = master["dsv4-fp4-gb300-trt-offline"] + + assert config["multinode"] is True + assert config["disagg"] is True + scenario = config["scenarios"]["fixed-seq-len"][0] + assert (scenario["isl"], scenario["osl"]) == (8192, 256) + rows = scenario["search-space"] + assert len(rows) == 12 + assert {row["spec-decoding"] for row in rows} == {"offline"} + + config_files = [] + for row in rows: + setting = row["prefill"]["additional-settings"][0] + key, relative_path = setting.split("=", 1) + assert key == "CONFIG_FILE" + source_path = REPO_ROOT / relative_path + assert source_path.is_file() + config_files.append(source_path) + source = _load_yaml(source_path) + metadata = validate_case_against_workflow( + source, + concurrency=row["conc-list"][0], + prefill_num_workers=row["prefill"]["num-worker"], + prefill_tp=row["prefill"]["tp"], + prefill_ep=row["prefill"]["ep"], + prefill_dp_attn=row["prefill"]["dp-attn"], + decode_num_workers=row["decode"]["num-worker"], + decode_tp=row["decode"]["tp"], + decode_ep=row["decode"]["ep"], + decode_dp_attn=row["decode"]["dp-attn"], + ) + effective, rendered_metadata = render_effective_config( + source, + output_path=tmp_path / source_path.name, + partition="batch_1", + account="benchmark", + job_name="gb300-nv_0", + container_image="/data/squash/trtllm.sqsh", + container_mount="/data/:/data/,/repo:/workspace", + model_path="/scratch/models/DeepSeek-V4-Pro", + dataset_root="/data/datasets/dsv4-trt-offline", + log_dir=f"/data/runs/{source_path.stem}", + decode_steps=scenario["osl"], + ) + assert rendered_metadata == metadata + assert effective["benchmark"]["input_length"] == scenario["isl"] + assert effective["benchmark"]["output_length"] == timed_max_tokens( + scenario["osl"], metadata.mtp + ) + assert effective["hardware"]["num_gen_servers"] == 1 + + assert len(set(config_files)) == 12 + selected_names = {path.name for path in config_files} + assert REQUIRED_CONFIGS <= selected_names + assert {case_metadata(_load_yaml(path)).node_count for path in config_files} == { + 2, + 3, + 8, + 9, + 12, + } + + +def test_case_metadata_is_json_serializable(): + source = _load_yaml( + CONFIG_ROOT / "ctx4_gen1_dep16_concurrency4096_mtp1.yaml" + ) + metadata = case_metadata(source) + encoded = json.dumps(metadata.__dict__, sort_keys=True) + assert '"mtp": 1' in encoded + assert '"node_count": 8' in encoded + + +def test_source_configs_target_upstream_gb300_ci_cluster(): + source_paths = sorted(CONFIG_ROOT.glob("*.yaml")) + assert len(source_paths) == 12 + source_names = {path.name for path in source_paths} + assert REQUIRED_CONFIGS <= source_names + + for source_path in source_paths: + source = _load_yaml(source_path) + metadata = case_metadata(source) + slurm = source["slurm"] + benchmark = source["benchmark"] + environment = source["environment"] + timed_tokens = timed_max_tokens(256, metadata.mtp) + + assert slurm["partition"] == "batch_1" + assert slurm["account"] == "benchmark" + assert slurm["job_time"] == "03:00:00" + assert slurm["job_name"] == f"dsv4-offline-{source_path.stem}" + for dead_field in ( + "use_hetjob", + "max_segment_nodes", + "max_hetjob_components", + ): + assert dead_field not in slurm + assert benchmark["output_length"] == timed_tokens + assert benchmark["dataset_file"] == str( + UPSTREAM_DATASET_ROOT + / f"DeepSeek-V4-8192-{timed_tokens}-16384-ratio-1_for_serve.json" + ) + assert environment["container_mount"] == ( + "/data/:/data/,/scratch/:/scratch/" + ) + assert environment["container_image"] == UPSTREAM_CONTAINER_IMAGE + assert environment["model_path"] == "/scratch/models/DeepSeek-V4-Pro" + assert "staging" not in environment + + serialized = source_path.read_text() + for forbidden in ( + "/lustre", + "/raid", + "coreai_", + ): + assert forbidden not in serialized + + launcher = (REPO_ROOT / "runners/launch_gb300-nv.sh").read_text() + assert 'SLURM_PARTITION="batch_1"' in launcher + assert 'SLURM_ACCOUNT="benchmark"' in launcher + assert ( + "srun --partition=$SLURM_PARTITION --exclusive --time=180" + ) in launcher + assert f'{UPSTREAM_RUNNER_ROOT}/squash/' in launcher + + wrapper = ( + REPO_ROOT / "benchmarks/multi_node/offline/dsv4_fp4_gb300_trt.sh" + ).read_text() + assert '--partition "${SLURM_PARTITION:-batch_1}"' in wrapper + assert '--account "${SLURM_ACCOUNT:-benchmark}"' in wrapper + assert str(UPSTREAM_RUNNER_ROOT) in wrapper + assert '--job-time "${SLURM_JOB_TIME:-03:00:00}"' in wrapper + + +def test_direct_submit_mounts_default_dataset_generator(tmp_path): + submit = ( + REPO_ROOT + / "benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/submit.py" + ) + source = CONFIG_ROOT / "ctx1_gen1_tep4_concurrency2_mtp3.yaml" + dataset_dir = ( + REPO_ROOT / "benchmarks/multi_node/offline/trtllm_gen_only/dataset" + ) + result = subprocess.run( + [ + sys.executable, + str(submit), + "--config", + str(source), + "--log-dir", + str(tmp_path / "logs"), + "--dry-run", + "--wait", + ], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + command = next( + line for line in result.stdout.splitlines() if line.startswith("sbatch --") + ) + assert "--time=03:00:00" in command + assert f"{dataset_dir}:{dataset_dir}" in command + assert f"--dataset-generator {dataset_dir / 'gen_dataset.sh'}" in command + + +def test_render_effective_config_changes_only_runtime_and_measurement_fields( + tmp_path, +): + source_path = CONFIG_ROOT / "ctx1_gen1_tep4_concurrency2_mtp3.yaml" + source = _load_yaml(source_path) + original = json.loads(json.dumps(source)) + output_path = tmp_path / "effective.yaml" + + effective, metadata = render_effective_config( + source, + output_path=output_path, + partition="batch_1", + account="benchmark", + job_name="gb300-nv_0", + container_image="/data/squash/trtllm.sqsh", + container_mount="/data/:/data/,/scratch/:/scratch/,/repo:/workspace", + model_path="/scratch/models/DeepSeek-V4-Pro", + dataset_root="/data/datasets/dsv4-trt-offline", + log_dir="/data/runs/case", + decode_steps=256, + ) + + assert source == original + assert _load_yaml(output_path) == effective + assert effective["slurm"]["partition"] == "batch_1" + assert effective["slurm"]["account"] == "benchmark" + assert effective["slurm"]["job_time"] == "03:00:00" + assert effective["slurm"]["job_name"] == "gb300-nv_0" + assert effective["benchmark"]["input_length"] == 8192 + assert effective["benchmark"]["output_length"] == 623 + assert effective["benchmark"]["multi_round"] == 1 + assert effective["benchmark"]["dataset_file"].endswith( + "DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json" + ) + assert effective["environment"]["container_image"].endswith("trtllm.sqsh") + assert effective["environment"]["model_path"].endswith("DeepSeek-V4-Pro") + assert effective["environment"]["log_dir"] == "/data/runs/case" + assert metadata.mtp == 3 + assert metadata.concurrency == 2 + + +def test_build_result_from_logs_writes_workflow_compatible_json(tmp_path): + source = _load_yaml( + CONFIG_ROOT / "ctx1_gen1_tep4_concurrency2_mtp3.yaml" + ) + log_dir = tmp_path / "logs" + iterlog = log_dir / "concurrency_2" / "gen_only_0.txt" + iterlog.parent.mkdir(parents=True) + iterlog.write_text( + "\n".join( + _iterlog_line( + iteration=index, + rank=0, + scheduled=2, + generation_tokens=8, + device_ms=10.0, + ) + for index in range(1, 257) + ) + ) + output_path = tmp_path / "result.json" + + result = build_result_from_logs( + source, + log_dir=log_dir, + output_path=output_path, + source_config="ctx1_gen1_tep4_concurrency2_mtp3.yaml", + model_id="deepseek-ai/DeepSeek-V4-Pro", + decode_steps=256, + ) + + assert json.loads(output_path.read_text()) == result + assert result["max_concurrency"] == 2 + assert result["num_ctx_gpu"] == 4 + assert result["num_gen_gpu"] == 4 + assert result["retained_iteration_count"] == 256 + assert result["mean_tpot_ms"] == pytest.approx(10.0) + assert result["iteration_log_source"] == "timed_slice" + assert result["iteration_log_file"] == "gen_only_0.txt" + + +def test_build_result_from_logs_falls_back_to_full_gen_worker_log(tmp_path): + source = _load_yaml( + CONFIG_ROOT / "ctx1_gen1_dep32_concurrency1024_mtp3.yaml" + ) + log_dir = tmp_path / "logs" + timed_log = log_dir / "concurrency_1024" / "gen_only_0.txt" + timed_log.parent.mkdir(parents=True) + timed_log.write_text( + _iterlog_line( + iteration=1, + rank=0, + scheduled=2, + generation_tokens=8, + device_ms=90.0, + ) + ) + full_log = log_dir / "3_output_GEN_0.log" + full_log.write_text( + "\n".join( + _iterlog_line( + iteration=index, + rank=0, + scheduled=32, + generation_tokens=128, + device_ms=device_ms, + ) + for index, device_ms in enumerate((20.0, 21.0, 22.0), start=1) + ) + ) + + result = build_result_from_logs( + source, + log_dir=log_dir, + output_path=tmp_path / "result.json", + source_config="ctx1_gen1_dep32_concurrency1024_mtp3.yaml", + model_id="deepseek-ai/DeepSeek-V4-Pro", + decode_steps=256, + ) + + assert result["raw_exact_batch_iteration_count"] == 3 + assert result["retained_iteration_count"] == 3 + assert result["mean_tpot_ms"] == pytest.approx(21.0) + assert result["iteration_log_source"] == "full_gen_worker_fallback" + assert result["iteration_log_file"] == "3_output_GEN_0.log" + + +def test_build_result_rejects_invalid_full_log_when_timed_slice_missing( + tmp_path, +): + source = _load_yaml( + CONFIG_ROOT / "ctx4_gen1_dep16_concurrency4096_mtp1.yaml" + ) + log_dir = tmp_path / "logs" + log_dir.mkdir(parents=True) + partial_row = _iterlog_line( + iteration=1, + rank=0, + scheduled=82, + generation_tokens=256, + device_ms=145.0, + ) + (log_dir / "3_output_GEN_0.log").write_text(partial_row) + + with pytest.raises( + ValueError, + match=( + "No timed GEN iterlog found.*fallback also failed.*" + "3_output_GEN_0.log" + ), + ): + build_result_from_logs( + source, + log_dir=log_dir, + output_path=tmp_path / "result.json", + source_config="ctx4_gen1_dep16_concurrency4096_mtp1.yaml", + model_id="deepseek-ai/DeepSeek-V4-Pro", + decode_steps=256, + ) + + +def test_gb300_launcher_routes_multinode_offline_to_shared_wrapper(): + launcher = (REPO_ROOT / "runners/launch_gb300-nv.sh").read_text() + wrapper_path = "benchmarks/multi_node/offline/dsv4_fp4_gb300_trt.sh" + assert '"$IS_MULTINODE" == "true"' in launcher + assert '"$SPEC_DECODING" == "offline"' in launcher + assert '"$FRAMEWORK" == "trt"' in launcher + assert '"$MODEL_PREFIX" == "dsv4"' in launcher + assert f'bash "{wrapper_path}"' in launcher + + wrapper = (REPO_ROOT / wrapper_path).read_text() + assert "submit.py" in wrapper + assert "--wait" in wrapper + assert "utils/bench_offline/trt_disagg_gen_only.py" in wrapper + assert 'python3 "${ADAPTER}" prepare' in wrapper + assert 'python3 "${ADAPTER}" result' in wrapper + + benchmark_script = ( + REPO_ROOT + / "benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/run_benchmark.sh" + ).read_text() + assert "Warming up one full cohort" in benchmark_script + assert "start_line=$((start_line + 1))" in benchmark_script diff --git a/utils/bench_offline/trt_disagg_gen_only.py b/utils/bench_offline/trt_disagg_gen_only.py new file mode 100644 index 0000000000..218da9ad12 --- /dev/null +++ b/utils/bench_offline/trt_disagg_gen_only.py @@ -0,0 +1,498 @@ +"""Helpers for the DSV4 TensorRT-LLM disaggregated offline benchmark. + +The deployment remains CTX/GEN disaggregated. Only generation-worker device +iterations are scored, using decode steps as the primary throughput unit. +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import statistics +from copy import deepcopy +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +import yaml + + +DEFAULT_ACCEPTANCE_LENGTHS = {1: 1.70, 3: 2.44} + + +@dataclass(frozen=True) +class CaseMetadata: + concurrency: int + ctx_num_workers: int + ctx_tp: int + ctx_ep: int + ctx_dp_attn: bool + gen_num_workers: int + gen_tp: int + gen_ep: int + gen_dp_attn: bool + mtp: int + node_count: int + + +@dataclass(frozen=True) +class IterationSummary: + raw_exact_batch_samples: int + retained_samples: int + mean_device_step_ms: float + median_device_step_ms: float + p90_device_step_ms: float + p99_device_step_ms: float + + +class NoExactFullBatchError(ValueError): + """Raised when an iterlog has no exact-full-batch generation rows.""" + + +_ITERLOG_RE = re.compile( + r"iter = (?P\d+), global_rank = (?P\d+), " + r"rank = \d+, currank_total_requests = \d+/\d+, " + r"host_step_time = [\d.]+ms, prev_device_step_time = " + r"(?P[\d.]+)ms, timestamp = .*?, " + r"num_scheduled_requests: (?P\d+), " + r"states = \{'num_ctx_requests': \d+, 'num_ctx_tokens': " + r"(?P\d+), 'num_generation_tokens': " + r"(?P\d+)\}" +) + + +def timed_max_tokens(decode_steps: int, mtp: int) -> int: + """Return the output-token cap targeting ``decode_steps`` model rounds.""" + if decode_steps <= 0: + raise ValueError("decode_steps must be positive") + try: + acceptance = DEFAULT_ACCEPTANCE_LENGTHS[mtp] + except KeyError as exc: + raise ValueError(f"No fixed acceptance length for MTP{mtp}") from exc + return 1 + round((decode_steps - 1) * acceptance) + + +def _world_size(worker: dict[str, Any]) -> int: + return ( + int(worker.get("tensor_parallel_size", 1)) + * int(worker.get("context_parallel_size", 1)) + * int(worker.get("pipeline_parallel_size", 1)) + ) + + +def case_metadata(config: dict[str, Any]) -> CaseMetadata: + benchmark = config["benchmark"] + hardware = config["hardware"] + ctx = config["worker_config"]["ctx"] + gen = config["worker_config"]["gen"] + ctx_workers = int(hardware["num_ctx_servers"]) + gen_workers = int(hardware["num_gen_servers"]) + gpus_per_node = int(hardware["gpus_per_node"]) + total_gpus = ctx_workers * _world_size(ctx) + gen_workers * _world_size(gen) + return CaseMetadata( + concurrency=int(str(benchmark["concurrency_list"]).split(",")[0]), + ctx_num_workers=ctx_workers, + ctx_tp=int(ctx["tensor_parallel_size"]), + ctx_ep=int(ctx["moe_expert_parallel_size"]), + ctx_dp_attn=bool(ctx["enable_attention_dp"]), + gen_num_workers=gen_workers, + gen_tp=int(gen["tensor_parallel_size"]), + gen_ep=int(gen["moe_expert_parallel_size"]), + gen_dp_attn=bool(gen["enable_attention_dp"]), + mtp=int(gen["speculative_config"]["max_draft_len"]), + node_count=math.ceil(total_gpus / gpus_per_node), + ) + + +def validate_case_against_workflow( + config: dict[str, Any], + *, + concurrency: int, + prefill_num_workers: int, + prefill_tp: int, + prefill_ep: int, + prefill_dp_attn: bool, + decode_num_workers: int, + decode_tp: int, + decode_ep: int, + decode_dp_attn: bool, +) -> CaseMetadata: + """Reject a master-config row that disagrees with its source case YAML.""" + metadata = case_metadata(config) + expected = { + "concurrency": concurrency, + "ctx_num_workers": prefill_num_workers, + "ctx_tp": prefill_tp, + "ctx_ep": prefill_ep, + "ctx_dp_attn": prefill_dp_attn, + "gen_num_workers": decode_num_workers, + "gen_tp": decode_tp, + "gen_ep": decode_ep, + "gen_dp_attn": decode_dp_attn, + } + mismatches = [ + f"{name}: source={getattr(metadata, name)!r}, workflow={value!r}" + for name, value in expected.items() + if getattr(metadata, name) != value + ] + if mismatches: + raise ValueError("Workflow/source topology mismatch: " + "; ".join(mismatches)) + if config["benchmark"]["mode"] != "gen_only": + raise ValueError("Offline disaggregated cases must use benchmark.mode=gen_only") + if metadata.gen_num_workers != 1: + raise ValueError("Offline disaggregated cases require exactly one GEN worker") + return metadata + + +def render_effective_config( + source: dict[str, Any], + *, + output_path: Path, + partition: str, + account: str, + job_name: str, + container_image: str, + container_mount: str, + model_path: str, + dataset_root: str, + log_dir: str, + decode_steps: int, + job_time: str = "03:00:00", +) -> tuple[dict[str, Any], CaseMetadata]: + """Render cluster-local paths without mutating the pinned source case.""" + effective = deepcopy(source) + metadata = case_metadata(effective) + output_tokens = timed_max_tokens(decode_steps, metadata.mtp) + dataset_name = ( + f"DeepSeek-V4-8192-{output_tokens}-16384-ratio-1_for_serve.json" + ) + + effective["slurm"]["partition"] = partition + effective["slurm"]["account"] = account + effective["slurm"]["job_time"] = job_time + effective["slurm"]["job_name"] = job_name + effective["benchmark"]["input_length"] = 8192 + effective["benchmark"]["output_length"] = output_tokens + effective["benchmark"]["multi_round"] = 1 + effective["benchmark"]["dataset_file"] = str( + Path(dataset_root) / dataset_name + ) + environment = effective["environment"] + environment["container_image"] = container_image + environment["container_mount"] = container_mount + environment["model_path"] = model_path + environment["log_dir"] = log_dir + environment["trtllm_repo"] = "" + environment["trtllm_wheel_path"] = "" + environment["build_wheel"] = False + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(yaml.safe_dump(effective, sort_keys=False)) + return effective, metadata + + +def _percentile(values: list[float], percentile: float) -> float: + if not values: + raise ValueError("Cannot calculate a percentile without samples") + ordered = sorted(values) + index = round((percentile / 100.0) * (len(ordered) - 1)) + return ordered[max(0, min(len(ordered) - 1, index))] + + +def parse_gen_iterlog( + path: Path, + *, + concurrency: int, + gen_tp: int, + mtp: int, + attention_dp: bool, +) -> IterationSummary: + """Parse exact-full-batch GEN device iterations from one timed iterlog.""" + if concurrency <= 0 or gen_tp <= 0: + raise ValueError("concurrency and gen_tp must be positive") + if attention_dp and concurrency % gen_tp: + raise ValueError("DEP concurrency must be divisible by GEN TP") + + scheduled = concurrency // gen_tp if attention_dp else concurrency + generation_tokens = scheduled * (mtp + 1) + rows: dict[tuple[int, int], float] = {} + for match in _ITERLOG_RE.finditer(path.read_text(errors="ignore")): + if int(match["ctx_tokens"]) != 0: + continue + if int(match["scheduled"]) != scheduled: + continue + if int(match["generation_tokens"]) != generation_tokens: + continue + rows[(int(match["iter"]), int(match["global_rank"]))] = float( + match["device_ms"] + ) + + exact = list(rows.values()) + if not exact: + raise NoExactFullBatchError( + f"No exact-full-batch GEN iterations found in {path}" + ) + median = statistics.median(exact) + retained = [value for value in exact if median * 0.8 <= value <= median * 1.2] + if not retained: + raise ValueError("Median filter removed every exact-full-batch iteration") + return IterationSummary( + raw_exact_batch_samples=len(exact), + retained_samples=len(retained), + mean_device_step_ms=statistics.fmean(retained), + median_device_step_ms=statistics.median(retained), + p90_device_step_ms=_percentile(retained, 90), + p99_device_step_ms=_percentile(retained, 99), + ) + + +def build_result( + *, + model_id: str, + source_config: str, + concurrency: int, + ctx_gpus: int, + gen_gpus: int, + gen_tp: int, + mtp: int, + mean_device_step_ms: float, + median_device_step_ms: float, + p90_device_step_ms: float, + p99_device_step_ms: float, + raw_samples: int, + retained_samples: int, + decode_steps: int, +) -> dict[str, Any]: + """Build the raw JSON consumed by ``utils/process_result.py``.""" + if mean_device_step_ms <= 0: + raise ValueError("mean_device_step_ms must be positive") + if gen_gpus <= 0 or ctx_gpus <= 0: + raise ValueError("CTX and GEN GPU counts must be positive") + acceptance = DEFAULT_ACCEPTANCE_LENGTHS[mtp] + step_throughput = concurrency / (mean_device_step_ms / 1000.0) + token_throughput = step_throughput * acceptance + return { + "model_id": model_id, + "engine_mode": "offline", + "measurement_boundary": "gen_iteration", + "tpot_unit": "decode_step", + "mtp": mtp, + "max_concurrency": concurrency, + "decode_steps_target": decode_steps, + "timed_max_tokens": timed_max_tokens(decode_steps, mtp), + "assumed_tokens_per_step": acceptance, + "moe_routing": "perfect", + "source_config": source_config, + "num_ctx_gpu": ctx_gpus, + "num_gen_gpu": gen_gpus, + "gen_tp": gen_tp, + "raw_exact_batch_iteration_count": raw_samples, + "retained_iteration_count": retained_samples, + "mean_tpot_ms": mean_device_step_ms, + "median_tpot_ms": median_device_step_ms, + "p90_tpot_ms": p90_device_step_ms, + "p99_tpot_ms": p99_device_step_ms, + "total_token_throughput": step_throughput, + "output_throughput": step_throughput, + "decode_step_throughput_per_gen_gpu": step_throughput / gen_gpus, + "token_equivalent_output_throughput": token_throughput, + "token_equivalent_output_throughput_per_gen_gpu": ( + token_throughput / gen_gpus + ), + } + + +def build_result_from_logs( + config: dict[str, Any], + *, + log_dir: Path, + output_path: Path, + source_config: str, + model_id: str, + decode_steps: int, +) -> dict[str, Any]: + """Parse a completed case and write its workflow-compatible raw JSON.""" + metadata = case_metadata(config) + candidates = sorted( + log_dir.glob(f"**/concurrency_{metadata.concurrency}/gen_only*.txt") + ) + if len(candidates) > 1: + raise ValueError( + "Expected at most one timed GEN iterlog for concurrency " + f"{metadata.concurrency}, found {len(candidates)} under {log_dir}" + ) + if candidates: + iteration_log = candidates[0] + iteration_log_source = "timed_slice" + try: + summary = parse_gen_iterlog( + iteration_log, + concurrency=metadata.concurrency, + gen_tp=metadata.gen_tp, + mtp=metadata.mtp, + attention_dp=metadata.gen_dp_attn, + ) + except NoExactFullBatchError as error: + timed_error = error + else: + timed_error = None + else: + timed_error = NoExactFullBatchError( + "No timed GEN iterlog found for concurrency " + f"{metadata.concurrency} under {log_dir}" + ) + + if timed_error is not None: + full_log_candidates = sorted(log_dir.glob("**/3_output_GEN_*.log")) + if len(full_log_candidates) != 1: + raise ValueError( + f"{timed_error}; expected exactly one full GEN worker log " + f"under {log_dir}, found {len(full_log_candidates)}" + ) from timed_error + iteration_log = full_log_candidates[0] + iteration_log_source = "full_gen_worker_fallback" + try: + summary = parse_gen_iterlog( + iteration_log, + concurrency=metadata.concurrency, + gen_tp=metadata.gen_tp, + mtp=metadata.mtp, + attention_dp=metadata.gen_dp_attn, + ) + except NoExactFullBatchError as full_log_error: + raise NoExactFullBatchError( + f"{timed_error}; fallback also failed: {full_log_error}" + ) from full_log_error + ctx = config["worker_config"]["ctx"] + gen = config["worker_config"]["gen"] + ctx_gpus = metadata.ctx_num_workers * _world_size(ctx) + gen_gpus = metadata.gen_num_workers * _world_size(gen) + result = build_result( + model_id=model_id, + source_config=source_config, + concurrency=metadata.concurrency, + ctx_gpus=ctx_gpus, + gen_gpus=gen_gpus, + gen_tp=metadata.gen_tp, + mtp=metadata.mtp, + mean_device_step_ms=summary.mean_device_step_ms, + median_device_step_ms=summary.median_device_step_ms, + p90_device_step_ms=summary.p90_device_step_ms, + p99_device_step_ms=summary.p99_device_step_ms, + raw_samples=summary.raw_exact_batch_samples, + retained_samples=summary.retained_samples, + decode_steps=decode_steps, + ) + result["iteration_log_source"] = iteration_log_source + result["iteration_log_file"] = iteration_log.name + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + return result + + +def _load_yaml(path: Path) -> dict[str, Any]: + return yaml.safe_load(path.read_text()) + + +def _bool_arg(value: str) -> bool: + normalized = value.lower() + if normalized not in {"true", "false"}: + raise argparse.ArgumentTypeError("expected true or false") + return normalized == "true" + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + prepare = subparsers.add_parser("prepare") + prepare.add_argument("--source-config", type=Path, required=True) + prepare.add_argument("--output-config", type=Path, required=True) + prepare.add_argument("--metadata-output", type=Path, required=True) + prepare.add_argument("--partition", required=True) + prepare.add_argument("--account", required=True) + prepare.add_argument("--job-time", default="03:00:00") + prepare.add_argument("--job-name", required=True) + prepare.add_argument("--container-image", required=True) + prepare.add_argument("--container-mount", required=True) + prepare.add_argument("--model-path", required=True) + prepare.add_argument("--dataset-root", required=True) + prepare.add_argument("--log-dir", required=True) + prepare.add_argument("--decode-steps", type=int, required=True) + prepare.add_argument("--concurrency", type=int, required=True) + prepare.add_argument("--prefill-num-workers", type=int, required=True) + prepare.add_argument("--prefill-tp", type=int, required=True) + prepare.add_argument("--prefill-ep", type=int, required=True) + prepare.add_argument("--prefill-dp-attn", type=_bool_arg, required=True) + prepare.add_argument("--decode-num-workers", type=int, required=True) + prepare.add_argument("--decode-tp", type=int, required=True) + prepare.add_argument("--decode-ep", type=int, required=True) + prepare.add_argument("--decode-dp-attn", type=_bool_arg, required=True) + + result = subparsers.add_parser("result") + result.add_argument("--config", type=Path, required=True) + result.add_argument("--log-dir", type=Path, required=True) + result.add_argument("--output", type=Path, required=True) + result.add_argument("--source-config", required=True) + result.add_argument("--model-id", required=True) + result.add_argument("--decode-steps", type=int, required=True) + return parser + + +def main() -> None: + args = _parser().parse_args() + if args.command == "prepare": + source = _load_yaml(args.source_config) + effective, metadata = render_effective_config( + source, + output_path=args.output_config, + partition=args.partition, + account=args.account, + job_time=args.job_time, + job_name=args.job_name, + container_image=args.container_image, + container_mount=args.container_mount, + model_path=args.model_path, + dataset_root=args.dataset_root, + log_dir=args.log_dir, + decode_steps=args.decode_steps, + ) + validate_case_against_workflow( + effective, + concurrency=args.concurrency, + prefill_num_workers=args.prefill_num_workers, + prefill_tp=args.prefill_tp, + prefill_ep=args.prefill_ep, + prefill_dp_attn=args.prefill_dp_attn, + decode_num_workers=args.decode_num_workers, + decode_tp=args.decode_tp, + decode_ep=args.decode_ep, + decode_dp_attn=args.decode_dp_attn, + ) + payload = { + **asdict(metadata), + "dataset_file": effective["benchmark"]["dataset_file"], + "timed_max_tokens": effective["benchmark"]["output_length"], + } + args.metadata_output.parent.mkdir(parents=True, exist_ok=True) + args.metadata_output.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n" + ) + print(json.dumps(payload, sort_keys=True)) + return + + result = build_result_from_logs( + _load_yaml(args.config), + log_dir=args.log_dir, + output_path=args.output, + source_config=args.source_config, + model_id=args.model_id, + decode_steps=args.decode_steps, + ) + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/utils/process_result.py b/utils/process_result.py index 97f8d4843e..1e3910dea0 100644 --- a/utils/process_result.py +++ b/utils/process_result.py @@ -42,10 +42,10 @@ def get_required_env_vars(required_vars): with open(f'{result_filename}.json') as f: bmk_result = json.load(f) -# Offline in-process runs (utils/bench_offline/run_offline.py) report -# per-decode-step TPOT and step throughput. Their dispatch value is -# SPEC_DECODING=offline, but the row should describe the actual run config: -# spec_decoding reflects whether MTP draft tokens were enabled. +# Offline runs report per-decode-step TPOT and step throughput. They can be +# in-process engine runs or disaggregated generation-worker measurements. +# Their dispatch value is SPEC_DECODING=offline, but the row should describe +# the actual run config: spec_decoding reflects whether MTP was enabled. is_offline = str(bmk_result.get('engine_mode', '')).lower() == 'offline' if is_offline: spec_decoding = 'mtp' if int(bmk_result.get('mtp', 0) or 0) > 0 else 'none' @@ -76,12 +76,26 @@ def get_required_env_vars(required_vars): if bmk_result.get('spec_tokens_per_step_observed') is not None: data['spec_tokens_per_step_observed'] = float( bmk_result['spec_tokens_per_step_observed']) + offline_passthrough_fields = ( + 'mtp', + 'measurement_boundary', + 'assumed_tokens_per_step', + 'timed_max_tokens', + 'raw_exact_batch_iteration_count', + 'retained_iteration_count', + 'decode_step_throughput_per_gen_gpu', + 'token_equivalent_output_throughput', + 'token_equivalent_output_throughput_per_gen_gpu', + 'source_config', + 'iteration_log_source', + 'iteration_log_file', + ) + for field in offline_passthrough_fields: + if bmk_result.get(field) is not None: + data[field] = bmk_result[field] is_multinode = os.environ.get('IS_MULTINODE', 'false').lower() == 'true' -if is_offline and is_multinode: - raise ValueError("Offline in-process results are single-node only.") - if is_multinode: # TODO: Eventually will have to have a separate condition in here for multinode disagg and # multinode agg. For now, just assume that multinode implies disagg. diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 11cb829b0d..4939ccb6bc 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -741,9 +741,50 @@ def test_offline_string_tpot_field_does_not_crash_intvty( assert "std_intvty" not in output_data assert output_data["std_tpot"] == pytest.approx(0.0) - def test_offline_multinode_rejected( + def test_offline_multinode_preserves_gen_only_metadata( self, tmp_path, offline_env_vars, offline_benchmark_result): env = {**offline_env_vars, "IS_MULTINODE": "true"} + env.update({ + "PREFILL_GPUS": "4", + "DECODE_GPUS": "32", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "4", + "PREFILL_EP": "4", + "PREFILL_DP_ATTN": "true", + "DECODE_NUM_WORKERS": "1", + "DECODE_TP": "32", + "DECODE_EP": "32", + "DECODE_DP_ATTN": "true", + "DISAGG": "true", + }) + offline_benchmark_result.update({ + "measurement_boundary": "gen_iteration", + "assumed_tokens_per_step": 2.44, + "timed_max_tokens": 623, + "retained_iteration_count": 250, + "token_equivalent_output_throughput": 8206.0, + "token_equivalent_output_throughput_per_gen_gpu": 256.4375, + "source_config": "ctx1_gen1_dep32_concurrency64_mtp3.yaml", + "iteration_log_source": "full_gen_worker_fallback", + "iteration_log_file": "3_output_GEN_0.log", + }) + result = run_script(tmp_path, env, offline_benchmark_result) - assert result.returncode != 0 - assert "single-node" in result.stderr + assert result.returncode == 0, result.stderr + output_data = json.loads(result.stdout) + assert output_data["is_multinode"] is True + assert output_data["mtp"] == 3 + assert output_data["num_prefill_gpu"] == 4 + assert output_data["num_decode_gpu"] == 32 + assert output_data["output_tput_per_gpu"] == pytest.approx( + offline_benchmark_result["output_throughput"] / 32 + ) + assert output_data["measurement_boundary"] == "gen_iteration" + assert output_data["assumed_tokens_per_step"] == pytest.approx(2.44) + assert output_data["timed_max_tokens"] == 623 + assert output_data["retained_iteration_count"] == 250 + assert output_data["source_config"].endswith("mtp3.yaml") + assert output_data["iteration_log_source"] == ( + "full_gen_worker_fallback" + ) + assert output_data["iteration_log_file"] == "3_output_GEN_0.log"