Add DSV4 GB300 TRT disaggregated offline benchmark / 新增 DSV4 GB300 TRT 分离式离线基准测试

.github/configs/nvidia-master.yaml

@@ -3615,6 +3615,130 @@ dsv4-fp4-gb300-vllm-offline-16chip:
       search-space:
       - { tp: 16, ep: 16, dp-attn: true, conc-start: 32, conc-end: 512, spec-decoding: offline }
 
+# GB300 TensorRT-LLM disaggregated generation-worker benchmark. Each row
+# selects one bundled frontier config; spec-decoding=offline routes it through
+# native trtllm-serve.
+dsv4-fp4-gb300-trt-offline:
+  image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc15.post1
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: gb300-nv
+  precision: fp4
+  framework: trt
+  multinode: true
+  disagg: true
+  scenarios:
+    fixed-seq-len:
+    - isl: 8192
+      osl: 256
+      search-space:
+      - spec-decoding: offline
+        conc-list: [1]
+        prefill: &dsv4-trt-offline-ctx1
+          num-worker: 1
+          tp: 4
+          ep: 4
+          dp-attn: true
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency1_mtp3.yaml"
+        decode: &dsv4-trt-offline-tep4
+          num-worker: 1
+          tp: 4
+          ep: 4
+          dp-attn: false
+      - spec-decoding: offline
+        conc-list: [2]
+        prefill:
+          <<: *dsv4-trt-offline-ctx1
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency2_mtp3.yaml"
+        decode: *dsv4-trt-offline-tep4
+      - spec-decoding: offline
+        conc-list: [4]
+        prefill:
+          <<: *dsv4-trt-offline-ctx1
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep4_concurrency4_mtp3.yaml"
+        decode: *dsv4-trt-offline-tep4
+      - spec-decoding: offline
+        conc-list: [1]
+        prefill:
+          <<: *dsv4-trt-offline-ctx1
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_tep8_concurrency1_mtp3.yaml"
+        decode:
+          num-worker: 1
+          tp: 8
+          ep: 8
+          dp-attn: false
+      - spec-decoding: offline
+        conc-list: [64]
+        prefill:
+          <<: *dsv4-trt-offline-ctx1
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency64_mtp3.yaml"
+        decode: &dsv4-trt-offline-dep32
+          num-worker: 1
+          tp: 32
+          ep: 32
+          dp-attn: true
+      - spec-decoding: offline
+        conc-list: [128]
+        prefill:
+          <<: *dsv4-trt-offline-ctx1
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency128_mtp3.yaml"
+        decode: *dsv4-trt-offline-dep32
+      - spec-decoding: offline
+        conc-list: [256]
+        prefill:
+          <<: *dsv4-trt-offline-ctx1
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency256_mtp3.yaml"
+        decode: *dsv4-trt-offline-dep32
+      - spec-decoding: offline
+        conc-list: [512]
+        prefill:
+          <<: *dsv4-trt-offline-ctx1
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency512_mtp3.yaml"
+        decode: *dsv4-trt-offline-dep32
+      - spec-decoding: offline
+        conc-list: [1024]
+        prefill:
+          <<: *dsv4-trt-offline-ctx1
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency1024_mtp3.yaml"
+        decode: *dsv4-trt-offline-dep32
+      - spec-decoding: offline
+        conc-list: [2048]
+        prefill:
+          <<: *dsv4-trt-offline-ctx1
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency2048_mtp3.yaml"
+        decode: *dsv4-trt-offline-dep32
+      - spec-decoding: offline
+        conc-list: [4096]
+        prefill: &dsv4-trt-offline-ctx4
+          num-worker: 4
+          tp: 4
+          ep: 4
+          dp-attn: true
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep16_concurrency4096_mtp1.yaml"
+        decode: &dsv4-trt-offline-dep16
+          num-worker: 1
+          tp: 16
+          ep: 16
+          dp-attn: true
+      - spec-decoding: offline
+        conc-list: [4096]
+        prefill:
+          <<: *dsv4-trt-offline-ctx4
+          additional-settings:
+          - "CONFIG_FILE=benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx4_gen1_dep32_concurrency4096_mtp1.yaml"
+        decode: *dsv4-trt-offline-dep32
+
 dsv4-fp4-b300-sglang-offline:
   image: lmsysorg/sglang:nightly-dev-cu13-20260610-f332e526
   model: deepseek-ai/DeepSeek-V4-Pro

benchmarks/multi_node/offline/dsv4_fp4_gb300_trt.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+
+# Adapt one bundled DSV4 generation-only case to the InferenceX multi-node contract.
+
+set -euo pipefail
+
+required=(
+    CONFIG_FILE RESULT_FILENAME GITHUB_WORKSPACE RUNNER_NAME MODEL
+    CONC_LIST ISL OSL PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP
+    PREFILL_DP_ATTN DECODE_NUM_WORKERS DECODE_TP DECODE_EP
+    DECODE_DP_ATTN TRT_GEN_ONLY_SQUASH_FILE MODEL_PATH
+)
+for name in "${required[@]}"; do
+    if [[ -z "${!name:-}" ]]; then
+        echo "ERROR: required environment variable ${name} is unset" >&2
+        exit 1
+    fi
+done
+
+if [[ $(wc -w <<<"${CONC_LIST}") -ne 1 ]]; then
+    echo "ERROR: each offline matrix row must select exactly one concurrency: ${CONC_LIST}" >&2
+    exit 1
+fi
+CONCURRENCY=${CONC_LIST}
+SOURCE_CONFIG="${GITHUB_WORKSPACE}/${CONFIG_FILE}"
+if [[ ! -f "${SOURCE_CONFIG}" ]]; then
+    echo "ERROR: source config does not exist: ${SOURCE_CONFIG}" >&2
+    exit 1
+fi
+
+CASE_NAME=$(basename "${SOURCE_CONFIG}" .yaml)
+RUN_KEY=$(printf '%s' "${RESULT_FILENAME}-${CASE_NAME}" | sha1sum | cut -c1-12)
+RUN_ROOT="/data/home/sa-shared/gharunners/trt-gen-only/${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}-${RUN_KEY}"
+LOG_DIR="${RUN_ROOT}/logs"
+EFFECTIVE_CONFIG="${RUN_ROOT}/effective_config.yaml"
+METADATA_FILE="${RUN_ROOT}/case_metadata.json"
+DATASET_ROOT="/data/home/sa-shared/gharunners/datasets/dsv4-trt-offline"
+ADAPTER="${GITHUB_WORKSPACE}/utils/bench_offline/trt_disagg_gen_only.py"
+SUBMIT="${GITHUB_WORKSPACE}/benchmarks/multi_node/offline/trtllm_gen_only/benchmark/scripts/submit.py"
+CONTAINER_MOUNT="/data/:/data/,/scratch/:/scratch/,${GITHUB_WORKSPACE}:/workspace"
+
+mkdir -p "${RUN_ROOT}" "${LOG_DIR}" "${DATASET_ROOT}"
+
+archive_logs() {
+    if [[ -d "${RUN_ROOT}" ]]; then
+        tar czf "${GITHUB_WORKSPACE}/multinode_server_logs.tar.gz" \
+            -C "${RUN_ROOT}" . 2>/dev/null || true
+    fi
+}
+trap archive_logs EXIT
+
+python3 "${ADAPTER}" prepare \
+    --source-config "${SOURCE_CONFIG}" \
+    --output-config "${EFFECTIVE_CONFIG}" \
+    --metadata-output "${METADATA_FILE}" \
+    --partition "${SLURM_PARTITION:-batch_1}" \
+    --account "${SLURM_ACCOUNT:-benchmark}" \
+    --job-time "${SLURM_JOB_TIME:-03:00:00}" \
+    --job-name "${RUNNER_NAME}" \
+    --container-image "${TRT_GEN_ONLY_SQUASH_FILE}" \
+    --container-mount "${CONTAINER_MOUNT}" \
+    --model-path "${MODEL_PATH}" \
+    --dataset-root "${DATASET_ROOT}" \
+    --log-dir "${LOG_DIR}" \
+    --decode-steps "${OSL}" \
+    --concurrency "${CONCURRENCY}" \
+    --prefill-num-workers "${PREFILL_NUM_WORKERS}" \
+    --prefill-tp "${PREFILL_TP}" \
+    --prefill-ep "${PREFILL_EP}" \
+    --prefill-dp-attn "${PREFILL_DP_ATTN}" \
+    --decode-num-workers "${DECODE_NUM_WORKERS}" \
+    --decode-tp "${DECODE_TP}" \
+    --decode-ep "${DECODE_EP}" \
+    --decode-dp-attn "${DECODE_DP_ATTN}"
+
+export TRT_GEN_ONLY_DATASET_GENERATOR="/workspace/benchmarks/multi_node/offline/trtllm_gen_only/dataset/gen_dataset.sh"
+python3 "${SUBMIT}" \
+    --config "${EFFECTIVE_CONFIG}" \
+    --log-dir "${LOG_DIR}" \
+    --wait
+
+CTX_GPUS=$((PREFILL_NUM_WORKERS * PREFILL_TP))
+GEN_GPUS=$((DECODE_NUM_WORKERS * DECODE_TP))
+TOTAL_GPUS=$((CTX_GPUS + GEN_GPUS))
+RAW_BASENAME="${RESULT_FILENAME}_${CASE_NAME}_conc${CONCURRENCY}_gpus_${TOTAL_GPUS}_ctx_${CTX_GPUS}_gen_${GEN_GPUS}"
+RAW_RESULT="${GITHUB_WORKSPACE}/${RAW_BASENAME}.json"
+
+python3 "${ADAPTER}" result \
+    --config "${EFFECTIVE_CONFIG}" \
+    --log-dir "${LOG_DIR}" \
+    --output "${RAW_RESULT}" \
+    --source-config "${CASE_NAME}.yaml" \
+    --model-id "${MODEL}" \
+    --decode-steps "${OSL}"
+
+echo "Offline disaggregated result: ${RAW_RESULT}"

benchmarks/multi_node/offline/trtllm_gen_only/.gitignore

@@ -0,0 +1,12 @@
+# Generated datasets
+dataset/data/
+
+# Python
+__pycache__/
+*.pyc
+
+# Editor / OS
+.DS_Store
+
+# Benchmark outputs (submit.py writes here when configs have no log_dir)
+benchmark/scripts/logs/

benchmarks/multi_node/offline/trtllm_gen_only/benchmark/configs/ctx1_gen1_dep32_concurrency1024_mtp3.yaml

@@ -0,0 +1,115 @@
+slurm:
+  script_file: disaggr_torch.slurm
+  partition: batch_1
+  account: benchmark
+  job_time: 03:00:00
+  job_name: dsv4-offline-ctx1_gen1_dep32_concurrency1024_mtp3
+  extra_args: --gres=gpu:4
+  set_segment: true
+  numa_bind: true
+benchmark:
+  mode: gen_only
+  use_nv_sa_benchmark: false
+  multi_round: 1
+  benchmark_ratio: 0.8
+  streaming: false
+  concurrency_list: '1024'
+  input_length: 8192
+  output_length: 623
+  dataset_file: /data/home/sa-shared/gharunners/datasets/dsv4-trt-offline/DeepSeek-V4-8192-623-16384-ratio-1_for_serve.json
+hardware:
+  gpus_per_node: 4
+  num_ctx_servers: 1
+  num_gen_servers: 1
+environment:
+  container_mount: /data/:/data/,/scratch/:/scratch/
+  container_image: /data/home/sa-shared/gharunners/squash/nvcr.io_nvidia_tensorrt-llm_release_1.3.0rc15.post1.sqsh
+  model_path: /scratch/models/DeepSeek-V4-Pro
+  trtllm_repo: ''
+  build_wheel: false
+  cuda_architectures: ''
+  trtllm_wheel_path: ''
+  worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1
+    TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0
+    PYTHONWARNINGS=ignore::DeprecationWarning:cutlass.cute.core ENABLE_PERFECT_ROUTER=1
+  server_env_var: TRTLLM_SERVER_DISABLE_GC=1
+  ctx_worker_env_var: ' PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True'
+  gen_worker_env_var: ''
+profiling:
+  nsys_on: false
+  ctx_profile_range: 10-30
+  gen_profile_range: 200-250
+accuracy:
+  enable_accuracy_test: false
+  tasks:
+    gsm8k:
+      model: local-completions
+      model_args_extra: num_concurrent=512,max_retries=3,tokenized_requests=false,timeout=7200,max_gen_toks=16384
+      extra_kwargs:
+        trust_remote_code: true
+worker_config:
+  gen:
+    tensor_parallel_size: 32
+    moe_expert_parallel_size: 32
+    enable_attention_dp: true
+    enable_lm_head_tp_in_adp: true
+    pipeline_parallel_size: 1
+    max_batch_size: 128
+    max_num_tokens: 512
+    max_seq_len: 9256
+    cuda_graph_config:
+      enable_padding: true
+      batch_sizes:
+      - 1
+      - 2
+      - 4
+      - 8
+      - 16
+      - 32
+      - 64
+      - 128
+    print_iter_log: true
+    kv_cache_config:
+      enable_block_reuse: false
+      free_gpu_memory_fraction: 0.7
+      dtype: fp8
+      tokens_per_block: 128
+    moe_config:
+      backend: MEGAMOE_DEEPGEMM
+      use_low_precision_moe_combine: true
+    cache_transceiver_config:
+      max_tokens_in_buffer: 8192
+      backend: NIXL
+      kv_transfer_timeout_ms: 600000
+      transceiver_runtime: PYTHON
+    stream_interval: 100
+    num_postprocess_workers: 4
+    speculative_config:
+      decoding_type: MTP
+      max_draft_len: 3
+  ctx:
+    max_batch_size: 2
+    max_num_tokens: 8192
+    max_seq_len: 8232
+    tensor_parallel_size: 4
+    moe_expert_parallel_size: 4
+    enable_attention_dp: true
+    pipeline_parallel_size: 1
+    print_iter_log: true
+    cuda_graph_config: null
+    disable_overlap_scheduler: true
+    moe_config:
+      backend: TRTLLM
+    kv_cache_config:
+      enable_block_reuse: false
+      free_gpu_memory_fraction: 0.5
+      dtype: fp8
+      tokens_per_block: 128
+    cache_transceiver_config:
+      max_tokens_in_buffer: 8192
+      backend: NIXL
+      kv_transfer_timeout_ms: 600000
+      transceiver_runtime: PYTHON
+    speculative_config:
+      decoding_type: MTP
+      max_draft_len: 3