diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh new file mode 100644 index 0000000000..d11740d596 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +apt-get -y update && apt-get install -y --no-install-recommends --allow-change-held-packages numactl + +pip install msgpack + +if [ -f /configs/patches/vllm_numa_bind_hash_fix.py ]; then + python3 /configs/patches/vllm_numa_bind_hash_fix.py +fi diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml new file mode 100644 index 0000000000..c548caf6cb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml @@ -0,0 +1,101 @@ +name: "dsv4-vllm-agg-h100-tep8pp2-1k1k" + +# DeepSeek-V4-Pro FP8 H100 AGGREGATED TP8 x PP2 recipe (2 nodes / 16 GPUs). +# The ~1TB FP8 checkpoint does not fit one 8xH100 node (640GB). TP8 x PP2 +# shards attention/dense (/8) and layers (/2) across 16 GPUs -- the original +# memory-correct sizing that loaded the model and reached prefill. That +# topology's ONLY failure was the disaggregated NIXL KV-transfer stall between +# the prefill and decode workers; running AGGREGATED (single worker, decode +# num-worker 0) removes the P/D split entirely, so there is no NIXL transfer to +# stall on. TP8 (not TP16) preserves the FP8 128-column block divisibility +# (moe_intermediate 3072 / 8 = 384, divisible by 128; / 16 = 192 is not). +# Node allocation for this framework is tp*pp/gpus_per_node = 8*2/8 = 2 nodes; +# EP and DP-attention do NOT contribute GPUs, which is why the earlier DP32 +# aggregated attempt collapsed onto a single node. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp8" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + block-size: 256 + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + enable-expert-parallel: true + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 + max-model-len: 2304 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + # CUDA graphs (not eager): the eager run served correctly but decode ran + # at ~144ms/token (Python dispatch-bound, flat vs concurrency), so the + # conc sweep could not finish in 8h. On-cluster prototype (job 11256) + # confirmed FULL_AND_PIECEWISE graphs FIT this 2-node TP8xPP2 layout at + # gpu-mem-util 0.92 and drop TPOT to ~15ms (~9.4x). No extra nodes needed. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + reasoning-parser: deepseek_v4 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.21.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml new file mode 100644 index 0000000000..b55cdc4b13 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml @@ -0,0 +1,103 @@ +name: "dsv4-vllm-agg-h100-tep8pp2-8k1k" + +# DeepSeek-V4-Pro FP8 H100 AGGREGATED TP8 x PP2 recipe (2 nodes / 16 GPUs). +# The ~1TB FP8 checkpoint does not fit one 8xH100 node (640GB). TP8 x PP2 +# shards attention/dense (/8) and layers (/2) across 16 GPUs -- the original +# memory-correct sizing that loaded the model and reached prefill. That +# topology's ONLY failure was the disaggregated NIXL KV-transfer stall between +# the prefill and decode workers; running AGGREGATED (single worker, decode +# num-worker 0) removes the P/D split entirely, so there is no NIXL transfer to +# stall on. TP8 (not TP16) preserves the FP8 128-column block divisibility +# (moe_intermediate 3072 / 8 = 384, divisible by 128; / 16 = 192 is not). +# Node allocation for this framework is tp*pp/gpus_per_node = 8*2/8 = 2 nodes; +# EP and DP-attention do NOT contribute GPUs, which is why the earlier DP32 +# aggregated attempt collapsed onto a single node. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp8" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + block-size: 256 + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + enable-expert-parallel: true + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 + max-model-len: 9472 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + # CUDA graphs (not eager): the eager run served correctly but decode ran + # at ~144ms/token (Python dispatch-bound, flat vs concurrency), so the + # conc sweep could not finish in 8h. On-cluster prototype (job 11256) + # confirmed FULL_AND_PIECEWISE graphs FIT this 2-node TP8xPP2 layout at + # gpu-mem-util 0.92 and drop TPOT to ~15ms (~9.4x). No extra nodes needed. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + reasoning-parser: deepseek_v4 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + # Capped at 256: conc512 (5120 x 8k-token requests) needs ~3.6h for that one + # level and exceeds the 8h SLURM limit; conc256 completes (~71min) and fits. + concurrencies: "1x2x4x8x16x32x64x128x256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.21.0" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f90c510186..3d99512a8a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4588,6 +4588,80 @@ dsr1-fp8-h100-dynamo-sglang: ep: 16 dp-attn: true +# DeepSeek-V4-Pro FP8 H100 aggregated vLLM via Dynamo (TP8 x PP2, 2 nodes). +# The native checkpoint is mixed FP4+FP8 (~1TB), too large for one 8xH100 node +# (640GB). Prior attempts: TP8xPP2 *disagg* loaded the model and reached prefill +# but hung on the NIXL P/D KV-transfer; DP16/2-node OOM'd; DP32 "aggregated" +# collapsed onto ONE node because this framework sizes nodes as tp*pp/gpus_per_node +# (EP and DP-attention add no GPUs), so tp1 => 1 node and vLLM could not place 32 +# DP ranks on 8 GPUs. The working topology is the memory-proven TP8xPP2 run made +# AGGREGATED (single worker, no P/D split => no NIXL transfer to stall on): tp8*pp2 +# = 16 GPUs = 2 nodes. TP8 (not TP16) keeps FP8 128-column block divisibility. +# Recipes: benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-tep8pp2.yaml. +dsv4-fp8-h100-dynamo-vllm: + image: vllm/vllm-openai:v0.21.0 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + # Runs-on label = this string verbatim. Must be a label a live runner + # carries: h100-disagg has 3 online runners (h100-dgxc-slurm_17-19) and is + # the disaggregated-H100 pool. NOT h100-multinode — no runner carries that + # label, so jobs requesting it queue forever (the dsr1-fp8-h100-dynamo-* + # reference configs above still use it and are non-functional for this). + runner: h100-disagg + precision: fp8 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } + kv-p2p-transfer: nixl + multinode: true + # Aggregated (not P/D-disaggregated): prefill num-worker 1 + decode + # num-worker 0 is how srtslurm signals AGG mode (see RECIPES.md §5). Node + # count = tp*pp/gpus_per_node = 8*2/8 = 2. tp:8 pp:2 with expert-parallel; + # dp-attn false (TP8 attention). Aggregated => no NIXL transfer to stall on. + disagg: true + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + # Aggregated TP8 x PP2 over 2 H100 nodes (16 GPUs), EP8. + - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + prefill: + num-worker: 1 + tp: 8 + pp: 2 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml" + decode: + num-worker: 0 + tp: 8 + pp: 2 + ep: 8 + dp-attn: false + - isl: 8192 + osl: 1024 + search-space: + # Aggregated TP8 x PP2 over 2 H100 nodes (16 GPUs), EP8. + # 8k1k capped at conc256: conc512 (5120 x 8k-token requests) needs ~3.6h + # for that single level and blew the 8h SLURM limit; conc256 completes in + # ~71min and fits with margin. 1k1k keeps 512 (short requests finish fast). + - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256] + prefill: + num-worker: 1 + tp: 8 + pp: 2 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml" + decode: + num-worker: 0 + tp: 8 + pp: 2 + ep: 8 + dp-attn: false + minimaxm3-fp8-h200-vllm: image: vllm/vllm-openai:minimax-m3 model: MiniMaxAI/MiniMax-M3-MXFP8 diff --git a/configs/runners.yaml b/configs/runners.yaml index 93da791b85..135c8fb9e8 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -27,6 +27,13 @@ labels: - h100-dgxc-slurm_17 - h100-dgxc-slurm_18 - h100-dgxc-slurm_19 + # Disaggregated-H100 pool. Unlike h100-multinode (whose key is not a label + # any runner carries, so jobs requesting it queue forever), these three + # runners carry the h100-disagg label, so runs-on: h100-disagg matches. + h100-disagg: + - h100-dgxc-slurm_17 + - h100-dgxc-slurm_18 + - h100-dgxc-slurm_19 h200: - h200-cw_00 - h200-cw_01 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e05c2ca1c4..b92f24cd37 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4751,6 +4751,22 @@ - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 +- config-keys: + - dsv4-fp8-h200-vllm + - dsv4-fp8-h200-vllm-mtp + description: + - "Bump vLLM image from v0.21.0 to v0.25.0 for DeepSeek-V4-Pro FP8 on H200, matching the B200/B300 dsv4 vLLM bump (#2169)" + - "Refresh stale H200 dsv4 submissions (last run 2026-05-21)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2191 + +- config-keys: + - dsv4-fp8-h200-sglang + - dsv4-fp8-h200-sglang-mtp + description: + - "Bump the pinned lmsysorg/sglang:deepseek-v4-hopper digest from the 2026-05-02 push (7f19c6dc) to the current 2026-05-13 push (1bf5d508)" + - "Refresh stale H200 dsv4 SGLang submissions (last run 2026-05-04)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2191 + - config-keys: - dsv4-fp4-mi355x-sglang description: @@ -4765,3 +4781,18 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - dsv4-fp8-h100-dynamo-vllm + description: + - "Add DeepSeek-V4-Pro FP8 H100 aggregated multinode vLLM benchmark via Dynamo (new SKU, previously no dsv4 data on H100)" + - "Native checkpoint is mixed FP4+FP8 (~1TB) and does not fit one 8xH100 node (640GB); vLLM auto-resolves the DSv4 quant config to a Hopper-compatible path (no explicit --quantization), so the only real blocker on H100 is memory, not FP4 support" + - "Aggregated TP8 x PP2 across 2 H100 nodes (16 GPUs) with expert-parallel: tensor-parallel-size 8, pipeline-parallel-size 2, enable-expert-parallel. TP8 shards attention/dense (/8) and PP2 splits layers (/2) -- the original memory-correct sizing that loaded the model and reached prefill. TP8 (not TP16) preserves the FP8 128-column quantization-block divisibility (moe_intermediate 3072/8=384 divisible by 128; /16=192 not)" + - "Native checkpoint quant note: vLLM auto-resolves the DSv4 config to a Hopper-compatible path (no explicit --quantization); the only blocker on H100 is memory, not FP4 support (all failed attempts loaded far enough to fail on placement/transfer, not on FP4)" + - "Supersedes three failed bring-up topologies: (1) TP8xPP2 *disaggregated* loaded the model and reached prefill but hung ~300s on the NIXL P/D KV-transfer then EngineCore died; (2) pure DP16 on 2 nodes OOM'd at model load; (3) DP32 'aggregated' collapsed onto ONE node -- this framework sizes nodes as tp*pp/gpus_per_node and EP/DP-attention add no GPUs, so tp1 => 1 node and vLLM could not place 32 DP ranks on 8 GPUs. Running the memory-proven TP8xPP2 AGGREGATED (single worker, no P/D split) removes the NIXL transfer entirely and allocates 2 nodes deterministically (8*2/8)" + - "Aggregated mode is signalled by prefill num-worker 1 + decode num-worker 0 (RECIPES.md section 5); the full vLLM serve config lives under backend.vllm_config.aggregated in the recipe" + - "CUDA graphs (compilation-config cudagraph_mode FULL_AND_PIECEWISE, custom_ops all) at gpu-memory-utilization 0.92 -- NOT enforce-eager. An eager run served correctly (eval passed) but decode ran at ~144ms/token (Python-dispatch-bound, flat vs concurrency), so the conc sweep could not finish in the 8h SLURM limit (killed mid-conc-128 after ~7h). An on-cluster srtctl prototype (2-node TP8xPP2, 1x8 sweep) confirmed FULL_AND_PIECEWISE graphs FIT this layout (no OOM at 0.92) and cut TPOT to ~15ms (~9.4x); no extra nodes required" + - "One TP8xPP2 topology per seq-len; 1k1k conc 1-512, 8k1k conc 1-256 (8k1k conc512 = 5120 x 8k-token requests needs ~3.6h for that single level and exceeds the 8h SLURM limit; conc256 completes in ~71min). FP8 KV cache, block size 256, max-model-len 2304/9472, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" + - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-tep8pp2.yaml; setup via NVIDIA srt-slurm's standard vllm-container-deps.sh" + - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 1334c95542..29aeab9330 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -16,7 +16,30 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # MODEL_PATH: Override with pre-downloaded paths on H100 runner # The yaml files specify HuggingFace model IDs for portability, but we use # local paths to avoid repeated downloading on the shared H100 cluster. - if [[ $FRAMEWORK == "dynamo-sglang" ]]; then + if [[ $FRAMEWORK == "dynamo-vllm" ]]; then + if [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp8" ]]; then + SELECTED_MODEL_PATH="" + if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then + SELECTED_MODEL_PATH="$MODEL_PATH" + else + for candidate in \ + /mnt/nfs/lustre/models/dsv4-fp8 \ + /mnt/nfs/lustre/models/deepseek-v4-pro \ + /mnt/nfs/lustre/models/dsv4-pro \ + /mnt/nfs/lustre/models/DeepSeek-V4-Pro; do + if [[ -d "$candidate" ]]; then + SELECTED_MODEL_PATH="$candidate" + break + fi + done + fi + export MODEL_PATH="${SELECTED_MODEL_PATH:-/mnt/nfs/lustre/models/dsv4-fp8}" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" + else + echo "Unsupported model prefix/precision for dynamo-vllm: $MODEL_PREFIX/$PRECISION" + exit 1 + fi + elif [[ $FRAMEWORK == "dynamo-sglang" ]]; then if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/mnt/nfs/lustre/models/dsr1-fp8" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" @@ -34,7 +57,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then exit 1 fi else - echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang" + echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang, dynamo-vllm" exit 1 fi @@ -49,6 +72,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" + elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout aflowers/vllm-gb200-v0.20.0 + mkdir -p recipes/vllm/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 + cp \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh" \ + configs/patches/vllm-container-deps.sh else git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" @@ -84,6 +116,23 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # SGLang container mapping SQUASH_FILE="/mnt/nfs/lustre/containers/lmsysorg_sglang_v0.5.8.post1-cu130.sqsh" CONTAINER_KEY="lmsysorg/sglang:v0.5.8-cu130" + elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then + # vLLM container mapping. Import the recipe image once into the shared + # squash directory so every Slurm node uses the exact configured tag. + CONTAINER_KEY="$IMAGE" + SQUASH_FILE="/mnt/nfs/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/+/g').sqsh" + mkdir -p "$(dirname "$SQUASH_FILE")" + ( + exec 9>"${SQUASH_FILE}.lock" + flock -w 1800 9 || { echo "Failed to acquire lock for $SQUASH_FILE" >&2; exit 1; } + if unsquashfs -l "$SQUASH_FILE" >/dev/null 2>&1; then + echo "Squash file already exists and is valid: $SQUASH_FILE" + else + rm -f "$SQUASH_FILE" "${SQUASH_FILE}.tmp."* + enroot import -o "${SQUASH_FILE}.tmp.$$" "docker://$IMAGE" + mv -f "${SQUASH_FILE}.tmp.$$" "$SQUASH_FILE" + fi + ) elif [[ $FRAMEWORK == "dynamo-trt" ]]; then # TRT-LLM container mapping - convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) CONTAINER_KEY=$(echo "$IMAGE" | sed 's|nvcr.io/|nvcr.io#|') @@ -115,6 +164,7 @@ model_paths: containers: dynamo-trtllm: "${SQUASH_FILE}" dynamo-sglang: "${SQUASH_FILE}" + dynamo-vllm: "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" latest: "${SQUASH_FILE}" "${CONTAINER_KEY}": "${SQUASH_FILE}" @@ -143,8 +193,10 @@ EOF # Override the job name in the config file with the runner name sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE" - # Raise sglang's torch-distributed TCPStore timeout from the 600s gloo default - sed -i '/^ watchdog-timeout:/a\ dist-timeout: 1800' "${CONFIG_FILE%%:*}" + # Raise sglang's torch-distributed TCPStore timeout from the 600s gloo default. + if [[ $FRAMEWORK == "dynamo-sglang" ]]; then + sed -i '/^ watchdog-timeout:/a\ dist-timeout: 1800' "${CONFIG_FILE%%:*}" + fi SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h100,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT"