From c2f8a5f93061f729200b24063149e728a710bdfe Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 9 Jul 2026 16:41:24 -0500 Subject: [PATCH 01/37] [AMD][AgentX] Add DeepSeek V4 MI355X agentic disagg --- .../agentic/dsv4_fp4_mi355x_sglang-disagg.sh | 172 ++++++++++++ benchmarks/multi_node/amd_utils/env.sh | 122 ++++++++- benchmarks/multi_node/amd_utils/job.slurm | 93 ++++++- benchmarks/multi_node/amd_utils/models.yaml | 2 +- .../patches/decode_tp_queue_agree.patch | 107 ++++++++ .../multi_node/amd_utils/server_sglang.sh | 258 +++++++++++++++++- benchmarks/multi_node/amd_utils/setup_deps.sh | 249 ++++++++++++----- .../multi_node/amd_utils/trace_replay.sh | 160 +++++++++++ configs/amd-master.yaml | 33 +++ perf-changelog.yaml | 7 + runners/launch_mi355x-amds.sh | 64 ++++- utils/agentic/aggregation/backends/sglang.py | 2 + .../test_process_agentic_result.py | 1 + 13 files changed, 1179 insertions(+), 91 deletions(-) create mode 100755 benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh create mode 100644 benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch create mode 100644 benchmarks/multi_node/amd_utils/trace_replay.sh diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh new file mode 100755 index 0000000000..e5dde8d120 --- /dev/null +++ b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X +# (DeepSeek-V4-Pro FP4, 1P1D TP8). +# +# CI-style sibling of dsr1_fp4_mi355x_sglang-disagg.sh: driven entirely by +# environment variables and submits a SLURM job via submit.sh. The agentic / +# HiCache-offload configuration mirrors the DSR1 recipe but uses DSV4-Pro +# specific flags (dsv4 attention backend, page-size 256, SWA settings). + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../../benchmark_lib.sh" + +check_env_vars \ + CONC_LIST \ + ISL \ + OSL \ + IMAGE \ + SPEC_DECODING \ + MODEL_PATH \ + PREFILL_NUM_WORKERS \ + PREFILL_TP \ + PREFILL_EP \ + PREFILL_DP_ATTN \ + DECODE_NUM_WORKERS \ + DECODE_TP \ + DECODE_EP \ + DECODE_DP_ATTN \ + PREFILL_NODES \ + DECODE_NODES \ + RANDOM_RANGE_RATIO \ + DURATION \ + KV_OFFLOADING \ + IS_AGENTIC \ + FRAMEWORK + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +set -x + +# Use upstreamed multi_node scripts (no external clone needed) +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 + +# Set up SGL launch script-specific environment variables +export TIME_LIMIT="${TIME_LIMIT:-08:00:00}" +export MODEL_PATH=$MODEL_PATH +export MODEL_NAME=$MODEL_NAME +export CONTAINER_IMAGE=$IMAGE + +# ── Identity / result naming ── +export MODEL_PREFIX="${MODEL_PREFIX:-dsv4}" +export PRECISION="${PRECISION:-fp4}" +export RESULT_FILENAME="${RESULT_FILENAME:-${RUNNER_NAME:-dsv4-fp4-agentic}}" + +# ── Agentic benchmark params ── +export DURATION="${DURATION:-1800}" +# DSV4-Pro max model len for agentic traces (matches single-node recipe). +export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1000000}" + +# ── In-tree sglang patches ── +# mori_conn.py targets hybrid-state bugs (GLM-5, Qwen3.5). DSV4-Pro uses a +# pure MoE/DSA architecture without hybrid state; skip to avoid interference. +export MORI_CONN_PATCH="${MORI_CONN_PATCH:-skip}" + +# ── Aiter fault mitigation ── +# --disable-custom-all-reduce avoids a known aiter fault on MI355X. +export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}" + +# ── KV cache offloading (HiCache) ── +# KV_OFFLOADING=none | dram (passed from YAML; default none for disagg). +# KV_OFFLOAD_BACKEND selects the backend when offloading is on; this recipe +# only implements HiCache, so "hicache" is the only supported value. +# HICACHE_TIER: L2 -> GPU + CPU-DRAM host pool. L3 -> + Mooncake store. +export KV_OFFLOADING="${KV_OFFLOADING:-none}" +if [[ "$KV_OFFLOADING" != "none" ]]; then + export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-hicache}" +fi +# HiCache/Mooncake tunables only matter when KV offloading is enabled. +if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then + export HICACHE_TIER="${HICACHE_TIER:-L2}" + export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" + # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. + export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}" + # HiCache ratio (host pool = ratio * GPU KV pool). Default derived in server_sglang.sh. + export HICACHE_RATIO="${HICACHE_RATIO:-}" + + # ── HiCache layout/backend by tier ── + # L3 (Mooncake): page_first + direct + write_through + storage=mooncake + # L2 (CPU DRAM): layer_first + direct + write_through_selective + storage=none + # NOTE: write_through_selective evicts only under GPU memory pressure, avoiding + # the mori RDMA race that causes GPU memory access faults with write_through. + if [[ "${HICACHE_TIER^^}" == "L3" ]]; then + export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-mooncake}" + else + export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" + fi + export HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-best_effort}" + # Shared nodes: use non-default Mooncake ports to avoid collisions. + export MC_MASTER_PORT="${MC_MASTER_PORT:-58137}" + export MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" + export MC_METRICS_PORT="${MC_METRICS_PORT:-19003}" + export MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" + export MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" + export MC_PATCH_HOSTPOOL="${MC_PATCH_HOSTPOOL:-1}" + export MC_PROTOCOL="${MC_PROTOCOL:-tcp}" + export MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" + export MC_DEVICE="${MC_DEVICE:-}" + export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}" + export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}" +fi + +# ── MoRIIO RDMA Send Queue tuning ── +export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" +export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-32768}" + +# ── SGLang PD router policy + server metrics ── +export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-cache_aware}" +export ENABLE_METRICS="${ENABLE_METRICS:-1}" + +# ── MTP ── +export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}" + +# Derive EP/DP enable flags from the topology inputs. +if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then +export PREFILL_ENABLE_EP=false +else +export PREFILL_ENABLE_EP=true +fi + +if [[ "$PREFILL_DP_ATTN" == "true" ]]; then +export PREFILL_ENABLE_DP=true +else +export PREFILL_ENABLE_DP=false +fi + +if [[ "${DECODE_EP:-1}" -eq 1 ]]; then +export DECODE_ENABLE_EP=false +else +export DECODE_ENABLE_EP=true +fi + +if [[ "$DECODE_DP_ATTN" == "true" ]]; then +export DECODE_ENABLE_DP=true +else +export DECODE_ENABLE_DP=false +fi + +# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'. +JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ + $PREFILL_NUM_WORKERS \ + $DECODE_NODES \ + $DECODE_NUM_WORKERS \ + $ISL $OSL "${CONC_LIST// /x}" inf \ + ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ + ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ + ${PREFILL_TP} ${DECODE_TP} \ + ${RANDOM_RANGE_RATIO}) + +if [[ $? -ne 0 ]]; then + echo "Failed to submit job" >&2 + exit 1 +fi + +echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index c75675cb1c..966cde4589 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -12,6 +12,23 @@ set -x ENGINE="${ENGINE:-sglang-disagg}" export PYTHONDONTWRITEBYTECODE=1 +# ============================================================================= +# HiCache / Mooncake settings from job.slurm +# ============================================================================= +# job.slurm writes the recipe-provided HiCache/Mooncake tunables to +# hicache_mc_.env and mounts it read-only at /config/hicache_mc.env. Source +# it here (auto-export) so values like HICACHE_PAGE_SIZE=256 reach the container +# before server_sglang.sh applies its "${VAR:-default}" fallbacks. Without this +# the vars arrive unset and server_sglang.sh defaults HICACHE_PAGE_SIZE to 1, +# overriding the recipe's --page-size. Empty values in the file are harmless: +# the "${VAR:-default}" fallbacks still treat "" as unset. +if [[ -f /config/hicache_mc.env ]]; then + set -a + source /config/hicache_mc.env + set +a + echo "[env.sh] sourced HiCache config from /config/hicache_mc.env (HICACHE_PAGE_SIZE=${HICACHE_PAGE_SIZE:-unset})" +fi + # ============================================================================= # Shared: IBDEVICES detection # ============================================================================= @@ -50,11 +67,11 @@ export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES} # ============================================================================= # Shared by the vLLM MoRIIOConnector and the SGLang/MoRI KV-transfer path. -export MORI_IO_SQ_BACKOFF_TIMEOUT_US=50000 -export MORI_IO_QP_MAX_SEND_WR=16384 -export MORI_IO_QP_MAX_CQE=32768 -export MORI_IO_QP_MAX_SGE=2 -export MORI_IO_TC_DISABLE=0 +export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-50000}" +export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" +export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-32768}" +export MORI_IO_QP_MAX_SGE="${MORI_IO_QP_MAX_SGE:-2}" +export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}" # QoS/DSCP configuration # Priority order: 1) Set by runner, 2) Detect via nicctl, 3) Detect from hostname @@ -197,14 +214,27 @@ else export AITER_LOG_LEVEL=ERROR export SGLANG_MORI_DISPATCH_DTYPE=auto - export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast - export MORI_COMBINE_DTYPE_DECODE=fp8 + # export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast + # export MORI_COMBINE_DTYPE_DECODE=fp8 + export MORI_COMBINE_DTYPE_PREFILL="" + export MORI_COMBINE_DTYPE_DECODE="" export SGLANG_MORI_QP_PER_TRANSFER=4 export SGLANG_MORI_NUM_WORKERS=4 + # Keep these as overridable defaults (not hard assignments), otherwise + # later tuning blocks cannot raise them for high-concurrency runs. + # export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" + + # export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}" + # export MORI_IO_QP_MAX_CQE=32768 + # export MORI_IO_QP_MAX_SGE=1 + + # export MORI_IO_TC_DISABLE=0 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 + export SGLANG_HEALTH_CHECK_TIMEOUT=600 + # GLM-5: uses NSA (not MLA), needs fused-decode-MLA disabled + fast loading if [[ "$MODEL_NAME" == "GLM-5-FP8" ]]; then export SGLANG_ROCM_FUSED_DECODE_MLA=0 @@ -300,4 +330,80 @@ else # FIXME: WA for latest upstream 0305 image export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH} -fi + # ========================================================================= + # DeepSeek-V4-Pro PD recipe overrides + # Placed at the end of the SGLang env block so it wins over the global + # MoRI/SGLang defaults set above. Mirrors the validated DSv4 manual PD + # commands (ported from InferenceX amd/dsv4_sgl_di). These SGLANG_OPT_* / + # AITER_* kernel-routing knobs steer DSv4 away from the default aiter CK + # fused-MoE path, which raises "Unsupported kernel config for moe heuristic + # dispatch" at decode time on this fp4 model (job 19034 crash). Only the + # SGLang/MoRI env knobs are pinned here; CLI flags live in models.yaml and + # the cluster NIC/socket vars stay runner-derived. + # ========================================================================= + if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then + # MoRI RDMA send-queue depth for DSv4 (overrides the global default above). + export MORI_IO_QP_MAX_SEND_WR=32767 + # Unified radix tree: cache impl with per-component (full-attn / SWA) + # management for hybrid-attention models. Set unconditionally (not gated on + # hicache) so all SGLang runs use it. + export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 + # Proactively free out-of-window SWA KV slots during chunked prefill. + # Without it, in-flight requests pin SWA KV for their whole context, keeping + # the SWA pool under constant eviction pressure; under LRU the trailing + # window of cached sessions gets flushed, making prefix-cache hits bimodal + # and collapsing the effective hit rate on multi-turn agentic workloads. + export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 + + # MoRI dispatch/combine dtypes: auto for both roles (not the fp8 split default) + export SGLANG_MORI_DISPATCH_DTYPE=auto + export MORI_COMBINE_DTYPE_PREFILL=auto + export MORI_COMBINE_DTYPE_DECODE=auto + + # Per-role MoRI dispatch sizing (used by the harness chunked/MoE math) + export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 + export MORI_MAX_DISPATCH_TOKENS_DECODE=64 + unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL + unset MORI_MOE_MAX_INPUT_TOKENS_DECODE + + # PER_RANK dispatch tokens pinned independently (16384 prefill / 128 + # decode); server_sglang.sh prefers these over the MORI_MAX_DISPATCH_* + # coupling when set. + export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL=16384 + export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE=128 + + # Fixed inter-kernel switch threshold (not derived). + export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=4096 + + # Overlap plan stream on for DSv4 (global default is 0) + # export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 + + # DSv4 model kernel routing (mirrors the single-node / manual PD recipe) + export SGLANG_DEFAULT_THINKING=1 + export SGLANG_DSV4_REASONING_EFFORT=max + export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false + export SGLANG_USE_AITER=1 + export SGLANG_USE_ROCM700A=0 + export SGLANG_OPT_USE_FUSED_COMPRESS=true + export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton + export SGLANG_OPT_FP8_WO_A_GEMM=false + export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false + export SGLANG_OPT_USE_TOPK_V2=false + export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true} + export SGLANG_OPT_USE_TILELANG_INDEXER=false + export SGLANG_OPT_USE_TILELANG_MHC_PRE=false + export SGLANG_OPT_USE_TILELANG_MHC_POST=false + export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 + export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true + export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false + export SGLANG_ROCM_USE_MULTI_STREAM=false + export AITER_BF16_FP8_MOE_BOUND=0 + export SGLANG_EAGER_INPUT_NO_COPY=true + export SGLANG_SHARED_EXPERT_TP1=1 + export SGLANG_DP_SHARED_EXPERT_LOCAL=1 + export SGLANG_DP_USE_GATHERV=1 + export SGLANG_DP_USE_REDUCE_SCATTER=1 + export GPU_MAX_HW_QUEUES=5 + fi + +fi \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 4638e5cadb..53d7e2bfcd 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -179,11 +179,41 @@ else fi } - if check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then + # Extract hf_dir from models.yaml (same as vllm-disagg path above) + SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} + found && /^[^ ]/{exit} + found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") + SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}" + + # Prefer the caller-supplied MODEL_PATH (recipe scripts set this explicitly); + # fall back to MODEL_DIR/hf_dir then MODEL_DIR/MODEL_NAME. + if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL_DIR" ]]; then + # Caller already resolved the path (e.g. MODEL_PATH=/it-share/hf_cache/models--...) + # Use it directly if it exists on all nodes, otherwise try subdirectory combos. + if check_model_path "$MODEL_PATH" "MODEL_PATH (caller-supplied)"; then + echo "Selected MODEL_PATH: $MODEL_PATH (caller-supplied, available on all nodes)" + elif check_model_path "$MODEL_PATH/$SGL_DISK_DIR_NAME" "$MODEL_PATH/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_PATH/$SGL_DISK_DIR_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + elif check_model_path "$MODEL_PATH/$MODEL_NAME" "$MODEL_PATH/$MODEL_NAME"; then + MODEL_PATH="$MODEL_PATH/$MODEL_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + else + echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" + echo " - $MODEL_PATH" + echo " - $MODEL_PATH/$SGL_DISK_DIR_NAME" + echo " - $MODEL_PATH/$MODEL_NAME" + exit 1 + fi + elif check_model_path "$MODEL_DIR/$SGL_DISK_DIR_NAME" "$MODEL_DIR/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_DIR/$SGL_DISK_DIR_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" + elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then MODEL_PATH="$MODEL_DIR/$MODEL_NAME" echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" else echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" + echo " - $MODEL_DIR/$SGL_DISK_DIR_NAME" echo " - $MODEL_DIR/$MODEL_NAME" exit 1 fi @@ -350,7 +380,6 @@ DOCKER_ENV_COMMON=( -e BENCH_MAX_CONCURRENCY=\$BENCH_MAX_CONCURRENCY -e BENCH_REQUEST_RATE=\$BENCH_REQUEST_RATE -e TQDM_MININTERVAL=\$TQDM_MININTERVAL - -e DRY_RUN=\$DRY_RUN -e BENCHMARK_LOGS_DIR=/benchmark_logs -e ENGINE=\$ENGINE -e WS_PATH=${WS_PATH} @@ -363,14 +392,44 @@ DOCKER_ENV_COMMON=( -e RUNNER_TYPE=\$RUNNER_TYPE -e RESULT_FILENAME=\$RESULT_FILENAME -e SPEC_DECODING=\$SPEC_DECODING + # DISAGG was never forwarded into the container at all (not even under a + # different name), so process_agentic_result.py's env_bool("DISAGG") always + # defaulted to false in the result JSON regardless of the actual topology. + -e DISAGG=\${DISAGG:-false} -e PREFILL_TP_SIZE=\$PREFILL_TP_SIZE + # PREFILL_TP/DECODE_TP/*_NUM_WORKERS (below, undecorated -- distinct from the + # *_SIZE vars server_sglang.sh uses for launch args) are what + # process_agentic_result.py's _gpu_shape() reads for multinode runs. Without + # these the container never sees them (only *_SIZE was passed), so agentic + # result JSONs silently recorded tp=0 / prefill_tp=0 / prefill_num_workers=0 + # for every multinode run. + -e PREFILL_TP=\$PREFILL_TP + -e PREFILL_NUM_WORKERS=\$PREFILL_NUM_WORKERS -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP -e PREFILL_ENABLE_DP=\$PREFILL_ENABLE_DP + -e PREFILL_CONTEXT_LENGTH=\${PREFILL_CONTEXT_LENGTH:-} + -e PREFILL_CHUNKED_PREFILL_SIZE=\${PREFILL_CHUNKED_PREFILL_SIZE:-} + -e SGLANG_AITER_MLA_PERSIST=\${SGLANG_AITER_MLA_PERSIST:-0} + -e DISABLE_CUSTOM_ALL_REDUCE=\${DISABLE_CUSTOM_ALL_REDUCE:-0} + -e MAX_MODEL_LEN=\${MAX_MODEL_LEN:-} + -e DURATION=\${DURATION:-1800} + -e IS_AGENTIC=\${IS_AGENTIC:-0} + -e KV_OFFLOADING=\${KV_OFFLOADING:-none} + -e KV_OFFLOAD_BACKEND=\${KV_OFFLOAD_BACKEND:-} + -e TOTAL_CPU_DRAM_GB=\${TOTAL_CPU_DRAM_GB:-} + -e ENABLE_METRICS=\${ENABLE_METRICS:-0} + -e PREFILL_ROUTER_POLICY=\${PREFILL_ROUTER_POLICY:-random} + -e DECODE_ROUTER_POLICY=\${DECODE_ROUTER_POLICY:-random} + -e MORI_IO_SQ_BACKOFF_TIMEOUT_US=\${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-} + -e MORI_IO_QP_MAX_SEND_WR=\${MORI_IO_QP_MAX_SEND_WR:-} -e DECODE_TP_SIZE=\$DECODE_TP_SIZE + -e DECODE_TP=\$DECODE_TP + -e DECODE_NUM_WORKERS=\$DECODE_NUM_WORKERS -e DECODE_ENABLE_EP=\$DECODE_ENABLE_EP -e DECODE_ENABLE_DP=\$DECODE_ENABLE_DP -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE -e IS_MULTINODE=\$IS_MULTINODE + -e DRY_RUN=\${DRY_RUN:-0} ) # Engine-specific env vars @@ -411,6 +470,35 @@ else ) fi +# HiCache / Mooncake settings are delivered via a bind-mounted config file rather +# than a long list of docker -e flags. Write it once to the shared benchmark-logs +# dir (already a host path, visible on every node) and mount it read-only at +# /config/hicache_mc.env, where env.sh sources it before applying its defaults. +# Empty values are preserved so env.sh's "${VAR:-default}" fallbacks still apply. +HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env" +cat > "$HICACHE_MC_CONFIG" < $HICACHE_MC_CONFIG" + # Engine-specific container filter for pre-clean CONT_FILTER="name=^container_${ENGINE}_" @@ -572,6 +660,7 @@ fi -v /tmp:/run_logs \ -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ + -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ ${EXTRA_DOCKER_MOUNTS:-} \ \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ ${DOCKER_ENV_COMMON[*]} \ diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index bc2b39aa07..24a588375d 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -372,7 +372,7 @@ DeepSeek-R1-0528-MXFP4-v2: # prefill.disable_cuda_graph routes prefill to --disable-cuda-graph; decode keeps # --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md. DeepSeek-V4-Pro: - base_flags: "--decode-log-interval 100 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori" + base_flags: "--decode-log-interval 100 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori" dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" prefill: diff --git a/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch b/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch new file mode 100644 index 0000000000..e09c02112f --- /dev/null +++ b/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch @@ -0,0 +1,107 @@ +--- a/python/sglang/srt/disaggregation/decode.py 2026-06-25 02:44:11.274035348 +0000 ++++ b/python/sglang/srt/disaggregation/decode.py 2026-06-25 02:44:11.276035310 +0000 +@@ -1616,8 +1616,57 @@ + ) + kv_manager._staging_handler = self.staging_handler + ++ def _agree_and_order_queue(self) -> Tuple[List["DecodeRequest"], List["DecodeRequest"]]: ++ """Split self.queue into (gated, deferred) using a tp-group agreement. ++ ++ The metadata gate (utils.poll_and_all_reduce) issues a tp-group all_reduce ++ whose tensor shape == len(self.queue) and whose per-index meaning is the ++ i-th queued request. That is only correct if every tp rank polls an ++ IDENTICAL queue (same requests, same order). Per-rank prealloc/enqueue ++ timing can leave queues with different membership or order, which desyncs ++ the collective -> deadlock (the JID-17417 decode hang). ++ ++ We all_gather the local request ids, keep only requests present on EVERY ++ rank, and order them deterministically (request ids are rank-invariant), so ++ the subsequent gate runs a matching collective on all ranks. Requests not ++ yet on every rank are returned as `deferred` and retried on a later call. ++ ++ NOTE: every rank that reaches pop_transferred must call this (it contains a ++ collective). That holds for the same reason the existing gate is safe: ++ pop_transferred is entered rank-symmetrically over self.gloo_group. ++ """ ++ tp_size = torch.distributed.get_world_size(self.gloo_group) ++ if tp_size <= 1: ++ return self.queue, [] ++ ++ local_rids = [dr.req.rid for dr in self.queue] ++ gathered: List[Optional[List[str]]] = [None] * tp_size ++ torch.distributed.all_gather_object( ++ gathered, local_rids, group=self.gloo_group ++ ) ++ ++ common = set(gathered[0] or []) ++ for rids in gathered[1:]: ++ common &= set(rids or []) ++ ++ if not common: ++ return [], list(self.queue) ++ ++ by_rid = {dr.req.rid: dr for dr in self.queue} ++ # sorted() yields an identical order on every rank (rids are rank-invariant). ++ gated = [by_rid[rid] for rid in sorted(common)] ++ deferred = [dr for dr in self.queue if dr.req.rid not in common] ++ return gated, deferred ++ + def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: ++ # Agree on a tp-rank-identical, identically-ordered subset BEFORE issuing any ++ # metadata-gate collective. Do NOT add a local `if not self.queue` early ++ # return here: an empty-queue rank must still join the agreement collective, ++ # otherwise it skips it and desyncs the tp group (root cause of the hang). ++ self.queue, _deferred = self._agree_and_order_queue() + if not self.queue: ++ # Empty agreement is rank-symmetric: all ranks skip the gate together. ++ self.queue = _deferred + return [] + + if self.scheduler.enable_decode_hicache: +@@ -1731,6 +1780,10 @@ + self.queue = [ + entry for i, entry in enumerate(self.queue) if i not in indices_to_remove + ] ++ # Re-attach requests that were not yet present on every tp rank this ++ # iteration; they are gated again on a later call once all ranks have them. ++ if _deferred: ++ self.queue.extend(_deferred) + + return transferred_reqs + +@@ -1946,8 +1999,33 @@ + # try to resume retracted requests if there are enough space for another `num_reserved_decode_tokens` decode steps + resumed_reqs = self.disagg_decode_prealloc_queue.resume_retracted_reqs() + self.waiting_queue.extend(resumed_reqs) +- if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0: +- # if there are still retracted requests, we do not allocate new requests ++ # PATCH(call-site tp-agreement): the retracted-queue early return below is ++ # rank-divergent — retracted_queue length is per-rank (per-rank KV-cache ++ # pressure). A divergent return skips the polling_count increment and the ++ # pop_transferred() collectives on some ranks, permanently desyncing the tp ++ # group: one rank ends up a full collective ahead, so pop_transferred's ++ # _agree_and_order_queue all_gather_object (decode.py:1644) on the lagging ++ # ranks lines up against the metadata-gate all_reduce (decode.py:1591) on ++ # the leading rank -> mismatched-collective deadlock (GPU 0%, detokenizer ++ # heartbeat freeze; JID-17445). process_decode_queue is called ++ # unconditionally every event-loop iteration on every rank, so an all_reduce ++ # here (before any divergent return) is rank-symmetric. Hold off new ++ # allocation iff ANY rank still has retracted reqs, so all ranks branch ++ # identically every iteration and pop_transferred is entered symmetrically. ++ _agree_gg = self.disagg_decode_transfer_queue.gloo_group ++ _local_retracted = ( ++ 1 if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0 else 0 ++ ) ++ if torch.distributed.get_world_size(_agree_gg) > 1: ++ _retracted_t = torch.tensor([_local_retracted], dtype=torch.int32) ++ torch.distributed.all_reduce( ++ _retracted_t, op=torch.distributed.ReduceOp.MAX, group=_agree_gg ++ ) ++ _any_retracted = bool(_retracted_t.item()) ++ else: ++ _any_retracted = bool(_local_retracted) ++ if _any_retracted: ++ # if any rank still has retracted requests, no rank allocates new ones + return + + if not hasattr(self, "polling_count"): diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index ec5805eb0e..05fdc7dbb3 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -146,6 +146,7 @@ print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') print(f'DECODE_MEM_FRACTION_STATIC=\"{decode.get(\"mem_fraction_static\", 0.85)}\"') print(f'DECODE_PREFILL_ROUND_ROBIN_BALANCE=\"{decode.get(\"prefill_round_robin_balance\", True)}\"') +print(f'DECODE_DISAGG_ENABLE_RADIX_CACHE=\"{decode.get(\"disagg_decode_enable_radix_cache\", False)}\"') dp = decode.get('dp', {}) ep_only = decode.get('ep_only', {}) @@ -235,6 +236,10 @@ fi if [[ "$PREFILL_DISABLE_RADIX_CACHE" == "True" ]] || [[ "$PREFILL_DISABLE_RADIX_CACHE" == "true" ]]; then PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --disable-radix-cache" fi +# Agentic runs: keep radix/prefix cache enabled by replacing --disable-radix-cache with empty. +if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + PREFILL_MODE_FLAGS="${PREFILL_MODE_FLAGS//--disable-radix-cache/}" +fi if [[ -n "$prefill_context_length" ]]; then PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --context-length ${prefill_context_length}" fi @@ -251,6 +256,9 @@ DECODE_MODE_FLAGS="--mem-fraction-static ${DECODE_MEM_FRACTION_STATIC} --max-run if [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "True" ]] || [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "true" ]]; then DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --prefill-round-robin-balance" fi +if [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "True" ]] || [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "true" ]]; then + DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --disaggregation-decode-enable-radix-cache" +fi if [[ "$DECODE_MTP_SIZE" -gt 0 ]]; then MORI_MAX_DISPATCH_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) @@ -270,10 +278,20 @@ NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) # Build prefill arguments dynamically based on xP PREFILL_HEADNODE_URLS=() PREFILL_ARGS="" +# Per-worker Prometheus /metrics endpoints (port 8000) for aiperf's +# --server-metrics scrape. The router on :30000 does not serve Prometheus, so +# aiperf must scrape each prefill/decode worker directly (see ENABLE_METRICS). +SERVER_METRICS_URLS=() +# Per-worker base URLs (port 8000) for direct cache flushing between +# concurrency points. The router (:30000) does not fan /flush_cache out, so +# trace_replay.sh must POST to each prefill/decode worker directly. +SERVER_FLUSH_URLS=() for i in $(seq 0 $((xP - 1))); do prefill_idx=$((i * PREFILL_NODES_PER_WORKER)) PREFILL_HEADNODE_URLS[$i]="${IP_ARRAY[$prefill_idx]}:${HEADNODE_PORT}" PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$prefill_idx]}:8000" + SERVER_METRICS_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000/metrics") + SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000") done # Build decode arguments dynamically based on yD @@ -283,10 +301,14 @@ for i in $(seq 0 $((yD - 1))); do decode_idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) DECODE_HEADNODE_URLS[$i]="${IP_ARRAY[$decode_idx]}:${HEADNODE_PORT}" DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$decode_idx]}:8000" + SERVER_METRICS_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000/metrics") + SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000") done echo "Prefill worker headnode list: ${PREFILL_HEADNODE_URLS[@]}" echo "Decode worker headnode list: ${DECODE_HEADNODE_URLS[@]}" +echo "Server metrics endpoints: ${SERVER_METRICS_URLS[@]}" +echo "Server flush endpoints: ${SERVER_FLUSH_URLS[@]}" # ============================================================================= # Configuration Builder Functions @@ -354,7 +376,7 @@ build_server_config() { specific_config="$DECODE_MODE_FLAGS" fi - # Combine: parallel args + base config + mtp config (decode only) + dp config + specific config + # Combine: parallel args + base config + ep config + mtp config (decode only) + dp config + specific config local full_config="$parallel_args" if [[ -n "$base_config" ]]; then full_config="$full_config $base_config" @@ -379,10 +401,101 @@ build_server_config() { PREFILL_SERVER_CONFIG=$(build_server_config "prefill" "$MODEL_NAME" "$PREFILL_TP_SIZE" "$PREFILL_ENABLE_EP" "$PREFILL_ENABLE_DP" "$DECODE_MTP_SIZE") DECODE_SERVER_CONFIG=$(build_server_config "decode" "$MODEL_NAME" "$DECODE_TP_SIZE" "$DECODE_ENABLE_EP" "$DECODE_ENABLE_DP" "$DECODE_MTP_SIZE") +# Expose Prometheus /metrics on the servers when requested (ENABLE_METRICS=1). +if [[ "${ENABLE_METRICS:-0}" == "1" ]]; then + [[ "$PREFILL_SERVER_CONFIG" != *"--enable-metrics"* ]] && PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG --enable-metrics" + [[ "$DECODE_SERVER_CONFIG" != *"--enable-metrics"* ]] && DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --enable-metrics" +fi + if [[ -n "$MODEL_NAME" ]]; then echo "Using model-specific configuration for: $MODEL_NAME" fi +# ============================================================================= +# Optional KV cache offloading (HiCache) — enabled when +# KV_OFFLOADING != none AND KV_OFFLOAD_BACKEND == hicache. +# HiCache extends RadixAttention, so radix cache MUST stay on (drop +# --disable-radix-cache). The --hicache-* flags are appended to BOTH the +# prefill and decode server configs. +# ============================================================================= +KV_OFFLOADING="${KV_OFFLOADING:-none}" +KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-}" +if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then + HICACHE_TOTAL_CPU_DRAM_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-2000}" + HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" + HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" + HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-wait_complete}" + + # Optional L3 storage tier behind the CPU-DRAM (L2) cache. + # "" -> CPU DRAM only (default) + # "mooncake"-> Mooncake distributed KV store (needs a mooncake_master) + HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" + + # Layout / IO backend / write policy are backend-specific: + # mooncake L3: page_first_direct + the "direct" IO backend (the Mooncake + # store maps a page-contiguous segment for RDMA/zero-copy). This layout + # asserts host_pool > device_pool, so it needs a large CPU-DRAM budget. + # L2-only (CPU DRAM): layer_first + the "kernel" IO backend. layer_first + # has no host>device constraint (the "direct" IO backend REQUIRES a + # page_first layout, so it cannot be paired with layer_first). + if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + else + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + fi + + # Mooncake master/connection settings (used only when storage=mooncake). + # The master runs once on node 0; every prefill/decode server connects to + # it via NODE0_ADDR so it is reachable across nodes. + MC_MASTER_PORT="${MC_MASTER_PORT:-50061}" + MC_METADATA_PORT="${MC_METADATA_PORT:-8080}" + MC_METRICS_PORT="${MC_METRICS_PORT:-9003}" + MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}" + MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}" + MC_PROTOCOL="${MC_PROTOCOL:-tcp}" + MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}" + MC_DEVICE="${MC_DEVICE:-$IBDEVICES}" + MC_MASTER_ADDR="${MC_MASTER_ADDR:-${NODE0_ADDR}:${MC_MASTER_PORT}}" + MC_METADATA_SERVER="${MC_METADATA_SERVER:-http://${NODE0_ADDR}:${MC_METADATA_PORT}/metadata}" + + # Emit the --hicache-storage-backend flags (empty unless mooncake). The + # extra-config JSON is single-quoted so it survives the later `eval` of the + # launch command as a single argument. + build_storage_flags() { + [[ "$HICACHE_STORAGE_BACKEND" != "mooncake" ]] && return 0 + local extra="{\"master_server_address\": \"${MC_MASTER_ADDR}\", \"protocol\": \"${MC_PROTOCOL}\", \"device_name\": \"${MC_DEVICE}\", \"local_hostname\": \"${host_ip}\", \"global_segment_size\": \"${MC_GLOBAL_SEG}\", \"metadata_server\": \"${MC_METADATA_SERVER}\", \"check_server\": false}" + echo "--hicache-storage-backend mooncake --hicache-storage-backend-extra-config '${extra}' --enable-metrics --enable-cache-report" + } + + # HiCache capacity via --hicache-ratio (scales with GPU KV pool). + HICACHE_RATIO="${HICACHE_RATIO:-16}" + + build_hicache_flags() { + echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache --hicache-ratio ${HICACHE_RATIO} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" + } + + # HiCache requires RadixAttention; strip any --disable-radix-cache. + PREFILL_SERVER_CONFIG="${PREFILL_SERVER_CONFIG//--disable-radix-cache/}" + DECODE_SERVER_CONFIG="${DECODE_SERVER_CONFIG//--disable-radix-cache/}" + + # Prefill always gets HiCache. + PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG $(build_hicache_flags "$PREFILL_TP_SIZE")" + + + DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --page-size ${HICACHE_PAGE_SIZE}" + echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND} applied to prefill only; decode mirrors --page-size ${HICACHE_PAGE_SIZE} for transfer compatibility (chunk cache under the mori transfer backend)" + echo "[HiCache] params: io_backend=${HICACHE_IO_BACKEND}, mem_layout=${HICACHE_MEM_LAYOUT}, page_size=${HICACHE_PAGE_SIZE}, write_policy=${HICACHE_WRITE_POLICY}, prefetch_policy=${HICACHE_PREFETCH_POLICY}, storage_backend=${HICACHE_STORAGE_BACKEND:-none}" + if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then + echo "[HiCache] Mooncake store: master=${MC_MASTER_ADDR} metadata=${MC_METADATA_SERVER} protocol=${MC_PROTOCOL} device=${MC_DEVICE} segment=${MC_GLOBAL_SEG} threads=${MC_MASTER_THREADS} eviction_watermark=${MC_EVICTION_HIGH_WATERMARK}" + fi +else + echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND:-none} (HiCache disabled)" +fi + if [[ "${EVAL_ONLY:-false}" == "true" ]] || [[ "${RUN_EVAL:-false}" == "true" ]]; then PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') @@ -449,6 +562,63 @@ if [ "$NODE_RANK" -eq 0 ]; then echo "================================================" + # Dump all resolved commands to a text file for debugging / reproducibility. + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + echo "# Model: ${MODEL_NAME} Image: ${DOCKER_IMAGE_NAME:-unknown}" >> "$CMD_DUMP" + + # Start the Mooncake store master (L3 HiCache backend) on node 0 only. + # All prefill/decode servers connect to it via NODE0_ADDR:MC_MASTER_PORT. + if [[ "${KV_OFFLOADING:-none}" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" && "${HICACHE_STORAGE_BACKEND:-}" == "mooncake" ]]; then + echo "Starting Mooncake master on ${host_ip}:${MC_MASTER_PORT} (metadata :${MC_METADATA_PORT}, metrics :${MC_METRICS_PORT})" + MC_MASTER_CMD="mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_host=0.0.0.0 \ + --http_metadata_server_port=${MC_METADATA_PORT} \ + --rpc_port=${MC_MASTER_PORT} \ + --rpc_thread_num=${MC_MASTER_THREADS} \ + --metrics_port=${MC_METRICS_PORT} \ + --enable_metric_reporting=true \ + --eviction_high_watermark_ratio=${MC_EVICTION_HIGH_WATERMARK}" + dump_cmd "MOONCAKE MASTER" "$MC_MASTER_CMD" + if [[ "$DRY_RUN" -eq 1 ]]; then + echo "DRY RUN: $MC_MASTER_CMD" + else + MC_MASTER_LOG="/run_logs/slurm_job-${SLURM_JOB_ID}/mooncake_master_${host_name}.log" + mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_host=0.0.0.0 \ + --http_metadata_server_port="${MC_METADATA_PORT}" \ + --rpc_port="${MC_MASTER_PORT}" \ + --rpc_thread_num="${MC_MASTER_THREADS}" \ + --metrics_port="${MC_METRICS_PORT}" \ + --enable_metric_reporting=true \ + --eviction_high_watermark_ratio="${MC_EVICTION_HIGH_WATERMARK}" \ + > "${MC_MASTER_LOG}" 2>&1 & + mc_master_pid=$! + sleep 3 + # Fail loudly on a port collision. On shared nodes the Mooncake RPC + # port may already be taken by another user's master; in that case the + # metrics-port health check below can still pass against the foreign + # master while our RPC port is dead, and the prefill then hangs. + if grep -qiE "Address already in use|bind .*error" "${MC_MASTER_LOG}" 2>/dev/null; then + echo "ERROR: mooncake_master failed to bind port ${MC_MASTER_PORT} (already in use)." + echo " Set MC_MASTER_PORT/MC_METRICS_PORT to free ports and resubmit." + grep -iE "Address already in use|bind .*error" "${MC_MASTER_LOG}" | tail -3 + exit 1 + fi + for ((i=3; i<=60; i+=3)); do + if curl -sf "http://127.0.0.1:${MC_METRICS_PORT}/get_all_segments" >/dev/null 2>&1; then + echo " mooncake master OK at ${i}s" + break + fi + sleep 3 + done + fi + fi + # start the head prefill server PREFILL_MORI_MOE_ENV="" set -x @@ -456,7 +626,7 @@ if [ "$NODE_RANK" -eq 0 ]; then PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" fi set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} python3 -m sglang.launch_server \ + PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ --model-path $MODEL_DIR/$MODEL_NAME \ --disaggregation-mode prefill \ --disaggregation-ib-device ${IBDEVICES} \ @@ -470,6 +640,7 @@ if [ "$NODE_RANK" -eq 0 ]; then fi + dump_cmd "PREFILL (node 0)" "$PREFILL_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $PREFILL_CMD" else @@ -506,16 +677,37 @@ if [ "$NODE_RANK" -eq 0 ]; then fi echo "Congratulations!!! All prefill and decode servers are up . . ." + # Router resilience: a single prefill worker doing huge long-context prefills + # (256K+ token agentic prompts in 65280-token chunks) can be slow to drain a + # concurrent burst. With defaults the circuit breaker opens after 10 failures + # and short-circuits the whole worker, so the aiperf profiling burst sees + # "No available prefill workers (all circuits open or unhealthy)" and aborts + # with 100% errors. Disable the breaker and relax health-check sensitivity so + # a busy-but-alive worker is not ejected. Override via ROUTER_RESILIENCE_FLAGS. + ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}" + + # Router scheduling policy. cache_aware prefill routing exploits HiCache/radix + # prefix reuse across the agentic trace; round_robin decode keeps the single + # decode worker fed evenly. cache_threshold / balance_*_threshold tune the + # cache_aware load-balancing (router defaults are 0.5 / 64 / 1.5). Override any + # of these via env. + ROUTER_PREFILL_POLICY="${ROUTER_PREFILL_POLICY:-cache_aware}" + ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" + ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}" + ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}" + ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}" + ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --prefill-policy ${ROUTER_PREFILL_POLICY} --decode-policy ${ROUTER_DECODE_POLICY} --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" + ROUTER_CMD="python -m sglang_router.launch_router \ --pd-disaggregation \ --port 30000 \ - --policy random \ - --prefill-policy random \ - --decode-policy random \ + ${ROUTER_POLICY_FLAGS} \ + ${ROUTER_RESILIENCE_FLAGS} \ ${PREFILL_ARGS} \ ${DECODE_ARGS}" + dump_cmd "ROUTER" "$ROUTER_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $ROUTER_CMD" else @@ -573,11 +765,41 @@ if [ "$NODE_RANK" -eq 0 ]; then export IS_MTP=false fi - # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier - BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} "${BENCH_MAX_CONCURRENCY}" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" + # Select the benchmark runner. + # IS_AGENTIC=1/true → agentic trace replay (trace_replay.sh) + # IS_AGENTIC unset/0 → fixed-seq-len throughput benchmark (bench.sh) + if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + # Point aiperf's server-metrics scrape at the per-worker Prometheus + # /metrics endpoints. The router (:30000) that aiperf auto-detects from + # --url does not expose Prometheus, so without this the scrape finds no + # reachable endpoint and all server-side cache/KV fields come out null. + # Only set it when the workers were actually started with --enable-metrics. + if [[ "${ENABLE_METRICS:-0}" == "1" && "${#SERVER_METRICS_URLS[@]}" -gt 0 ]]; then + AIPERF_SERVER_METRICS_URLS=$(IFS=,; echo "${SERVER_METRICS_URLS[*]}") + export AIPERF_SERVER_METRICS_URLS + echo "AIPERF_SERVER_METRICS_URLS=${AIPERF_SERVER_METRICS_URLS}" + fi + # Per-worker base URLs for cache flushing between concurrency points. + # trace_replay.sh consults these when CLEAR_CACHE_BETWEEN_CONC=1. + if [[ "${#SERVER_FLUSH_URLS[@]}" -gt 0 ]]; then + SERVER_FLUSH_URLS_CSV=$(IFS=,; echo "${SERVER_FLUSH_URLS[*]}") + export SERVER_FLUSH_URLS_CSV + echo "SERVER_FLUSH_URLS_CSV=${SERVER_FLUSH_URLS_CSV}" + fi + # trace_replay.sh signature: model_path model_name concurrency_list log_path + BENCH_CMD="bash $SGLANG_WS_PATH/trace_replay.sh \ + $MODEL_DIR $MODEL_NAME $BENCH_MAX_CONCURRENCY /run_logs/slurm_job-${SLURM_JOB_ID}" + echo "Benchmark runner: trace_replay.sh (agentic, KV_OFFLOADING=${KV_OFFLOADING:-none}, backend=${KV_OFFLOAD_BACKEND:-none}, CONC=${BENCH_MAX_CONCURRENCY})" + else + # bench.sh signature: + # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path + # isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier + BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ + $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ + ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ + ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" + echo "Benchmark runner: bench.sh (fixed-seq-len)" + fi if [[ "${EVAL_ONLY:-false}" == "true" ]]; then echo "EVAL_ONLY mode: skipping throughput benchmark" @@ -713,13 +935,18 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then echo "Using prefill config: $PREFILL_SERVER_CONFIG" echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}" + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + PREFILL_MORI_MOE_ENV="" set -x if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" fi set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} python3 -m sglang.launch_server \ + PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ --model-path $MODEL_DIR/${MODEL_NAME} \ --disaggregation-mode prefill \ --disaggregation-ib-device ${IBDEVICES} \ @@ -734,6 +961,7 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[$prefill_idx]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank $rank" fi + dump_cmd "PREFILL (rank ${NODE_RANK})" "$PREFILL_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $PREFILL_CMD" else @@ -788,13 +1016,18 @@ else echo "Decode node rank: $RANK" echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}" + CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" + dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } + echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" + echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" + DECODE_MORI_MOE_ENV="" set -x if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_DECODE" ]]; then DECODE_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE}" fi set +x - DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} python3 -m sglang.launch_server \ + DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ --model-path ${MODEL_DIR}/${MODEL_NAME} \ --disaggregation-mode decode \ --disaggregation-ib-device ${IBDEVICES} \ @@ -809,6 +1042,7 @@ else DECODE_CMD="$DECODE_CMD --dist-init-addr ${DECODE_HEADNODE_URLS[$decode_idx]} --nnodes ${DECODE_NODES_PER_WORKER} --node-rank $rank" fi + dump_cmd "DECODE (rank ${NODE_RANK})" "$DECODE_CMD" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $DECODE_CMD" else diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh index 35eaf17dc0..5ee7078467 100644 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ b/benchmarks/multi_node/amd_utils/setup_deps.sh @@ -34,7 +34,6 @@ git_clone_retry() { return 1 } - # --------------------------------------------------------------------------- # 5. Container RDMA/net tools # - ibv_devinfo comes from ibverbs-utils @@ -80,86 +79,202 @@ install_amd_quark() { } # --------------------------------------------------------------------------- -# SGLang: Patch aiter gluon pa_mqa_logits — fix 2D → 3D instr_shape for -# Triton ≥ 3.5. -# -# Bug: _gluon_deepgemm_fp8_paged_mqa_logits (the non-preshuffle variant) -# hardcodes AMDMFMALayout(instr_shape=[16, 16]) which fails on Triton -# builds where AMDMFMALayout requires 3D (M, N, K) format. -# -# The two preshuffle variants already conditionally select 2D vs 3D via -# the module-level _Use_2d_instr_shape_mfma_layout flag, but the base -# variant was missed. This patch brings it in line. +# SGLang: prevent TP-rank collective desync deadlock in disaggregation prefill. # -# Affects: GLM-5 (NSA attention) and any future model that uses -# deepgemm_fp8_paged_mqa_logits with Preshuffle=False. +# resolve_waiting_queue_bootstrap() runs poll_and_all_reduce_attn_cp_tp_group() +# over `candidates`. The upstream candidate set (all non-aborted waiting reqs) +# can differ across TP ranks, so some ranks enter the all_reduce while others +# skip it -> hang. Narrow candidates to optimistic (pending_bootstrap) requests, +# which is consistent across ranks and is the only set finalize_bootstrap acts on. # --------------------------------------------------------------------------- -patch_gluon_pa_mqa_logits_instr_shape() { +patch_disagg_prefill_bootstrap_desync() { python3 -c ' import os, sys -target = "/sgl-workspace/aiter/aiter/ops/triton/gluon/pa_mqa_logits.py" +target = "/sgl-workspace/sglang/python/sglang/srt/disaggregation/prefill.py" if not os.path.isfile(target): - print("[SETUP] gluon pa_mqa_logits.py not found, skipping") + print("[SETUP] disaggregation/prefill.py not found, skipping") sys.exit(0) src = open(target).read() -if "[PATCHED] 3D instr_shape for base gluon variant" in src: - print("[SETUP] gluon pa_mqa_logits 3D instr_shape patch already applied") - sys.exit(0) +old = " candidates = [req for req in self.waiting_queue if not is_aborted(req)]" +new = ( + " candidates = [\n" + " req\n" + " for req in self.waiting_queue\n" + " if req.pending_bootstrap and not is_aborted(req)\n" + " ]" +) -# The buggy code: the base _gluon_deepgemm_fp8_paged_mqa_logits uses 2D -# instr_shape unconditionally. We replace it with a conditional that -# mirrors the preshuffle variants. -old = """\ - mfma_layout: gl.constexpr = gl.amd.AMDMFMALayout( - version=CDNA_VERSION, - instr_shape=[16, 16], - transposed=False, - warps_per_cta=[1, NumWarps], - ) - mfma_layout_a: gl.constexpr = gl.DotOperandLayout( - operand_index=0, parent=mfma_layout, k_width=16 - ) - mfma_layout_b: gl.constexpr = gl.DotOperandLayout( - operand_index=1, parent=mfma_layout, k_width=16 - )""" - -new = """\ - # [PATCHED] 3D instr_shape for base gluon variant - if _Use_2d_instr_shape_mfma_layout: - mfma_layout: gl.constexpr = gl.amd.AMDMFMALayout( - version=CDNA_VERSION, - instr_shape=[16, 16], - transposed=False, - warps_per_cta=[1, NumWarps], - ) - else: - mfma_layout: gl.constexpr = gl.amd.AMDMFMALayout( - version=CDNA_VERSION, - instr_shape=[16, 16, 32], - transposed=False, - warps_per_cta=[1, NumWarps], - ) - mfma_layout_a: gl.constexpr = gl.DotOperandLayout( - operand_index=0, parent=mfma_layout, k_width=16 - ) - mfma_layout_b: gl.constexpr = gl.DotOperandLayout( - operand_index=1, parent=mfma_layout, k_width=16 - )""" +if new in src: + print("[SETUP] prefill bootstrap-desync patch already applied") + sys.exit(0) if old not in src: - print("[SETUP] WARN: gluon pa_mqa_logits pattern not found — aiter version may have changed") + print("[SETUP] WARN: resolve_waiting_queue_bootstrap pattern not found — sglang version may have changed") sys.exit(0) -# Only replace the FIRST occurrence (the base variant, not preshuffle ones) -new_src = src.replace(old, new, 1) - -open(target, "w").write(new_src) -print("[SETUP] Patched: gluon pa_mqa_logits 3D instr_shape for base variant") +open(target, "w").write(src.replace(old, new)) +print("[SETUP] Patched: disaggregation/prefill.py resolve_waiting_queue_bootstrap candidates") ' - _SETUP_INSTALLED+=("gluon-instr-shape-fix") + _SETUP_INSTALLED+=("prefill-bootstrap-desync-fix") +} + +# --------------------------------------------------------------------------- +# SGLang: tp-group agreement for disagg decode queue (decode_tp_queue_agree). +# +# The metadata gate (poll_and_all_reduce) issues a tp-group collective whose +# shape/order == self.queue. Per-rank prealloc/enqueue timing can leave ranks +# with different queue membership/order, desyncing the collective -> decode +# hang (JID-17417 / JID-17445). This inserts _agree_and_order_queue() so every +# rank polls an identical, identically-ordered subset, and replaces the +# rank-divergent retracted-queue early return in process_decode_queue with a +# rank-symmetric MAX all_reduce. Mirrors patches/decode_tp_queue_agree.patch. +# --------------------------------------------------------------------------- +patch_decode_tp_queue_agree() { + python3 - <<'PYEOF' +import os, sys + +target = "/sgl-workspace/sglang/python/sglang/srt/disaggregation/decode.py" +if not os.path.isfile(target): + print("[SETUP] disaggregation/decode.py not found, skipping") + sys.exit(0) + +src = open(target).read() + +if "_agree_and_order_queue" in src: + print("[SETUP] decode tp-queue-agree patch already applied") + sys.exit(0) + +# --- Hunk 1+2: insert agreement method + gate pop_transferred ------------- +old1 = ( + " def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:\n" + " if not self.queue:\n" + " return []\n" +) +new1 = ''' def _agree_and_order_queue(self) -> Tuple[List["DecodeRequest"], List["DecodeRequest"]]: + """Split self.queue into (gated, deferred) using a tp-group agreement. + + The metadata gate (utils.poll_and_all_reduce) issues a tp-group all_reduce + whose tensor shape == len(self.queue) and whose per-index meaning is the + i-th queued request. That is only correct if every tp rank polls an + IDENTICAL queue (same requests, same order). Per-rank prealloc/enqueue + timing can leave queues with different membership or order, which desyncs + the collective -> deadlock (the JID-17417 decode hang). + + We all_gather the local request ids, keep only requests present on EVERY + rank, and order them deterministically (request ids are rank-invariant), so + the subsequent gate runs a matching collective on all ranks. Requests not + yet on every rank are returned as `deferred` and retried on a later call. + + NOTE: every rank that reaches pop_transferred must call this (it contains a + collective). That holds for the same reason the existing gate is safe: + pop_transferred is entered rank-symmetrically over self.gloo_group. + """ + tp_size = torch.distributed.get_world_size(self.gloo_group) + if tp_size <= 1: + return self.queue, [] + + local_rids = [dr.req.rid for dr in self.queue] + gathered: List[Optional[List[str]]] = [None] * tp_size + torch.distributed.all_gather_object( + gathered, local_rids, group=self.gloo_group + ) + + common = set(gathered[0] or []) + for rids in gathered[1:]: + common &= set(rids or []) + + if not common: + return [], list(self.queue) + + by_rid = {dr.req.rid: dr for dr in self.queue} + # sorted() yields an identical order on every rank (rids are rank-invariant). + gated = [by_rid[rid] for rid in sorted(common)] + deferred = [dr for dr in self.queue if dr.req.rid not in common] + return gated, deferred + + def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: + # Agree on a tp-rank-identical, identically-ordered subset BEFORE issuing any + # metadata-gate collective. Do NOT add a local `if not self.queue` early + # return here: an empty-queue rank must still join the agreement collective, + # otherwise it skips it and desyncs the tp group (root cause of the hang). + self.queue, _deferred = self._agree_and_order_queue() + if not self.queue: + # Empty agreement is rank-symmetric: all ranks skip the gate together. + self.queue = _deferred + return [] +''' + +# --- Hunk 3: re-attach deferred reqs after removal ------------------------ +old3 = ( + " entry for i, entry in enumerate(self.queue) if i not in indices_to_remove\n" + " ]\n" + "\n" + " return transferred_reqs\n" +) +new3 = ( + " entry for i, entry in enumerate(self.queue) if i not in indices_to_remove\n" + " ]\n" + " # Re-attach requests that were not yet present on every tp rank this\n" + " # iteration; they are gated again on a later call once all ranks have them.\n" + " if _deferred:\n" + " self.queue.extend(_deferred)\n" + "\n" + " return transferred_reqs\n" +) + +# --- Hunk 4: rank-symmetric retracted-queue gate -------------------------- +old4 = ( + " if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0:\n" + " # if there are still retracted requests, we do not allocate new requests\n" + " return\n" +) +new4 = ''' # PATCH(call-site tp-agreement): the retracted-queue early return below is + # rank-divergent — retracted_queue length is per-rank (per-rank KV-cache + # pressure). A divergent return skips the polling_count increment and the + # pop_transferred() collectives on some ranks, permanently desyncing the tp + # group: one rank ends up a full collective ahead, so pop_transferred's + # _agree_and_order_queue all_gather_object on the lagging ranks lines up + # against the metadata-gate all_reduce on the leading rank -> mismatched- + # collective deadlock (GPU 0%, detokenizer heartbeat freeze; JID-17445). + # process_decode_queue is called unconditionally every event-loop iteration + # on every rank, so an all_reduce here (before any divergent return) is + # rank-symmetric. Hold off new allocation iff ANY rank still has retracted + # reqs, so all ranks branch identically every iteration and pop_transferred + # is entered symmetrically. + _agree_gg = self.disagg_decode_transfer_queue.gloo_group + _local_retracted = ( + 1 if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0 else 0 + ) + if torch.distributed.get_world_size(_agree_gg) > 1: + _retracted_t = torch.tensor([_local_retracted], dtype=torch.int32) + torch.distributed.all_reduce( + _retracted_t, op=torch.distributed.ReduceOp.MAX, group=_agree_gg + ) + _any_retracted = bool(_retracted_t.item()) + else: + _any_retracted = bool(_local_retracted) + if _any_retracted: + # if any rank still has retracted requests, no rank allocates new ones + return +''' + +for label, old, new in ( + ("agreement-method+pop_transferred", old1, new1), + ("deferred-reattach", old3, new3), + ("retracted-gate", old4, new4), +): + if old not in src: + print(f"[SETUP] WARN: decode.py anchor for '{label}' not found — sglang version may have changed") + sys.exit(0) + src = src.replace(old, new, 1) + +open(target, "w").write(src) +print("[SETUP] Patched: disaggregation/decode.py tp-group decode queue agreement") +PYEOF + _SETUP_INSTALLED+=("decode-tp-queue-agree-fix") } # --------------------------------------------------------------------------- @@ -202,7 +317,9 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" else - patch_gluon_pa_mqa_logits_instr_shape + patch_disagg_prefill_bootstrap_desync + # patch_decode_tp_queue_agree + install_transformers_glm5 fi diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh new file mode 100644 index 0000000000..b3eb0b8fd7 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/trace_replay.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# Dual-Engine Disaggregated Benchmark Runner +# +# ENGINE=sglang (default): SGLang benchmark +# ENGINE=vllm: vLLM benchmark +# +# Produces JSON result files via benchmark_serving.py so that the CI pipeline +# can collect and process results. +# +# Usage: bash bench.sh \ +# \ +# + +ENGINE="${ENGINE:-sglang-disagg}" + +model_path=$1 +model_name=$2 +concurrency_list=${3:-"1"} +MODEL_PATH="${MODEL_PATH:-${model_path}/${model_name}}" +# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH +if [[ "$ENGINE" == "vllm-disagg" ]]; then + MODEL="${MODEL_NAME:-${MODEL_PATH}}" +else + MODEL="${MODEL_PATH}" +fi +log_path=${4:-/run_logs} + +# Split BENCH_MAX_CONCURRENCY (x-delimited, e.g. "8x16x32") into an array. +# Falls back to 1 if unset so the loop always runs at least once. +IFS='x' read -r -a chosen_concurrencies <<< "${concurrency_list}" + + +ROUTER_PORT="${ROUTER_PORT:-30000}" + +export TRANSFORMERS_VERBOSITY=error +export TOKENIZERS_PARALLELISM=false + +# echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" + +RESULT_DIR="${RESULT_DIR:-${log_path}/agentic}" +mkdir -p "$RESULT_DIR" + +source "$(dirname "$0")/../../benchmark_lib.sh" + +# clear_kv_caches — wipe all KV cache tiers on every backend worker before a +# concurrency point, so each conc is measured cold (no prefix reuse bleeding in +# from the previous conc). Mirrors mori-scheduler/scripts/benchmark/lib/ +# clear_caches.sh, but the worker base URLs are already resolved by +# server_sglang.sh (SERVER_FLUSH_URLS_CSV) so no SSH/IP lookup is needed. +# +# Tiers (SGLang server APIs), hit on EACH worker directly (the router does not +# fan /flush_cache out): +# L1 (GPU radix) + L2 (host hicache): POST /flush_cache — NO-OP while any +# request is in flight, so we drain-retry until "Cache flushed" or +# FLUSH_DRAIN_TIMEOUT (default 120s) elapses. +# L3 (umbp / mooncake store): POST /hicache/storage-backend/clear +# — HTTP != 200 when L3 is off, tolerated. +# Best-effort: logs WARN, never hard-fails the sweep. +clear_kv_caches() { + local drain_tmo="${FLUSH_DRAIN_TIMEOUT:-120}" + local urls_csv="${SERVER_FLUSH_URLS_CSV:-}" + if [[ -z "$urls_csv" ]]; then + echo "[clear_caches] WARN: SERVER_FLUSH_URLS_CSV unset; skipping cache flush" >&2 + return 0 + fi + local -a urls + IFS=',' read -r -a urls <<< "$urls_csv" + local url start ok resp code + for url in "${urls[@]}"; do + [[ -n "$url" ]] || continue + # L1 + L2: drain-retry until flushed (no-op while requests in flight). + start=$(date +%s); ok=0; resp="" + while :; do + resp=$(curl -sf -m 10 -X POST "${url}/flush_cache" 2>/dev/null || true) + echo "$resp" | grep -qi "Cache flushed" && { ok=1; break; } + (( $(date +%s) - start >= drain_tmo )) && break + sleep 3 + done + if [[ "$ok" == 1 ]]; then + echo "[clear_caches] ${url}: L1+L2 flushed" + else + echo "[clear_caches] WARN ${url}: L1+L2 flush NOT confirmed after ${drain_tmo}s (resp='${resp:0:80}')" >&2 + fi + # L3: storage-backend clear (umbp / mooncake). 200 when a backend is attached. + code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' -X POST "${url}/hicache/storage-backend/clear" 2>/dev/null || echo 000) + if [[ "$code" == 200 ]]; then + echo "[clear_caches] ${url}: L3 store cleared" + else + echo "[clear_caches] ${url}: L3 clear http=${code} (no storage backend / L3 off — ok)" + fi + done +} + +# REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" + +PORT="${ROUTER_PORT}" +MODEL="${MODEL:-${BENCH_MODEL}}" +DURATION="${DURATION:-1800}" +export MODEL DURATION MAX_MODEL_LEN +RESULT_DIR="${RESULT_DIR:-${profile_folder}}" +# Base name for the per-conc aggregate written by the existing +# utils.agentic.aggregation.process_agentic_result module. +# The workflow guard / upload steps expect a "${RESULT_FILENAME}_conc.json" +# file per concurrency, so each concurrency below is always suffixed with +# _conc (matching agentic_srt.sh on the gb200 path). +RESULT_FILENAME_BASE="${RESULT_FILENAME:-agentic_bench}" + +mkdir -p "$RESULT_DIR" + +resolve_trace_source +install_agentic_deps + +ANY_FAILED=0 +for max_concurrency in "${chosen_concurrencies[@]}"; do + + echo "==========================================" + echo "Agentic trace replay: conc=$max_concurrency" + echo "==========================================" + + # Clear all KV cache tiers on every backend before this conc point so it is + # measured cold (no prefix reuse from the previous conc). Default on; set + # CLEAR_CACHE_BETWEEN_CONC=0 to disable. Best-effort — never fails the run. + if [[ "${CLEAR_CACHE_BETWEEN_CONC:-1}" == "1" ]]; then + echo "conc=$max_concurrency: clearing L1/L2/L3 on all backends (no server restart)" + clear_kv_caches || echo "WARNING: cache clear had issues for conc=$max_concurrency" >&2 + fi + + # Mirror agentic_srt.sh (the srtctl/gb200 path): every concurrency writes + # its artifacts into a conc_/ subdir of RESULT_DIR. The CI matrix explodes + # agentic runs to one concurrency per job, but benchmark-multinode-tmpl.yml + # still expects the per-conc nesting (LOGS/agentic/conc_*/...) and the + # _conc result-file suffix, so we always nest to keep the layout identical + # across runners and avoid overwriting earlier runs in local multi-conc sweeps. + CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}" + mkdir -p "$CONC_RESULT_DIR" + + CONC="$max_concurrency" + USERS="$max_concurrency" + export CONC USERS + build_replay_cmd "$CONC_RESULT_DIR" + + # Per-conc result name consumed by write_agentic_result_json. Always suffix + # with _conc so the file matches + # the workflow guard's "${RESULT_FILENAME}_conc*.json" glob (and the agg / + # checkpoint upload steps) for both single-conc CI runs and multi-conc sweeps. + export RESULT_FILENAME="${RESULT_FILENAME_BASE}_conc${max_concurrency}" + if ! run_agentic_replay_and_write_outputs "$CONC_RESULT_DIR"; then + echo "WARNING: agentic trace replay for conc=$max_concurrency failed (replay or validation) after writing available results" >&2 + ANY_FAILED=1 + fi + + echo "-----------------------------------------" + +done + +export RESULT_FILENAME="$RESULT_FILENAME_BASE" + +if [ "$ANY_FAILED" -ne 0 ]; then + echo "WARNING: at least one conc had a non-zero exit; per-conc result files were still written when possible." >&2 +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4b1ea4aed3..11830c03d0 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3094,3 +3094,36 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } } + +dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260709 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang-disagg + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: "none" + conc-list: [ 16,32,48,64 ] + kv-offloading: dram + kv-offload-backend: hicache + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 57d510dd15..a7972e2475 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4750,3 +4750,10 @@ - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache + description: + - "Bump the SGLang ROCm image from 20260706 to 20260709 and retire the DeepSeek-V4 compress-state, DSA paged-MQA, AITER instruction-shape, and HiCache host-pool shims that are already included upstream." + - "Remove the SWA re-prefill and unified-KV HiCache overlays after their upstream PRs merged; retain only patch code without a merged upstream equivalent." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2127 diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 8cb92b7a16..1ff0dffd59 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -78,7 +78,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_mi355x_${FRAMEWORK}.sh" if [[ "$FRAMEWORK" == "sglang-disagg" ]] || [[ "$FRAMEWORK" == "vllm-disagg" ]] || [[ "$FRAMEWORK" == "atom-disagg" ]]; then - BENCHMARK_SUBDIR="multi_node" + # Agentic recipes live under multi_node/agentic/ and export the + # HiCache tunables (page-size, io-backend, ...); fixed-seq-len recipes + # live at the multi_node/ root. Honor SCENARIO_SUBDIR so agentic-coding + # configs pick the agentic recipe instead of the root one. + if [[ "${SCENARIO_SUBDIR}" == "agentic/" ]]; then + BENCHMARK_SUBDIR="multi_node/agentic" + else + BENCHMARK_SUBDIR="multi_node" + fi else BENCHMARK_SUBDIR="single_node/fixed_seq_len" fi @@ -126,7 +134,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # search for "FRAMEWORK_DIFF_IF_STATEMENT #3" for this if-statement # Find the latest log directory that contains the data - if [[ "${EVAL_ONLY:-false}" != "true" ]]; then + if [[ "${EVAL_ONLY:-false}" != "true" && "${IS_AGENTIC:-0}" != "1" ]]; then cat > collect_latest_results.py <<'PY' import os, sys job_dir, isl, osl, nexp, framework = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] @@ -181,6 +189,58 @@ PY fi fi + # Stage agentic raw artifacts + server logs for the CI upload steps. + # server_sglang.sh copies /run_logs/slurm_job- to + # $BENCHMARK_LOGS_DIR/logs/slurm_job- on shared storage, and + # trace_replay.sh writes each concurrency's aiperf artifacts under + # agentic/conc_/ (mirroring agentic_srt.sh). benchmark-multinode-tmpl.yml + # uploads them from $GITHUB_WORKSPACE/LOGS/agentic/conc_*/... plus a + # multinode_server_logs.tar.gz, so preserve the conc_/ nesting here + # before the logs dir is removed below. The agg result JSON is already + # written straight to the mounted workspace by the existing agentic + # aggregation module. + if [[ "${IS_AGENTIC:-0}" == "1" ]]; then + JOB_LOGS_DIR="$BENCHMARK_LOGS_DIR/logs/slurm_job-${JOB_ID}" + if [ -d "$JOB_LOGS_DIR" ]; then + # trace_replay.sh always nests artifacts under agentic/conc_/. + # Copy the whole agentic/ tree so the conc_/ subdirs are + # preserved for the LOGS/agentic/conc_*/... upload globs. + AGENTIC_SRC="$JOB_LOGS_DIR/agentic" + if [ -d "$AGENTIC_SRC" ] && find "$AGENTIC_SRC" -mindepth 1 -maxdepth 1 -type d -name 'conc_*' -print -quit 2>/dev/null | grep -q .; then + echo "Staging agentic raw artifacts from $AGENTIC_SRC" + mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" + cp -r "$AGENTIC_SRC"/. "$GITHUB_WORKSPACE/LOGS/agentic/" + # The source artifacts are created inside the container as root + # (--container-remap-root), so depending on how the runner + # invokes this script the copies can end up root-owned and/or + # read-only (aiperf/server_sglang make some dirs mode 0555). If + # the staged tree isn't owned+writable by the runner user, the + # next checkout's `git clean` fails with + # EACCES: permission denied, rmdir '.../LOGS/agentic'. + # chown to the invoking user (the same one that runs git clean) + # via sudo (already passwordless here for rm -rf). The follow-up + # chmod uses a+rwX (not just u+rwX): the *next* job against this + # same $GITHUB_WORKSPACE may be picked up by a different runner + # process running as a different OS user, which would otherwise + # fall outside the owner bits and still fail the same + # `git clean` with EACCES despite the chown above. + sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true + chmod -R a+rwX "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true + ls -laR "$GITHUB_WORKSPACE/LOGS/agentic" + else + echo "WARNING: no agentic conc_*/ artifacts found under $JOB_LOGS_DIR/agentic" + fi + # Server/router/prefill/decode logs for the multinode_server_logs_* artifact. + if tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$JOB_LOGS_DIR" . 2>/dev/null; then + echo "Created multinode_server_logs.tar.gz" + else + echo "WARNING: failed to create multinode_server_logs.tar.gz" + fi + else + echo "WARNING: agentic staging skipped; $JOB_LOGS_DIR not found" + fi + fi + echo "All result files processed" # Use sync scancel to ensure nfs file handle is released in time set +x diff --git a/utils/agentic/aggregation/backends/sglang.py b/utils/agentic/aggregation/backends/sglang.py index 2f560f3bf7..acbbc12b02 100644 --- a/utils/agentic/aggregation/backends/sglang.py +++ b/utils/agentic/aggregation/backends/sglang.py @@ -61,6 +61,8 @@ def populate( flat["server_gpu_cache_hit_rate"] = rate(device_hits, prompt_total) flat["server_cpu_cache_hit_rate"] = rate(host_hits, prompt_total) + # HiCache host hits are external to the GPU cache. + flat["server_external_cache_hit_rate"] = flat["server_cpu_cache_hit_rate"] flat["server_overall_cache_hit_rate"] = rate(total_cached, prompt_total) if flat["server_overall_cache_hit_rate"] is None: diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index 15a6a1482c..21a7307630 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -1119,6 +1119,7 @@ def test_processor_normalizes_sglang_server_metrics(tmp_path: Path): _assert_stable_server_metrics_schema(agg) assert agg["server_metrics"]["cache"]["gpu_cache_hit_rate"] == pytest.approx(0.4) assert agg["server_metrics"]["cache"]["cpu_cache_hit_rate"] == pytest.approx(0.1) + assert agg["server_metrics"]["cache"]["external_cache_hit_rate"] == pytest.approx(0.1) assert agg["server_metrics"]["cache"]["overall_cache_hit_rate"] == pytest.approx(0.5) assert agg["server_metrics"]["kv_cache"]["gpu_usage_pct"] == pytest.approx(0.75) assert agg["server_metrics"]["kv_cache"]["cpu_usage_pct"] == pytest.approx(0.3) From 5f2cc06ac93e28c935d6212a430224117bdc3350 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Thu, 9 Jul 2026 06:31:15 +0000 Subject: [PATCH 02/37] run-sweep: fix broken conc-list interpolation for multi-node agentic matrix.config.conc for multi-node agentic is already a JSON array (e.g. [16,32,64] from generate_sweep_configs.py). Wrapping it in a string literal as '[${{ matrix.config.conc }}]' can't correctly interpolate an array into a string, producing a malformed conc-list so fromJson(inputs.conc-list) in the reusable template ended up with an empty/wrong CONC_LIST. Use toJson(matrix.config.conc) instead, matching the pattern already used for the other two conc-list inputs in this file. Co-authored-by: Cursor --- .github/workflows/run-sweep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index b983982af8..4e415f9350 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -556,7 +556,7 @@ jobs: precision: ${{ matrix.config.precision }} router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} - conc-list: '[${{ matrix.config.conc }}]' + conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} prefill-hardware: ${{ matrix.config.prefill.hardware }} From 7bfdf387e4a87f6509552a227359258e10923552 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 03:02:32 +0000 Subject: [PATCH 03/37] bump image to rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0706 Signed-off-by: Theresa Shan --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 11830c03d0..db1687ca3f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3096,7 +3096,7 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260709 + image: rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0706 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds From 1a7b88cf913dcd6dfb097f6a240f57031c9a3524 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 03:14:15 +0000 Subject: [PATCH 04/37] run-sweep: fix sequence-typed conc input for multi-node agentic matrix.config.conc for multi-node agentic entries is a JSON array (chunked concurrencies per allocation), but sweep-multi-node-agentic passed it directly into benchmark-multinode-tmpl.yml's `conc` input, which is declared `type: string` ("First concurrency for agentic-coding scenarios; CONC_LIST carries the full batch"). GitHub Actions' reusable-workflow input validator rejects a sequence value for a string-typed input at evaluation time, so the whole job failed to even load: evaluate reusable workflow inputs: .github/workflows/run-sweep.yml (Line: 554, Col: 19): A sequence was not expected Since the job never materializes when this happens, sweep-multi-node- agentic silently disappeared from run summaries entirely instead of showing as failed. Slice to the first element (matching the intended "first concurrency" semantics and the same fix already applied elsewhere, e.g. PR #2122) to restore a scalar value. Co-authored-by: Cursor --- .github/workflows/run-sweep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 4e415f9350..fdb34e1e4d 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -577,7 +577,7 @@ jobs: decode-ep: ${{ matrix.config.decode.ep }} decode-dp-attn: ${{ matrix.config.decode.dp-attn }} decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - conc: ${{ matrix.config.conc }} + conc: ${{ matrix.config.conc[0] }} kv-offloading: ${{ matrix.config.kv-offloading }} kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} From 5fa76de0709cc5b5bfa7d59e33fc771e2a4ef6ba Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 03:36:55 +0000 Subject: [PATCH 05/37] agentic: run one task per concurrency for multi-node sweeps Multi-node agentic sweeps batched up to 4 concurrencies per SLURM allocation, running them sequentially against one shared server session. A slow/hung conc could block the rest of the batch from ever producing results, which is why run #6719 only reported c16 despite a 16/32/48/64 conc-list. Drop the batch size to 1 so each concurrency gets its own task/allocation, matching the granularity already used for single-node agentic sweeps. Cherry-picked from backup/agentx-v1.0-rebase-pre-upstream-rewrite-20260710 (082a59de0), adapted for the current test suite. Co-authored-by: Cursor --- utils/matrix_logic/generate_sweep_configs.py | 3 +- .../test_generate_sweep_configs.py | 120 +++--------------- 2 files changed, 21 insertions(+), 102 deletions(-) diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index babc328a81..d260549789 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -24,7 +24,8 @@ MIN_EVAL_CONC = 16 # Bound how many multinode agentic conc points share one server allocation. -MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 4 +# 1 = one task/SLURM allocation per concurrency (matches single-node agentic). +MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 1 BYTES_PER_MIB = 1024 * 1024 BYTES_PER_GB = 1_000_000_000 # 3 TB decimal DRAM cap, expressed in MiB, before utilization scaling. diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 7f2583f57e..b77a1cdc48 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -68,7 +68,6 @@ def sample_multinode_config(): "runner": "gb200", "multinode": True, "disagg": True, - "kv-p2p-transfer": "nixl", "scenarios": { "fixed-seq-len": [ @@ -133,39 +132,6 @@ def sample_runner_config(): } -@pytest.fixture -def sample_single_node_agentic_config(): - """Single-node agentic config with explicit and default spec decoding.""" - return { - "kimik2.6-fp4-b300-trt-agentic": { - "image": "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc5", - "model": "moonshotai/Kimi-K2.5", - "model-prefix": "kimik2.6", - "precision": "fp4", - "framework": "trt", - "runner": "cluster:b300-nv", - "multinode": False, - "scenarios": { - "agentic-coding": [{ - "search-space": [ - { - "tp": 8, - "spec-decoding": "mtp", - "kv-offloading": "none", - "conc-list": [16], - }, - { - "tp": 8, - "kv-offloading": "none", - "conc-list": [32], - }, - ], - }], - }, - }, - } - - @pytest.fixture def full_sweep_args_single_node(): """Args for full-sweep single-node command.""" @@ -722,22 +688,6 @@ def test_matrix_entry_structure(self, sample_single_node_config, sample_runner_c for row in explicit_result } == {(2, 2, 2)} - def test_agentic_spec_decoding_is_propagated( - self, - sample_single_node_agentic_config, - sample_runner_config, - full_sweep_args_single_node, - ): - result = generate_full_sweep( - full_sweep_args_single_node, - sample_single_node_agentic_config, - sample_runner_config, - ) - - assert [entry["spec-decoding"] for entry in result] == ["mtp", "none"] - assert result[0]["exp-name"].endswith("_kvnone_spec-mtp") - assert result[1]["exp-name"].endswith("_kvnone") - def test_filter_by_model_prefix(self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node): """Filter by model prefix should work.""" full_sweep_args_single_node.model_prefix = ["dsr1"] @@ -1963,29 +1913,6 @@ def test_single_node_parallelism_fields_are_generated( for row in explicit_result ] == [(2, 2, 2)] - def test_single_node_agentic_spec_decoding_is_propagated( - self, - sample_single_node_agentic_config, - sample_runner_config, - ): - args = argparse.Namespace( - config_keys=["kimik2.6-fp4-b300-trt-agentic"], - seq_lens=None, - conc=None, - scenario_type=["agentic-coding"], - runner_node_filter=None, - ) - - result = generate_test_config_sweep( - args, - sample_single_node_agentic_config, - sample_runner_config, - ) - - assert [entry["spec-decoding"] for entry in result] == ["mtp", "none"] - assert result[0]["exp-name"].endswith("_kvnone_spec-mtp") - assert result[1]["exp-name"].endswith("_kvnone") - def test_multinode_parallelism_fields_are_generated( self, sample_multinode_config, @@ -2019,9 +1946,6 @@ def test_multinode_parallelism_fields_are_generated( def test_runner_node_filter_expands_config_runner(self, sample_multinode_config, sample_runner_config): """test-config should allow targeting one concrete runner node.""" - master_entry = sample_multinode_config["dsr1-fp4-gb200-dynamo-trt"] - master_entry["router"] = {"name": "trt-router", "version": "0.20.0"} - master_entry["kv-p2p-transfer"] = "nixl" args = argparse.Namespace( config_keys=["dsr1-fp4-gb200-dynamo-trt"], seq_lens=None, @@ -2037,8 +1961,6 @@ def test_runner_node_filter_expands_config_runner(self, sample_multinode_config, assert len(result) == 1 assert result[0]["runner"] == "gb200-nv_0" - assert result[0]["router"] == {"name": "trt-router", "version": "0.20.0"} - assert result[0]["kv-p2p-transfer"] == "nixl" def test_runner_node_filter_no_match_skips_config(self, sample_multinode_config, sample_runner_config): """Unmatched node filters should produce no entries.""" @@ -2068,7 +1990,6 @@ def test_runner_node_filter_expands_agentic_config_runner(self, sample_runner_co "framework": "sglang", "runner": "cluster:b300-nv", "multinode": False, - "router": {"name": "default-router", "version": "1.0.0"}, "scenarios": { "agentic-coding": [ { @@ -2078,7 +1999,7 @@ def test_runner_node_filter_expands_agentic_config_runner(self, sample_runner_co "tp": 8, "ep": 1, "kv-offloading": "dram", - "kv-offload-backend": {"name": "hicache"}, + "kv-offload-backend": "hicache", "conc-list": [64], } ], @@ -2102,7 +2023,6 @@ def test_runner_node_filter_expands_agentic_config_runner(self, sample_runner_co assert result[0]["scenario-type"] == "agentic-coding" assert result[0]["total-cpu-dram-gb"] == 2399 assert result[0]["duration"] == 3600 - assert result[0]["router"] == {"name": "default-router", "version": "1.0.0"} def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config): config = { @@ -2121,7 +2041,7 @@ def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config): { "tp": 4, "kv-offloading": "dram", - "kv-offload-backend": {"name": "native"}, + "kv-offload-backend": "native", "conc-list": [32], }, { @@ -2129,7 +2049,7 @@ def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config): "dcp-size": 2, "pcp-size": 1, "kv-offloading": "dram", - "kv-offload-backend": {"name": "native"}, + "kv-offload-backend": "native", "conc-list": [32], }, { @@ -2137,14 +2057,14 @@ def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config): "dcp-size": 1, "pcp-size": 2, "kv-offloading": "dram", - "kv-offload-backend": {"name": "native"}, + "kv-offload-backend": "native", "conc-list": [32], }, { "tp": 4, "pp": 2, "kv-offloading": "dram", - "kv-offload-backend": {"name": "native"}, + "kv-offload-backend": "native", "conc-list": [32], }, ], @@ -2191,7 +2111,7 @@ def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_conf { "tp": 4, "kv-offloading": "dram", - "kv-offload-backend": {"name": "native"}, + "kv-offload-backend": "native", "conc-list": [32], }, ], @@ -2213,7 +2133,7 @@ def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_conf generate_test_config_sweep(args, config, runner_config) def test_multinode_agentic_groups_concurrencies_per_search_entry(self): - """One server allocation should run the selected concurrency batch.""" + """One server allocation should run exactly one concurrency (one task per conc).""" config = { "dsv4-agentic-2p1d": { "image": "vllm/vllm-openai:v0.23.0", @@ -2230,8 +2150,6 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): "search-space": [ { "conc-list": [16, 32, 64, 128, 256], - "router": {"name": "dynamo-router", "version": "1.3.0"}, - "kv-p2p-transfer": "nixl", "prefill": {"hardware": "gb200", "num-worker": 2, "tp": 4, "pp": 2, "dcp-size": 2, "pcp-size": 2, "ep": 4, "dp-attn": False}, "decode": {"hardware": "h100", "num-worker": 1, "tp": 4, "pp": 2, "dcp-size": 2, "pcp-size": 1, "ep": 1, "dp-attn": False}, } @@ -2251,19 +2169,21 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): result = generate_test_config_sweep(args, config) - assert len(result) == 2 - assert result[0]["conc"] == [16, 32, 64, 128] - assert result[0]["exp-name"] == "dsv4_p2x4_d1x4_conc16x32x64x128" + assert len(result) == 5 + assert [entry["conc"] for entry in result] == [[16], [32], [64], [128], [256]] + assert [entry["exp-name"] for entry in result] == [ + "dsv4_p2x4_d1x4_conc16", + "dsv4_p2x4_d1x4_conc32", + "dsv4_p2x4_d1x4_conc64", + "dsv4_p2x4_d1x4_conc128", + "dsv4_p2x4_d1x4_conc256", + ] assert result[0]["prefill"]["pp"] == 2 assert result[0]["prefill"]["dcp-size"] == 2 assert result[0]["prefill"]["pcp-size"] == 2 assert result[0]["decode"]["pp"] == 2 assert result[0]["decode"]["dcp-size"] == 2 assert result[0]["decode"]["pcp-size"] == 1 - assert result[1]["conc"] == [256] - assert result[1]["exp-name"] == "dsv4_p2x4_d1x4_conc256" - assert all(entry["router"] == {"name": "dynamo-router", "version": "1.3.0"} for entry in result) - assert all(entry["kv-p2p-transfer"] == "nixl" for entry in result) def test_multinode_agentic_preserves_kv_offload_fields(self): config = { @@ -2276,13 +2196,12 @@ def test_multinode_agentic_preserves_kv_offload_fields(self): "runner": "cluster:mi355x-amds", "multinode": True, "disagg": True, - "kv-p2p-transfer": "mori", "scenarios": { "agentic-coding": [{ "search-space": [{ "conc-list": [16], "kv-offloading": "dram", - "kv-offload-backend": {"name": "hicache"}, + "kv-offload-backend": "hicache", "prefill": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, }], @@ -2302,7 +2221,7 @@ def test_multinode_agentic_preserves_kv_offload_fields(self): assert len(result) == 1 assert result[0]["kv-offloading"] == "dram" - assert result[0]["kv-offload-backend"] == {"name": "hicache"} + assert result[0]["kv-offload-backend"] == "hicache" assert result[0]["exp-name"] == "dsv4_p1x8_d1x8_conc16_kvdram-hicache" @@ -2422,12 +2341,11 @@ def test_node_type_filters_apply_to_agentic_configs( "runner": "cluster:gb200-nv", "multinode": True, "disagg": True, - "kv-p2p-transfer": "nixl", "scenarios": { "agentic-coding": [{ "search-space": [ { - "conc-list": [16, 32], + "conc-list": [16], "prefill": {"hardware": "gb200", "num-worker": 2, "tp": 4, "pp": 2, "dcp-size": 2, "pcp-size": 2, "ep": 4, "dp-attn": False}, "decode": {"hardware": "h100", "num-worker": 1, "tp": 4, "pp": 2, "dcp-size": 2, "pcp-size": 1, "ep": 1, "dp-attn": False}, }, From a5763f704bc03952a21d6f4e99cdf8003cd26dc5 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 15:55:51 +0800 Subject: [PATCH 06/37] Bump image to latest upstream image with 4 PR fixes included. --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index db1687ca3f..c0331ff2ba 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3096,7 +3096,7 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - image: rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0706 + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds From ea548386bb3f7f574395dc65b434a163f24759dd Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 11:11:33 +0000 Subject: [PATCH 07/37] update con-list Signed-off-by: Theresa Shan --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c0331ff2ba..37d46638da 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3109,7 +3109,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - dram-utilization: 0.80 search-space: - spec-decoding: "none" - conc-list: [ 16,32,48,64 ] + conc-list: [ 2,4,8,16,32 ] kv-offloading: dram kv-offload-backend: hicache prefill: From 183a987ad2b5c3a0e5aee4e37f3b5b3cd2cf54de Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 11:18:26 +0000 Subject: [PATCH 08/37] setup_deps: remove disabled decode_tp_queue_agree patch patch_decode_tp_queue_agree()'s invocation was already commented out (dead code with no runtime effect), and the reference-only patches/decode_tp_queue_agree.patch it mirrored had no other callers. Drop both plus the README bullet pointing at it. Co-authored-by: Cursor --- .../multi_node/amd_utils/patches/README.md | 94 +++++++++++ .../patches/decode_tp_queue_agree.patch | 107 ------------ benchmarks/multi_node/amd_utils/setup_deps.sh | 157 ------------------ 3 files changed, 94 insertions(+), 264 deletions(-) create mode 100644 benchmarks/multi_node/amd_utils/patches/README.md delete mode 100644 benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch diff --git a/benchmarks/multi_node/amd_utils/patches/README.md b/benchmarks/multi_node/amd_utils/patches/README.md new file mode 100644 index 0000000000..765d571b27 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/patches/README.md @@ -0,0 +1,94 @@ +# In-tree patches for the MoRI / MoRIIO PD-disagg path + +This directory carries small overlays that fix up the engine source inside +the docker container at runtime. They are needed because some published +images ship known bugs in the (MoRI / MoRIIO) disaggregation backend that +block our benchmark + accuracy configs — so we can keep reusing the +**stock image** instead of rebuilding a patched one. + +- `mori_conn.py` — single-file overlay (bind-mounted) for the **sglang** + MoRI backend. + +> Note: the vLLM MoRIIO `minimax-m3` overlay (`moriio/`) was retired once the +> upstream fixes (vLLM #46039 / #46290 / #46332) shipped in the ROCm nightly +> image; `minimaxm3-fp8-mi355x-vllm-disagg` now runs the stock nightly directly. + +The `mori_conn.py` overlay is wired through the `EXTRA_DOCKER_MOUNTS` env +var that `job.slurm` consumes (an opt-in `${EXTRA_DOCKER_MOUNTS:-}` after +the existing `-v` block). The local-test driver scripts under +`scripts/sglang_disagg/` pre-set this env var to the path of the relevant +overlay; CI runners that need the patch can do the same. + +## `mori_conn.py` + +Overlays +`/sgl-workspace/sglang/python/sglang/srt/disaggregation/mori/conn.py`. + +Source: forked from the file shipped in +`lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523` +(sglang [v0.5.12.post1](https://github.com/sgl-project/sglang/tree/v0.5.12.post1)). +Four logical edits, all confined to `MoriKVReceiver.send_state`, +`MoriKVReceiver._register_kv_args`, and +`MoriKVReceiver._send_swa_dsa_state`: + +1. **Sender flatten** — handle the framework's nested + `state_item_lens: List[List[int]]` instead of crashing in the + naked `struct.pack("I", item_len)` (the legacy `List[int]` + assumption). Idempotent for legacy flat callers. +2. **`state_type` legacy fallback** — when the legacy singular + `kv_args.state_type` is `'none'` but `state_mem_descs` is non-empty, + read `kv_args.state_types[0]` (the modern plural API that Mooncake + and NIXL already use). Routes `MAMBA → _send_mamba_state` and + `DSA/SWA → _send_swa_dsa_state` correctly. +3. **Consumer normalization** — flatten `state_item_lens` and + `state_dim_per_tensor` to flat `List[int]` once at the entry of + `send_state`, so the existing per-tensor index arithmetic + (`state_item_lens[i]`) and length checks + (`len(state_item_lens) == len(state_mem_descs)`) keep working. +4. **DSA index rank+length normalization** — inside + `_send_swa_dsa_state`, before the `group_concurrent_contiguous` + call, ravel both `src_state_indices` and `dst_state_indices` to 1-D + and re-truncate to common length. Upstream's existing truncation + only slices the outer axis, leaving 2-D `(1, N)` arrays unchanged + and triggering an `np.diff` broadcasting error + (`shapes (1,12) (0,)`) for GLM-5 (single-DSA-component) prefill + traffic. See + `scripts/sglang_disagg/docs_glm5/01-bug-analysis.md` for the full + write-up. + +Verified passing GSM8K = 0.978 ± 0.004 on Qwen3.5-397B-A17B-FP8 1P+1D +TP=8 dp-attn=false (matches and slightly exceeds upstream +[PR #22665](https://github.com/sgl-project/sglang/pull/22665)'s +reported 0.970 GSM8K on the bf16 baseline). GLM-5 (DSA) verification +in progress under +`scripts/sglang_disagg/docs_glm5/02-fix-and-verification.md`. + +This is a stop-gap. The proper upstream fix is to migrate MoRI to the +plural `state_types: List[StateType]` API (full design + diff in +`scripts/sglang_disagg/docs/03-upstream-pr-proposal.md`). + +## How to enable + +```bash +export EXTRA_DOCKER_MOUNTS="-v $DI_REPO_DIR/benchmarks/multi_node/amd_utils/patches/mori_conn.py:/sgl-workspace/sglang/python/sglang/srt/disaggregation/mori/conn.py:ro" +``` + +`$DI_REPO_DIR` is the InferenceX checkout root that `job.slurm` +already mounts into the container at `/workspace`. + +When this env var is unset (CI default for runs that don't need the +patch), `${EXTRA_DOCKER_MOUNTS:-}` expands to the empty string and +container behavior is byte-identical to the unpatched path. + +## When to use which patch + +| Image / version | Need `mori_conn.py` overlay? | +|---|---| +| `lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523` | yes (Qwen3.5-MoE-FP8, GLM-5, any hybrid model on this image) | +| `lmsysorg/sglang-rocm:v0.5.10.post1-rocm720-mi35x-*` (used by `dsr1-fp4-*-disagg`) | not validated; same code path likely affected — try with the overlay if you hit the same `struct.error` | +| `rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-*` (used by `dsr1-fp8-*-disagg`, `glm5-*-disagg`) | predates [PR #22665](https://github.com/sgl-project/sglang/pull/22665); different code paths; **do not** apply this overlay | + +When upstream merges the proper fix (see +`scripts/sglang_disagg/docs/03-upstream-pr-proposal.md`) and that +fix lands in a published image, retire this overlay and the +`EXTRA_DOCKER_MOUNTS` knob can stay (still useful for future patches). diff --git a/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch b/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch deleted file mode 100644 index e09c02112f..0000000000 --- a/benchmarks/multi_node/amd_utils/patches/decode_tp_queue_agree.patch +++ /dev/null @@ -1,107 +0,0 @@ ---- a/python/sglang/srt/disaggregation/decode.py 2026-06-25 02:44:11.274035348 +0000 -+++ b/python/sglang/srt/disaggregation/decode.py 2026-06-25 02:44:11.276035310 +0000 -@@ -1616,8 +1616,57 @@ - ) - kv_manager._staging_handler = self.staging_handler - -+ def _agree_and_order_queue(self) -> Tuple[List["DecodeRequest"], List["DecodeRequest"]]: -+ """Split self.queue into (gated, deferred) using a tp-group agreement. -+ -+ The metadata gate (utils.poll_and_all_reduce) issues a tp-group all_reduce -+ whose tensor shape == len(self.queue) and whose per-index meaning is the -+ i-th queued request. That is only correct if every tp rank polls an -+ IDENTICAL queue (same requests, same order). Per-rank prealloc/enqueue -+ timing can leave queues with different membership or order, which desyncs -+ the collective -> deadlock (the JID-17417 decode hang). -+ -+ We all_gather the local request ids, keep only requests present on EVERY -+ rank, and order them deterministically (request ids are rank-invariant), so -+ the subsequent gate runs a matching collective on all ranks. Requests not -+ yet on every rank are returned as `deferred` and retried on a later call. -+ -+ NOTE: every rank that reaches pop_transferred must call this (it contains a -+ collective). That holds for the same reason the existing gate is safe: -+ pop_transferred is entered rank-symmetrically over self.gloo_group. -+ """ -+ tp_size = torch.distributed.get_world_size(self.gloo_group) -+ if tp_size <= 1: -+ return self.queue, [] -+ -+ local_rids = [dr.req.rid for dr in self.queue] -+ gathered: List[Optional[List[str]]] = [None] * tp_size -+ torch.distributed.all_gather_object( -+ gathered, local_rids, group=self.gloo_group -+ ) -+ -+ common = set(gathered[0] or []) -+ for rids in gathered[1:]: -+ common &= set(rids or []) -+ -+ if not common: -+ return [], list(self.queue) -+ -+ by_rid = {dr.req.rid: dr for dr in self.queue} -+ # sorted() yields an identical order on every rank (rids are rank-invariant). -+ gated = [by_rid[rid] for rid in sorted(common)] -+ deferred = [dr for dr in self.queue if dr.req.rid not in common] -+ return gated, deferred -+ - def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: -+ # Agree on a tp-rank-identical, identically-ordered subset BEFORE issuing any -+ # metadata-gate collective. Do NOT add a local `if not self.queue` early -+ # return here: an empty-queue rank must still join the agreement collective, -+ # otherwise it skips it and desyncs the tp group (root cause of the hang). -+ self.queue, _deferred = self._agree_and_order_queue() - if not self.queue: -+ # Empty agreement is rank-symmetric: all ranks skip the gate together. -+ self.queue = _deferred - return [] - - if self.scheduler.enable_decode_hicache: -@@ -1731,6 +1780,10 @@ - self.queue = [ - entry for i, entry in enumerate(self.queue) if i not in indices_to_remove - ] -+ # Re-attach requests that were not yet present on every tp rank this -+ # iteration; they are gated again on a later call once all ranks have them. -+ if _deferred: -+ self.queue.extend(_deferred) - - return transferred_reqs - -@@ -1946,8 +1999,33 @@ - # try to resume retracted requests if there are enough space for another `num_reserved_decode_tokens` decode steps - resumed_reqs = self.disagg_decode_prealloc_queue.resume_retracted_reqs() - self.waiting_queue.extend(resumed_reqs) -- if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0: -- # if there are still retracted requests, we do not allocate new requests -+ # PATCH(call-site tp-agreement): the retracted-queue early return below is -+ # rank-divergent — retracted_queue length is per-rank (per-rank KV-cache -+ # pressure). A divergent return skips the polling_count increment and the -+ # pop_transferred() collectives on some ranks, permanently desyncing the tp -+ # group: one rank ends up a full collective ahead, so pop_transferred's -+ # _agree_and_order_queue all_gather_object (decode.py:1644) on the lagging -+ # ranks lines up against the metadata-gate all_reduce (decode.py:1591) on -+ # the leading rank -> mismatched-collective deadlock (GPU 0%, detokenizer -+ # heartbeat freeze; JID-17445). process_decode_queue is called -+ # unconditionally every event-loop iteration on every rank, so an all_reduce -+ # here (before any divergent return) is rank-symmetric. Hold off new -+ # allocation iff ANY rank still has retracted reqs, so all ranks branch -+ # identically every iteration and pop_transferred is entered symmetrically. -+ _agree_gg = self.disagg_decode_transfer_queue.gloo_group -+ _local_retracted = ( -+ 1 if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0 else 0 -+ ) -+ if torch.distributed.get_world_size(_agree_gg) > 1: -+ _retracted_t = torch.tensor([_local_retracted], dtype=torch.int32) -+ torch.distributed.all_reduce( -+ _retracted_t, op=torch.distributed.ReduceOp.MAX, group=_agree_gg -+ ) -+ _any_retracted = bool(_retracted_t.item()) -+ else: -+ _any_retracted = bool(_local_retracted) -+ if _any_retracted: -+ # if any rank still has retracted requests, no rank allocates new ones - return - - if not hasattr(self, "polling_count"): diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh index 5ee7078467..9f10958de1 100644 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ b/benchmarks/multi_node/amd_utils/setup_deps.sh @@ -121,162 +121,6 @@ print("[SETUP] Patched: disaggregation/prefill.py resolve_waiting_queue_bootstra _SETUP_INSTALLED+=("prefill-bootstrap-desync-fix") } -# --------------------------------------------------------------------------- -# SGLang: tp-group agreement for disagg decode queue (decode_tp_queue_agree). -# -# The metadata gate (poll_and_all_reduce) issues a tp-group collective whose -# shape/order == self.queue. Per-rank prealloc/enqueue timing can leave ranks -# with different queue membership/order, desyncing the collective -> decode -# hang (JID-17417 / JID-17445). This inserts _agree_and_order_queue() so every -# rank polls an identical, identically-ordered subset, and replaces the -# rank-divergent retracted-queue early return in process_decode_queue with a -# rank-symmetric MAX all_reduce. Mirrors patches/decode_tp_queue_agree.patch. -# --------------------------------------------------------------------------- -patch_decode_tp_queue_agree() { - python3 - <<'PYEOF' -import os, sys - -target = "/sgl-workspace/sglang/python/sglang/srt/disaggregation/decode.py" -if not os.path.isfile(target): - print("[SETUP] disaggregation/decode.py not found, skipping") - sys.exit(0) - -src = open(target).read() - -if "_agree_and_order_queue" in src: - print("[SETUP] decode tp-queue-agree patch already applied") - sys.exit(0) - -# --- Hunk 1+2: insert agreement method + gate pop_transferred ------------- -old1 = ( - " def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:\n" - " if not self.queue:\n" - " return []\n" -) -new1 = ''' def _agree_and_order_queue(self) -> Tuple[List["DecodeRequest"], List["DecodeRequest"]]: - """Split self.queue into (gated, deferred) using a tp-group agreement. - - The metadata gate (utils.poll_and_all_reduce) issues a tp-group all_reduce - whose tensor shape == len(self.queue) and whose per-index meaning is the - i-th queued request. That is only correct if every tp rank polls an - IDENTICAL queue (same requests, same order). Per-rank prealloc/enqueue - timing can leave queues with different membership or order, which desyncs - the collective -> deadlock (the JID-17417 decode hang). - - We all_gather the local request ids, keep only requests present on EVERY - rank, and order them deterministically (request ids are rank-invariant), so - the subsequent gate runs a matching collective on all ranks. Requests not - yet on every rank are returned as `deferred` and retried on a later call. - - NOTE: every rank that reaches pop_transferred must call this (it contains a - collective). That holds for the same reason the existing gate is safe: - pop_transferred is entered rank-symmetrically over self.gloo_group. - """ - tp_size = torch.distributed.get_world_size(self.gloo_group) - if tp_size <= 1: - return self.queue, [] - - local_rids = [dr.req.rid for dr in self.queue] - gathered: List[Optional[List[str]]] = [None] * tp_size - torch.distributed.all_gather_object( - gathered, local_rids, group=self.gloo_group - ) - - common = set(gathered[0] or []) - for rids in gathered[1:]: - common &= set(rids or []) - - if not common: - return [], list(self.queue) - - by_rid = {dr.req.rid: dr for dr in self.queue} - # sorted() yields an identical order on every rank (rids are rank-invariant). - gated = [by_rid[rid] for rid in sorted(common)] - deferred = [dr for dr in self.queue if dr.req.rid not in common] - return gated, deferred - - def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: - # Agree on a tp-rank-identical, identically-ordered subset BEFORE issuing any - # metadata-gate collective. Do NOT add a local `if not self.queue` early - # return here: an empty-queue rank must still join the agreement collective, - # otherwise it skips it and desyncs the tp group (root cause of the hang). - self.queue, _deferred = self._agree_and_order_queue() - if not self.queue: - # Empty agreement is rank-symmetric: all ranks skip the gate together. - self.queue = _deferred - return [] -''' - -# --- Hunk 3: re-attach deferred reqs after removal ------------------------ -old3 = ( - " entry for i, entry in enumerate(self.queue) if i not in indices_to_remove\n" - " ]\n" - "\n" - " return transferred_reqs\n" -) -new3 = ( - " entry for i, entry in enumerate(self.queue) if i not in indices_to_remove\n" - " ]\n" - " # Re-attach requests that were not yet present on every tp rank this\n" - " # iteration; they are gated again on a later call once all ranks have them.\n" - " if _deferred:\n" - " self.queue.extend(_deferred)\n" - "\n" - " return transferred_reqs\n" -) - -# --- Hunk 4: rank-symmetric retracted-queue gate -------------------------- -old4 = ( - " if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0:\n" - " # if there are still retracted requests, we do not allocate new requests\n" - " return\n" -) -new4 = ''' # PATCH(call-site tp-agreement): the retracted-queue early return below is - # rank-divergent — retracted_queue length is per-rank (per-rank KV-cache - # pressure). A divergent return skips the polling_count increment and the - # pop_transferred() collectives on some ranks, permanently desyncing the tp - # group: one rank ends up a full collective ahead, so pop_transferred's - # _agree_and_order_queue all_gather_object on the lagging ranks lines up - # against the metadata-gate all_reduce on the leading rank -> mismatched- - # collective deadlock (GPU 0%, detokenizer heartbeat freeze; JID-17445). - # process_decode_queue is called unconditionally every event-loop iteration - # on every rank, so an all_reduce here (before any divergent return) is - # rank-symmetric. Hold off new allocation iff ANY rank still has retracted - # reqs, so all ranks branch identically every iteration and pop_transferred - # is entered symmetrically. - _agree_gg = self.disagg_decode_transfer_queue.gloo_group - _local_retracted = ( - 1 if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0 else 0 - ) - if torch.distributed.get_world_size(_agree_gg) > 1: - _retracted_t = torch.tensor([_local_retracted], dtype=torch.int32) - torch.distributed.all_reduce( - _retracted_t, op=torch.distributed.ReduceOp.MAX, group=_agree_gg - ) - _any_retracted = bool(_retracted_t.item()) - else: - _any_retracted = bool(_local_retracted) - if _any_retracted: - # if any rank still has retracted requests, no rank allocates new ones - return -''' - -for label, old, new in ( - ("agreement-method+pop_transferred", old1, new1), - ("deferred-reattach", old3, new3), - ("retracted-gate", old4, new4), -): - if old not in src: - print(f"[SETUP] WARN: decode.py anchor for '{label}' not found — sglang version may have changed") - sys.exit(0) - src = src.replace(old, new, 1) - -open(target, "w").write(src) -print("[SETUP] Patched: disaggregation/decode.py tp-group decode queue agreement") -PYEOF - _SETUP_INSTALLED+=("decode-tp-queue-agree-fix") -} - # --------------------------------------------------------------------------- # SGLang: Install latest transformers for GLM-5 model type support. # @@ -318,7 +162,6 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" else patch_disagg_prefill_bootstrap_desync - # patch_decode_tp_queue_agree install_transformers_glm5 fi From 711baf8c2f87ab8ff3b8246015067f13073c6012 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 12:54:34 +0000 Subject: [PATCH 09/37] setup_deps: broaden GLM transformers gate; disable prefill bootstrap-desync patch install_transformers_glm5() was gated on an exact MODEL_NAME == "GLM-5-FP8" match; broaden to any model name containing "GLM" so other GLM variants pick up the same glm_moe_dsa transformers fix. Also disable patch_disagg_prefill_bootstrap_desync's invocation (commented out, matching the already-disabled decode_tp_queue_agree pattern removed earlier). Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/setup_deps.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh index 9f10958de1..f1ae30b925 100644 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ b/benchmarks/multi_node/amd_utils/setup_deps.sh @@ -122,14 +122,15 @@ print("[SETUP] Patched: disaggregation/prefill.py resolve_waiting_queue_bootstra } # --------------------------------------------------------------------------- -# SGLang: Install latest transformers for GLM-5 model type support. +# SGLang: Install latest transformers for GLM model type support. # # GLM-5 (zai-org/GLM-5-FP8) requires a transformers build that includes -# the glm_moe_dsa model type. The mori images do not ship it. -# Only install if GLM-5 is the active model (avoid overhead otherwise). +# the glm_moe_dsa model type. The mori images do not ship it. Gated on any +# GLM model name (not just GLM-5-FP8) so other GLM variants pick up the same +# fix; only installs when a GLM model is active (avoid overhead otherwise). # --------------------------------------------------------------------------- install_transformers_glm5() { - if [[ "$MODEL_NAME" != "GLM-5-FP8" ]]; then + if [[ "$MODEL_NAME" != *GLM* ]]; then return 0 fi @@ -161,7 +162,7 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" else - patch_disagg_prefill_bootstrap_desync + # patch_disagg_prefill_bootstrap_desync install_transformers_glm5 fi From 5d671a3045616e78f1b8e6c06d4cb01bdd8e6614 Mon Sep 17 00:00:00 2001 From: AMD-yanfeiwang Date: Fri, 10 Jul 2026 21:06:08 +0800 Subject: [PATCH 10/37] agentic: node-0 sibling benchmark-client container for DSv4 sweeps (#2147) * agentic: add node-0 sibling benchmark-client container for DSv4 sweeps Port the "same-node sibling container" client mode from ROCm/InferenceY: when CLIENT_IMAGE is set (and no CLIENT_NODES), node 0 launches the aiperf trace replay in its own pre-baked container via the host docker socket, instead of rebuilding the aiperf venv inside the server container and running it co-located. This keeps the client's CPU-heavy tokenize/aggregate work off the sglang scheduler + router, which inflates TTFT/E2E and lowers throughput under agentic concurrency. - server_sglang.sh: add IS_AGENTIC_RUN and a CLIENT_IMAGE sibling-container branch that writes client.env and docker-runs the client against the local router (--network host). - job.slurm: define CLIENT_CONT_NAME; when CLIENT_IMAGE is set, mount the host docker socket + CLI into the server container, forward HOST_REPO_DIR/HOST_MODEL_DIR/HOST_BENCH_LOGS/CLIENT_CONT_NAME, pre-pull the client image, and clean up the client container on teardown. - amd-master.yaml: enable the sibling client on dsv4-fp4-mi355x-sglang-disagg-agentic-hicache via CLIENT_IMAGE. The separate-client-NODE mode is intentionally not ported. * agentic: use server image for sibling client so upstream CI can pull it The pre-baked rocm/pytorch-private aiperf client image is not pullable by upstream CI runners. Reuse the (public) server image as CLIENT_IMAGE and build aiperf on the fly from /workspace/utils/aiperf, matching the co-located path. Gate the pre-baked-venv env (AIPERF_USE_PREBUILT / AIPERF_VENV) behind an optional CLIENT_AIPERF_VENV so a real pre-baked client image can still opt in. --- benchmarks/multi_node/amd_utils/job.slurm | 27 +++++++- .../multi_node/amd_utils/server_sglang.sh | 61 +++++++++++++++++++ configs/amd-master.yaml | 7 +++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 53d7e2bfcd..404260ae83 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -326,6 +326,8 @@ export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${S # dated tags are garbage-collected (manifest unknown) VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260629-e667ebb}" ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" +# Separate agentic benchmark-client container (see CLIENT_IMAGE handling below). +CLIENT_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_client_${SLURM_JOB_ID}" export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) @@ -502,6 +504,27 @@ echo "[config] wrote HiCache/Mooncake settings -> $HICACHE_MC_CONFIG" # Engine-specific container filter for pre-clean CONT_FILTER="name=^container_${ENGINE}_" +# ============================================================================= +# Optional: separate benchmark-client image (agentic runs) — node-0 sibling +# ============================================================================= +# When CLIENT_IMAGE is set, node 0 runs the aiperf trace replay in its own +# sibling container built from CLIENT_IMAGE (which ships a pre-baked aiperf + +# deps), instead of rebuilding the aiperf venv inside the server container every +# run. Give the server container access to the host docker socket + CLI and the +# host paths the sibling needs for its bind mounts. These fragments are expanded +# at submit time and injected into the server `docker run` below; empty (no-op) +# when CLIENT_IMAGE is unset, so the in-container aiperf path is unchanged. +CLIENT_DOCKER_MOUNTS="" +CLIENT_DOCKER_ENV="" +if [[ -n "${CLIENT_IMAGE:-}" ]]; then + HOST_DOCKER_BIN="$(command -v docker || echo /usr/bin/docker)" + CLIENT_DOCKER_MOUNTS="-v /var/run/docker.sock:/var/run/docker.sock -v ${HOST_DOCKER_BIN}:/usr/bin/docker" + CLIENT_DOCKER_ENV="-e CLIENT_IMAGE=${CLIENT_IMAGE} -e HOST_REPO_DIR=${DI_REPO_DIR} -e HOST_MODEL_DIR=${MODEL_DIR} -e HOST_BENCH_LOGS=${BENCHMARK_LOGS_DIR} -e CLIENT_CONT_NAME=${CLIENT_CONT_NAME}" + echo "[client] node-0 sibling benchmark-client image enabled: ${CLIENT_IMAGE}" + # Best-effort pre-pull on all nodes so node 0's sibling launch doesn't stall. + srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD pull '"$CLIENT_IMAGE"' >/dev/null 2>&1 || true' 2>/dev/null || true +fi + srun \ --nodelist="$SELECTED_NODELIST_SRUN" \ --kill-on-bad-exit=1 \ @@ -662,9 +685,11 @@ fi -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ ${EXTRA_DOCKER_MOUNTS:-} \ + ${CLIENT_DOCKER_MOUNTS} \ \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ ${DOCKER_ENV_COMMON[*]} \ ${DOCKER_ENV_ENGINE[*]} \ + ${CLIENT_DOCKER_ENV} \ --name \"$DOCKER_CONT_NAME\" \ --entrypoint \"\" \ \"$DOCKER_IMAGE_NAME\" bash -lc ' @@ -681,7 +706,7 @@ exit \$DOCKER_EXIT_CODE " if [[ "${KEEP_CONTAINERS}" != "1" ]]; then - srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' + srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' '"$CLIENT_CONT_NAME"' 2>/dev/null || true' # Clean up vLLM external router container on node 0 if [[ "$ENGINE" == "vllm-disagg" && "$ROUTER_TYPE" == "vllm-router" ]]; then diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 05fdc7dbb3..c57c7c3103 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -801,10 +801,71 @@ if [ "$NODE_RANK" -eq 0 ]; then echo "Benchmark runner: bench.sh (fixed-seq-len)" fi + IS_AGENTIC_RUN=0 + if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + IS_AGENTIC_RUN=1 + fi + if [[ "${EVAL_ONLY:-false}" == "true" ]]; then echo "EVAL_ONLY mode: skipping throughput benchmark" elif [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BENCH_CMD" + elif [[ -n "${CLIENT_IMAGE:-}" && "$IS_AGENTIC_RUN" == "1" ]]; then + # Separate client image (node-0 sibling container): run the aiperf trace + # replay in its own sibling container built from CLIENT_IMAGE (which ships + # a pre-baked aiperf + deps) instead of rebuilding the aiperf venv inside + # this server container. The server/router stay up in this container while + # the client container drives the benchmark against the router on + # localhost (--network host). job.slurm mounts the host docker socket + CLI + # into this container and forwards HOST_REPO_DIR / HOST_MODEL_DIR / + # HOST_BENCH_LOGS / CLIENT_CONT_NAME so the sibling can be launched here. + CLIENT_ENV_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/client.env" + mkdir -p "/run_logs/slurm_job-${SLURM_JOB_ID}" + # Forward the benchmark-relevant env (incl. runtime-computed metrics/flush + # URLs) to the client container; override the few paths/flags that differ + # inside the pre-baked image. Unset vars are skipped, so the client keeps + # its own defaults for anything not exported here. + { + for _v in ENGINE MODEL_NAME MODEL_PREFIX PRECISION FRAMEWORK SPEC_DECODING \ + DURATION MAX_MODEL_LEN RESULT_FILENAME RUNNER_NAME \ + AIPERF_SERVER_METRICS_URLS SERVER_FLUSH_URLS_CSV \ + ENABLE_METRICS IS_AGENTIC CLEAR_CACHE_BETWEEN_CONC \ + KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB \ + WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ + AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ + AIPERF_TRAJECTORY_START_MIN_RATIO AIPERF_TRAJECTORY_START_MAX_RATIO \ + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES ROUTER_PORT TQDM_MININTERVAL; do + if [[ -n "${!_v+x}" ]]; then printf '%s=%s\n' "$_v" "${!_v}"; fi + done + echo "INFMAX_CONTAINER_WORKSPACE=/workspace" + echo "AGENTIC_OUTPUT_DIR=/run_logs/slurm_job-${SLURM_JOB_ID}" + echo "HF_HOME=/run_logs/hf_cache" + echo "MODEL_DIR=/models" + # A pre-baked client image ships aiperf at CLIENT_AIPERF_VENV; when + # unset (e.g. reusing the server image, which carries no pre-baked + # venv), trace_replay builds aiperf on the fly from + # /workspace/utils/aiperf — same as the co-located path. + if [[ -n "${CLIENT_AIPERF_VENV:-}" ]]; then + echo "AIPERF_USE_PREBUILT=1" + echo "AIPERF_VENV=${CLIENT_AIPERF_VENV}" + fi + } > "$CLIENT_ENV_FILE" + + echo "Launching agentic benchmark in separate client container: ${CLIENT_IMAGE}" + docker rm -f "${CLIENT_CONT_NAME}" 2>/dev/null || true + set -x + docker run --rm --network host \ + --name "${CLIENT_CONT_NAME}" \ + --shm-size 32G \ + -v "${HOST_REPO_DIR}:/workspace" \ + -v "${HOST_MODEL_DIR}:/models" \ + -v /tmp:/run_logs \ + -v "${HOST_BENCH_LOGS}:/benchmark_logs" \ + --env-file "${CLIENT_ENV_FILE}" \ + --entrypoint "" \ + "${CLIENT_IMAGE}" \ + bash -lc "cd /workspace/benchmarks/multi_node/amd_utils && bash trace_replay.sh /models ${MODEL_NAME} \"${BENCH_MAX_CONCURRENCY}\" /run_logs/slurm_job-${SLURM_JOB_ID}" + set +x else set -x eval "$BENCH_CMD" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 37d46638da..b1a54f8a7d 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3119,6 +3119,13 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: dp-attn: false additional-settings: - "PREFILL_NODES=1" + # Node-0 sibling client: run the aiperf trace-replay in its own sibling + # container on node 0 (via the host docker socket) instead of inside the + # server container. Reuse the (publicly pullable) server image so + # upstream CI can fetch it; aiperf is built on the fly from + # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) + # selects the sibling-container mode. + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" decode: num-worker: 1 tp: 8 From 8278751f00c226ff526c0a4a51253e0f404e7011 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 10 Jul 2026 13:10:07 +0000 Subject: [PATCH 11/37] enable log info and extend timeout Signed-off-by: Theresa Shan --- benchmarks/multi_node/amd_utils/models.yaml | 47 ++++++++++++---- .../multi_node/amd_utils/server_sglang.sh | 55 +++++++++++++++---- 2 files changed, 80 insertions(+), 22 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 24a588375d..ef8a708447 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -363,16 +363,43 @@ DeepSeek-R1-0528-MXFP4-v2: chunked_prefill_size: 262144 cuda_graph_bs_range: "1-128" -# DeepSeek-V4-Pro PD-disaggregation recipe (MI355X, SGLang + MoRI). -# KV transfer = mori for both topologies (pure-TP and DEP); the DP path additionally -# routes the MoE all-to-all through mori (--moe-a2a-backend mori) with dp-attention. -# DSv4-specific kernel routing (unified_kv_triton, AITER indexer, fp8 wo_a fallback, -# thinking/reasoning-effort, dispatch dtypes, per-role PER_RANK dispatch tokens) is set -# in env.sh's DeepSeek-V4-Pro block. The bench client uses --dsv4 framing (bench.sh). -# prefill.disable_cuda_graph routes prefill to --disable-cuda-graph; decode keeps -# --cuda-graph-bs. See dsv4_mi355x_sglang_disagg_plan.md. -DeepSeek-V4-Pro: - base_flags: "--decode-log-interval 100 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori" +DeepSeek-V4-Pro-AgentX: + base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" + dp_flags: "--enable-dp-attention --enable-prefill-delayer --enable-two-batch-overlap" + ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" + prefill: + mem_fraction_static: 0.8 + disable_radix_cache: false + disable_cuda_graph: false + dp: + max_running_requests: 1024 + chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" # dsv4 compressor kernel uint16 token cap (255*256) + context_length: 1048576 + # max_total_tokens: 1048576 + no_dp: + max_running_requests: 128 + # Small prefill chunks interleave long-context agentic prefills across + # requests instead of letting one ~100K-token prefill monopolize the + # engine (the conc>=16 queue-saturation / decode-stall failure mode). + # Mirrors the single-node DSv4 agentic recipe (dsv4_fp4_mi355x.sh=8192). + # Was 65280 (255*256, the dsv4 compressor kernel uint16 token cap); 8192 + # (32*256) stays a page-size multiple well under that cap. + chunked_prefill_size: 8192 + context_length: 1048576 + max_total_tokens: 1048576 + decode: + mem_fraction_static: 0.85 + prefill_round_robin_balance: true + disagg_decode_enable_radix_cache: false + dp: + max_running_requests: 1024 + cuda_graph_bs_range: "1-128" + no_dp: + max_running_requests: 128 + cuda_graph_bs_range: "1-128" + +DeepSeek-V4-Pro-DI: + base_flags: "--decode-log-interval 100 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori" dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" prefill: diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index c57c7c3103..f9aa3bb7f0 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -82,14 +82,24 @@ import yaml, sys, os config_path = '${MODELS_YAML}' model_name = '${MODEL_NAME}' +# Select the models.yaml recipe variant by run type: agentic runs (IS_AGENTIC) +# use the '-AgentX' entry, non-agentic disaggregated runs use '-DI'. +# Fall back to the bare model name if the variant-specific key is absent. +is_agentic = '${IS_AGENTIC:-0}'.strip().lower() in ('1', 'true') +model_key = f'{model_name}-AgentX' if is_agentic else f'{model_name}-DI' + with open(config_path) as f: models = yaml.safe_load(f) -if model_name not in models: - print(f'echo \"ERROR: Model {model_name} not in models.yaml\"; exit 1') - sys.exit(0) +if model_key not in models: + if model_name in models: + model_key = model_name + else: + print(f'echo \"ERROR: Model {model_key} not in models.yaml\"; exit 1') + sys.exit(0) -m = models[model_name] +m = models[model_key] +print(f'echo \"Selected models.yaml entry: {model_key} (IS_AGENTIC={is_agentic})\"') def eval_formula(val): \"\"\"Evaluate chunked_prefill_size: if string, resolve variable names from env and compute.\"\"\" @@ -233,6 +243,7 @@ if [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "True" ]] || [[ "$PREFILL_DISABLE_CUDA_GR else PREFILL_MODE_FLAGS="--mem-fraction-static ${PREFILL_MEM_FRACTION_STATIC} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --cuda-graph-bs ${prefill_cuda_graph_bs[*]} " fi + if [[ "$PREFILL_DISABLE_RADIX_CACHE" == "True" ]] || [[ "$PREFILL_DISABLE_RADIX_CACHE" == "true" ]]; then PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --disable-radix-cache" fi @@ -472,7 +483,7 @@ if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then } # HiCache capacity via --hicache-ratio (scales with GPU KV pool). - HICACHE_RATIO="${HICACHE_RATIO:-16}" + HICACHE_RATIO="${HICACHE_RATIO:-5}" build_hicache_flags() { echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache --hicache-ratio ${HICACHE_RATIO} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" @@ -507,15 +518,28 @@ fi # Container Synchronization # ============================================================================= +# sync.py barrier/health-barrier exits 1 on timeout (and prints which +# node/port never became ready), but without an explicit check here the +# script would silently continue past a timed-out barrier -- printing a +# misleading "success" message and launching the next stage against +# servers/routers that never actually came up, instead of failing fast. +run_barrier_or_die() { + local desc="$1" cmd="$2" + if ! eval "$cmd"; then + echo "FATAL: ${desc} failed — see the sync.py timeout output above for which node/port never became ready." >&2 + exit 1 + fi +} + echo "Waiting at the container creation barrier on $host_name" -python3 $SGLANG_WS_PATH/sync.py barrier \ +run_barrier_or_die "container creation barrier" "python3 $SGLANG_WS_PATH/sync.py barrier \ --local-ip ${host_ip} \ --local-port 5000 \ --enable-port \ --node-ips ${IPADDRS} \ --node-ports 5000 \ --wait-for-all-ports \ - --timeout 300 + --timeout 300" # ============================================================================= @@ -668,7 +692,7 @@ if [ "$NODE_RANK" -eq 0 ]; then --node-ips ${IPADDRS} \ --node-ports 8000 \ --wait-for-all-ports \ - --timeout 1800" + --timeout 2400" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BARRIER_CMD" @@ -711,7 +735,7 @@ if [ "$NODE_RANK" -eq 0 ]; then if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $ROUTER_CMD" else - ROUTER_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/proxy_${host_name}.log" + ROUTER_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/router_${host_name}.log" # sgl-router (Rust/tracing) emits ANSI color codes. NO_COLOR asks it to # skip them at the source; the sed strip guarantees a clean file even if # it doesn't honor NO_COLOR. Both branches use process substitution so @@ -741,7 +765,7 @@ if [ "$NODE_RANK" -eq 0 ]; then --node-ports 30000 \ --wait-for-all-health \ --health-endpoint /readiness \ - --timeout 1800" + --timeout 3000" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $HEALTH_BARRIER_CMD" @@ -827,9 +851,13 @@ if [ "$NODE_RANK" -eq 0 ]; then # its own defaults for anything not exported here. { for _v in ENGINE MODEL_NAME MODEL_PREFIX PRECISION FRAMEWORK SPEC_DECODING \ - DURATION MAX_MODEL_LEN RESULT_FILENAME RUNNER_NAME \ + DURATION MAX_MODEL_LEN RESULT_FILENAME RUNNER_NAME RUNNER_TYPE IMAGE \ AIPERF_SERVER_METRICS_URLS SERVER_FLUSH_URLS_CSV \ ENABLE_METRICS IS_AGENTIC CLEAR_CACHE_BETWEEN_CONC \ + DISAGG IS_MULTINODE \ + TP EP_SIZE DP_ATTENTION DCP_SIZE PCP_SIZE \ + PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_HARDWARE \ + DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_HARDWARE \ KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB \ WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ @@ -838,7 +866,10 @@ if [ "$NODE_RANK" -eq 0 ]; then if [[ -n "${!_v+x}" ]]; then printf '%s=%s\n' "$_v" "${!_v}"; fi done echo "INFMAX_CONTAINER_WORKSPACE=/workspace" - echo "AGENTIC_OUTPUT_DIR=/run_logs/slurm_job-${SLURM_JOB_ID}" + # Do NOT pin AGENTIC_OUTPUT_DIR: it must default to /workspace (the + # host repo mount == GITHUB_WORKSPACE) so the aggregated + # ${RESULT_FILENAME}_conc.json lands where the workflow guard globs + # it. /workspace is bind-mounted writable, same as the co-located path. echo "HF_HOME=/run_logs/hf_cache" echo "MODEL_DIR=/models" # A pre-baked client image ships aiperf at CLIENT_AIPERF_VENV; when From 0225236c358916e5b21b92e35d997f8284bd4a3e Mon Sep 17 00:00:00 2001 From: Duyi-Wang Date: Sat, 11 Jul 2026 13:20:40 +0800 Subject: [PATCH 12/37] fix: address PR 2170 review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the DSv4 MI355X agentic perf-changelog entry to point at PR 2170 with an accurate description, and move hf_dir into the DeepSeek-V4-Pro YAML block so the SGLang model path extractor can read it. 中文：修复 PR 2170 的 review 反馈：将 DSv4 MI355X agentic 的 perf-changelog 条目改为指向 PR 2170 并更新为准确描述，同时把 hf_dir 移入 DeepSeek-V4-Pro YAML 配置块，确保 SGLang 模型路径提取逻辑可以读取。 --- perf-changelog.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a7972e2475..7c3b771328 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4754,6 +4754,7 @@ - config-keys: - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache description: - - "Bump the SGLang ROCm image from 20260706 to 20260709 and retire the DeepSeek-V4 compress-state, DSA paged-MQA, AITER instruction-shape, and HiCache host-pool shims that are already included upstream." - - "Remove the SWA re-prefill and unified-KV HiCache overlays after their upstream PRs merged; retain only patch code without a merged upstream equivalent." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2127 + - "Add DeepSeek-V4-Pro FP4 MI355X SGLang-disagg agentic-coding benchmark with HiCache DRAM KV offloading." + - "Add the multi-node agentic runtime plumbing: node-0 sibling benchmark-client container, per-worker metrics scrape, per-concurrency cache flushing, trace replay client, and one allocation per concurrency." + - "Image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2170 From cd4c3f911d52aae9da1d3b0852dfd2739e83e151 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 13 Jul 2026 02:33:28 +0000 Subject: [PATCH 13/37] setup_deps: remove prefill bootstrap-desync patch entirely The disaggregation-prefill bootstrap-desync patch was already disabled (commented out); drop the function, its comment block, and the stale invocation now that the fix is carried upstream in the pinned image. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/setup_deps.sh | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh index f1ae30b925..8fa49dfb27 100644 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ b/benchmarks/multi_node/amd_utils/setup_deps.sh @@ -78,49 +78,6 @@ install_amd_quark() { _SETUP_INSTALLED+=("amd-quark") } -# --------------------------------------------------------------------------- -# SGLang: prevent TP-rank collective desync deadlock in disaggregation prefill. -# -# resolve_waiting_queue_bootstrap() runs poll_and_all_reduce_attn_cp_tp_group() -# over `candidates`. The upstream candidate set (all non-aborted waiting reqs) -# can differ across TP ranks, so some ranks enter the all_reduce while others -# skip it -> hang. Narrow candidates to optimistic (pending_bootstrap) requests, -# which is consistent across ranks and is the only set finalize_bootstrap acts on. -# --------------------------------------------------------------------------- -patch_disagg_prefill_bootstrap_desync() { - python3 -c ' -import os, sys - -target = "/sgl-workspace/sglang/python/sglang/srt/disaggregation/prefill.py" -if not os.path.isfile(target): - print("[SETUP] disaggregation/prefill.py not found, skipping") - sys.exit(0) - -src = open(target).read() - -old = " candidates = [req for req in self.waiting_queue if not is_aborted(req)]" -new = ( - " candidates = [\n" - " req\n" - " for req in self.waiting_queue\n" - " if req.pending_bootstrap and not is_aborted(req)\n" - " ]" -) - -if new in src: - print("[SETUP] prefill bootstrap-desync patch already applied") - sys.exit(0) - -if old not in src: - print("[SETUP] WARN: resolve_waiting_queue_bootstrap pattern not found — sglang version may have changed") - sys.exit(0) - -open(target, "w").write(src.replace(old, new)) -print("[SETUP] Patched: disaggregation/prefill.py resolve_waiting_queue_bootstrap candidates") -' - _SETUP_INSTALLED+=("prefill-bootstrap-desync-fix") -} - # --------------------------------------------------------------------------- # SGLang: Install latest transformers for GLM model type support. # @@ -162,8 +119,6 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" else - # patch_disagg_prefill_bootstrap_desync - install_transformers_glm5 fi From a361ad84bb8d9b627b1019135b72d3a8d3fc1c57 Mon Sep 17 00:00:00 2001 From: billishyahao Date: Mon, 13 Jul 2026 03:12:46 +0000 Subject: [PATCH 14/37] fix perf changelog --- perf-changelog.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7c3b771328..f9f75b9804 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4743,14 +4743,6 @@ - "Bump vLLM image to v0.25.0 for DeepSeek-V4-Pro FP4 on B200 and B300." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2169 -- config-keys: - - qwen3.5-fp8-gb300-dynamo-sglang - description: - - "Add Qwen3.5-397B-A17B-FP8 GB300 disaggregated multinode SGLang benchmarks via Dynamo" - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 - - config-keys: - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache description: @@ -4758,3 +4750,4 @@ - "Add the multi-node agentic runtime plumbing: node-0 sibling benchmark-client container, per-worker metrics scrape, per-concurrency cache flushing, trace replay client, and one allocation per concurrency." - "Image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2170 + From 775c22cbae3813fed4956f2b19b7ea5a59c6aa2a Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 13 Jul 2026 06:41:22 +0000 Subject: [PATCH 15/37] consolidate sglang agentic envs Signed-off-by: Theresa Shan --- benchmarks/multi_node/amd_utils/env.sh | 55 -------------------------- 1 file changed, 55 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index 966cde4589..2a3005c644 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -272,61 +272,6 @@ else # 1 mirrors router logs to stdout via tee (useful for live debugging). export SGLANG_ROUTER_STDOUT_LOGS="${SGLANG_ROUTER_STDOUT_LOGS:-0}" - # ========================================================================= - # DeepSeek-V4-Pro PD recipe overrides - # Placed at the end of the SGLang env block so it wins over the global - # MoRI/SGLang defaults set above. Mirrors the validated DSv4 manual PD - # commands (see dsv4_mi355x_sglang_disagg_plan.md §2). Only the SGLang/MoRI - # env knobs are pinned here; CLI flags live in models.yaml and the cluster - # NIC/socket vars (NCCL_IB_HCA, *_SOCKET_IFNAME, IBDEVICES) stay runner-derived. - # ========================================================================= - if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then - # MoRI dispatch/combine dtypes: auto for both roles (not the fp8 split default) - export SGLANG_MORI_DISPATCH_DTYPE=auto - export MORI_COMBINE_DTYPE_PREFILL=auto - export MORI_COMBINE_DTYPE_DECODE=auto - - # Per-role MoRI dispatch sizing (used by the harness chunked/MoE math) - export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 - export MORI_MAX_DISPATCH_TOKENS_DECODE=64 - unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL - unset MORI_MOE_MAX_INPUT_TOKENS_DECODE - - # PER_RANK dispatch tokens are pinned independently of the sizing above - # (16384 prefill / 128 decode in the reference recipe). server_sglang.sh - # prefers these over the MORI_MAX_DISPATCH_TOKENS_* coupling when set. - export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL=16384 - export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE=128 - - # Fixed inter-kernel switch threshold (not derived). NOTE: the DP+EP path in - # server_sglang.sh recomputes this dynamically for the DEP topology. - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=4096 - - # Overlap plan stream on for DSv4 (global default is 0) - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 - - # DSv4 model kernel routing (mirrors the single-node / manual PD recipe) - export SGLANG_DEFAULT_THINKING=1 - export SGLANG_DSV4_REASONING_EFFORT=max - export SGLANG_USE_ROCM700A=0 - export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton - export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false - export SGLANG_OPT_USE_FUSED_COMPRESS=true - export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true - export SGLANG_OPT_FP8_WO_A_GEMM=false - export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false - export SGLANG_OPT_USE_TOPK_V2=false - export SGLANG_OPT_USE_AITER_INDEXER=true - export SGLANG_OPT_USE_TILELANG_INDEXER=false - export SGLANG_OPT_USE_TILELANG_MHC_PRE=false - export SGLANG_OPT_USE_TILELANG_MHC_POST=false - export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 - export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false - export SGLANG_ROCM_USE_MULTI_STREAM=false - export AITER_BF16_FP8_MOE_BOUND=0 - export SGLANG_EAGER_INPUT_NO_COPY=true - fi - # FIXME: WA for latest upstream 0305 image export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH} From 36613c78bedb391c4004748e235277bb5d206b17 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 13 Jul 2026 08:18:30 +0000 Subject: [PATCH 16/37] select models.yaml recipe by IS_AGENTIC (AgentX/DI variants) Resolve the models.yaml entry per run type: agentic runs (IS_AGENTIC) use the '-AgentX' recipe, non-agentic disaggregated runs use '-DI', falling back to the bare model name when no variant key exists. Apply the same logic in both server_sglang.sh (lookup) and job.slurm (validation guard) so a bare MODEL_NAME (e.g. DeepSeek-V4-Pro) correctly maps to the suffixed key. Add DeepSeek-V4-Pro-DI and rename the agentic recipe to DeepSeek-V4-Pro-AgentX. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/job.slurm | 25 +++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 404260ae83..1950353268 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -42,13 +42,26 @@ if [[ -z "${DOCKER_IMAGE_NAME:-}" ]]; then fi MODEL_NAME="${MODEL_NAME:-None}" -if ! grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then - echo "Error: Model '$MODEL_NAME' not found in $MODELS_YAML" - echo "Available models:" - grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/ - /' - exit 1 +# Resolve the models.yaml entry the same way server_sglang.sh does: agentic runs +# (IS_AGENTIC) use the '-AgentX' recipe, non-agentic disaggregated runs use +# '-DI'. Fall back to the bare model name if the variant key is absent. +# MODEL_NAME itself is left unchanged so env.sh/server_sglang.sh still see the base name. +if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + MODEL_YAML_KEY="${MODEL_NAME}-AgentX" +else + MODEL_YAML_KEY="${MODEL_NAME}-DI" +fi +if ! grep -q "^${MODEL_YAML_KEY}:" "$MODELS_YAML"; then + if grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then + MODEL_YAML_KEY="$MODEL_NAME" + else + echo "Error: Model '$MODEL_YAML_KEY' (nor bare '$MODEL_NAME') not found in $MODELS_YAML" + echo "Available models:" + grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/ - /' + exit 1 + fi fi -echo "Model found: $MODEL_NAME" +echo "Model found: $MODEL_YAML_KEY (MODEL_NAME=$MODEL_NAME, IS_AGENTIC=${IS_AGENTIC:-0})" RUN_FILE="server.sh" echo "Runfile set: $RUN_FILE" From 9ccd03214300046e0a87403fbd2345305d8c5f29 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 13 Jul 2026 08:59:07 +0000 Subject: [PATCH 17/37] DPTP enabling. Signed-off-by: Theresa Shan --- configs/amd-master.yaml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b1a54f8a7d..c6675769af 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3109,7 +3109,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - dram-utilization: 0.80 search-space: - spec-decoding: "none" - conc-list: [ 2,4,8,16,32 ] + conc-list: [ 2,4,8,16 ] kv-offloading: dram kv-offload-backend: hicache prefill: @@ -3134,3 +3134,29 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: additional-settings: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" + - spec-decoding: "none" + conc-list: [ 32, 64, 128 ] + kv-offloading: dram + kv-offload-backend: hicache + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "PREFILL_NODES=1" + # Node-0 sibling client: run the aiperf trace-replay in its own sibling + # container on node 0 (via the host docker socket) instead of inside the + # server container. Reuse the (publicly pullable) server image so + # upstream CI can fetch it; aiperf is built on the fly from + # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) + # selects the sibling-container mode. + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" From 0df082aa46413f21ca17590ac94fdd310ed16ebb Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 13 Jul 2026 16:01:15 +0000 Subject: [PATCH 18/37] tune hicache ratio, router barrier timeout, and agentic conc sweep - server_sglang.sh: lower default HICACHE_RATIO 24->5 (cut host-pool pin/registration startup cost) and raise router health barrier timeout 1800->3000 so a slow server bring-up no longer trips the cross-node router wait first. - amd-master.yaml: restore conc 32 on the non-DP-attn (TP8) point and comment out the DP-attn c64/128 sweep block. - models.yaml: drop explicit max_total_tokens override for the AgentX dp variant (let it derive from context_length). Co-authored-by: Cursor --- configs/amd-master.yaml | 54 ++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c6675769af..ca35755aca 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3109,7 +3109,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - dram-utilization: 0.80 search-space: - spec-decoding: "none" - conc-list: [ 2,4,8,16 ] + conc-list: [ 2,4,8,16,32 ] kv-offloading: dram kv-offload-backend: hicache prefill: @@ -3134,29 +3134,29 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: additional-settings: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" - - spec-decoding: "none" - conc-list: [ 32, 64, 128 ] - kv-offloading: dram - kv-offload-backend: hicache - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - # Node-0 sibling client: run the aiperf trace-replay in its own sibling - # container on node 0 (via the host docker socket) instead of inside the - # server container. Reuse the (publicly pullable) server image so - # upstream CI can fetch it; aiperf is built on the fly from - # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) - # selects the sibling-container mode. - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" + # - spec-decoding: "none" + # conc-list: [ 64, 128 ] + # kv-offloading: dram + # kv-offload-backend: hicache + # prefill: + # num-worker: 1 + # tp: 8 + # ep: 1 + # dp-attn: true + # additional-settings: + # - "PREFILL_NODES=1" + # # Node-0 sibling client: run the aiperf trace-replay in its own sibling + # # container on node 0 (via the host docker socket) instead of inside the + # # server container. Reuse the (publicly pullable) server image so + # # upstream CI can fetch it; aiperf is built on the fly from + # # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) + # # selects the sibling-container mode. + # - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" + # decode: + # num-worker: 1 + # tp: 8 + # ep: 1 + # dp-attn: true + # additional-settings: + # - "DECODE_NODES=1" + # - "DECODE_MTP_SIZE=0" From f588eda65dfe7e7bf7f12fcf5e93fac00309837c Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 05:05:24 +0000 Subject: [PATCH 19/37] raise hicache ratio to 16 Signed-off-by: Theresa Shan --- benchmarks/multi_node/amd_utils/server_sglang.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index f9aa3bb7f0..63bc4f1306 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -483,7 +483,7 @@ if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then } # HiCache capacity via --hicache-ratio (scales with GPU KV pool). - HICACHE_RATIO="${HICACHE_RATIO:-5}" + HICACHE_RATIO="${HICACHE_RATIO:-16}" build_hicache_flags() { echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache --hicache-ratio ${HICACHE_RATIO} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" From c9e7a32c42014610aa178786c17c126461d4ccf9 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 05:09:23 +0000 Subject: [PATCH 20/37] fix: restore qwen3.5 changelog entry removed during rebase perf-changelog.yaml must only gain entries; re-adding the upstream qwen3.5-fp8-gb300-dynamo-sglang block fixes process_changelog.py. Co-authored-by: Cursor --- perf-changelog.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f9f75b9804..1b75bd77c6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4743,6 +4743,14 @@ - "Bump vLLM image to v0.25.0 for DeepSeek-V4-Pro FP4 on B200 and B300." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2169 +- config-keys: + - qwen3.5-fp8-gb300-dynamo-sglang + description: + - "Add Qwen3.5-397B-A17B-FP8 GB300 disaggregated multinode SGLang benchmarks via Dynamo" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 + - config-keys: - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache description: From 3cc26257fc7a52639a229d34bf102a5f92efe4e8 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 05:13:14 +0000 Subject: [PATCH 21/37] fix: validate agentic hicache master config for changelog gate Use kv-offload-backend object form and add kv-p2p-transfer: mori for the disaggregated dsv4-fp4-mi355x-sglang-disagg-agentic-hicache entry. Co-authored-by: Cursor --- configs/amd-master.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ca35755aca..9d0c5c8d1b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -3102,6 +3102,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: runner: cluster:mi355x-amds precision: fp4 framework: sglang-disagg + kv-p2p-transfer: mori multinode: true disagg: true scenarios: @@ -3111,7 +3112,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - spec-decoding: "none" conc-list: [ 2,4,8,16,32 ] kv-offloading: dram - kv-offload-backend: hicache + kv-offload-backend: { name: hicache } prefill: num-worker: 1 tp: 8 @@ -3137,7 +3138,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: # - spec-decoding: "none" # conc-list: [ 64, 128 ] # kv-offloading: dram - # kv-offload-backend: hicache + # kv-offload-backend: { name: hicache } # prefill: # num-worker: 1 # tp: 8 From 444e44bc64ee3ed92d659b7f0c6cb8d7779be924 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 05:25:29 +0000 Subject: [PATCH 22/37] feat: wire KV_P2P_TRANSFER env into SGLang launch flags Override models.yaml --disaggregation-transfer-backend when kv-p2p-transfer from amd-master.yaml is set via CI, and forward KV_P2P_TRANSFER to the agentic client env for result metadata. Co-authored-by: Cursor --- .../multi_node/amd_utils/server_sglang.sh | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 63bc4f1306..819353f7dd 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -325,6 +325,22 @@ echo "Server flush endpoints: ${SERVER_FLUSH_URLS[@]}" # Configuration Builder Functions # ============================================================================= +# KV_P2P_TRANSFER (from amd-master.yaml kv-p2p-transfer) overrides the +# --disaggregation-transfer-backend baked into models.yaml base_flags. +apply_kv_p2p_transfer_override() { + local flags="$1" + if [[ -z "${KV_P2P_TRANSFER:-}" ]]; then + printf '%s' "$flags" + return 0 + fi + local stripped + stripped="$(echo "$flags" | sed -E 's/--disaggregation-transfer-backend[[:space:]]+[^[:space:]]+//g')" + stripped="${stripped#"${stripped%%[![:space:]]*}"}" + stripped="${stripped%"${stripped##*[![:space:]]}"}" + echo "[KV_P2P] Using disaggregation-transfer-backend=${KV_P2P_TRANSFER} (KV_P2P_TRANSFER env)" >&2 + printf '%s --disaggregation-transfer-backend %s' "$stripped" "$KV_P2P_TRANSFER" +} + build_server_config() { local mode="$1" local model_name="$2" @@ -357,7 +373,8 @@ build_server_config() { fi # Get model-specific configuration from YAML-loaded variables - local base_config="$MODEL_BASE_FLAGS" + local base_config + base_config="$(apply_kv_p2p_transfer_override "$MODEL_BASE_FLAGS")" local mtp_config="" local dp_config="" local ep_config="" @@ -858,7 +875,7 @@ if [ "$NODE_RANK" -eq 0 ]; then TP EP_SIZE DP_ATTENTION DCP_SIZE PCP_SIZE \ PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_HARDWARE \ DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_HARDWARE \ - KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB \ + KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB KV_P2P_TRANSFER \ WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ AIPERF_TRAJECTORY_START_MIN_RATIO AIPERF_TRAJECTORY_START_MAX_RATIO \ From cadad833612826199bd4d3ff86dd383860aa0799 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 05:34:32 +0000 Subject: [PATCH 23/37] chore: drop max_total_tokens from DSv4 AgentX prefill no_dp recipe Let SGLang auto-size the prefill KV pool instead of pinning 1M tokens. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/models.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index ef8a708447..a3b3436400 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -386,7 +386,6 @@ DeepSeek-V4-Pro-AgentX: # (32*256) stays a page-size multiple well under that cap. chunked_prefill_size: 8192 context_length: 1048576 - max_total_tokens: 1048576 decode: mem_fraction_static: 0.85 prefill_round_robin_balance: true From c68375d37b93fadd7a93abaef2134a14d22059ef Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 06:04:57 +0000 Subject: [PATCH 24/37] tune: raise router barrier timeout to 3000s for worker nodes Give prefill/decode workers more time to wait for the node-0 proxy during long agentic server startup. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/server_sglang.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 819353f7dd..a8f3e9b376 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -1091,7 +1091,7 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then --node-ips ${NODE0_ADDR} \ --node-ports 30000 \ --wait-for-all-ports \ - --timeout 1800" + --timeout 3000" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BARRIER_CMD" From 6344aaa5cfb51187bef3a69a9c39fcdf7bdf485e Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 06:09:17 +0000 Subject: [PATCH 25/37] test: fix sweep config fixtures for kv-p2p-transfer validation Add kv-p2p-transfer to disagg multinode test configs and use KVOffloadBackendMetadata dict form for kv-offload-backend fields. Co-authored-by: Cursor --- .../test_generate_sweep_configs.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index b77a1cdc48..a4baa81ac9 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -68,6 +68,7 @@ def sample_multinode_config(): "runner": "gb200", "multinode": True, "disagg": True, + "kv-p2p-transfer": "nixl", "scenarios": { "fixed-seq-len": [ @@ -1999,7 +2000,7 @@ def test_runner_node_filter_expands_agentic_config_runner(self, sample_runner_co "tp": 8, "ep": 1, "kv-offloading": "dram", - "kv-offload-backend": "hicache", + "kv-offload-backend": {"name": "hicache"}, "conc-list": [64], } ], @@ -2041,7 +2042,7 @@ def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config): { "tp": 4, "kv-offloading": "dram", - "kv-offload-backend": "native", + "kv-offload-backend": {"name": "native"}, "conc-list": [32], }, { @@ -2049,7 +2050,7 @@ def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config): "dcp-size": 2, "pcp-size": 1, "kv-offloading": "dram", - "kv-offload-backend": "native", + "kv-offload-backend": {"name": "native"}, "conc-list": [32], }, { @@ -2057,14 +2058,14 @@ def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config): "dcp-size": 1, "pcp-size": 2, "kv-offloading": "dram", - "kv-offload-backend": "native", + "kv-offload-backend": {"name": "native"}, "conc-list": [32], }, { "tp": 4, "pp": 2, "kv-offloading": "dram", - "kv-offload-backend": "native", + "kv-offload-backend": {"name": "native"}, "conc-list": [32], }, ], @@ -2111,7 +2112,7 @@ def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_conf { "tp": 4, "kv-offloading": "dram", - "kv-offload-backend": "native", + "kv-offload-backend": {"name": "native"}, "conc-list": [32], }, ], @@ -2144,6 +2145,7 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): "runner": "gb200", "multinode": True, "disagg": True, + "kv-p2p-transfer": "nixl", "scenarios": { "agentic-coding": [ { @@ -2196,12 +2198,13 @@ def test_multinode_agentic_preserves_kv_offload_fields(self): "runner": "cluster:mi355x-amds", "multinode": True, "disagg": True, + "kv-p2p-transfer": "mori", "scenarios": { "agentic-coding": [{ "search-space": [{ "conc-list": [16], "kv-offloading": "dram", - "kv-offload-backend": "hicache", + "kv-offload-backend": {"name": "hicache"}, "prefill": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, }], @@ -2221,7 +2224,7 @@ def test_multinode_agentic_preserves_kv_offload_fields(self): assert len(result) == 1 assert result[0]["kv-offloading"] == "dram" - assert result[0]["kv-offload-backend"] == "hicache" + assert result[0]["kv-offload-backend"] == {"name": "hicache"} assert result[0]["exp-name"] == "dsv4_p1x8_d1x8_conc16_kvdram-hicache" @@ -2341,6 +2344,7 @@ def test_node_type_filters_apply_to_agentic_configs( "runner": "cluster:gb200-nv", "multinode": True, "disagg": True, + "kv-p2p-transfer": "nixl", "scenarios": { "agentic-coding": [{ "search-space": [ From 90af0b61b242a316b899f699243f5207f32cee34 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 07:29:18 +0000 Subject: [PATCH 26/37] refactor: centralize sync barrier timeouts for DSV4 startup Extract SYNC_BARRIER_TIMEOUT (3000s for DeepSeek-V4, 1800s otherwise) and apply it to server-up, router readiness, and prefill/decode proxy barriers. Also lower the default HICACHE_RATIO from 16 to 5. Co-authored-by: Cursor --- .../multi_node/amd_utils/server_sglang.sh | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index a8f3e9b376..a5cda8cc46 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -439,6 +439,16 @@ if [[ -n "$MODEL_NAME" ]]; then echo "Using model-specific configuration for: $MODEL_NAME" fi +# sync.py barrier timeout for server-up (port 8000). DSV4 needs more headroom. +# Override via SYNC_BARRIER_TIMEOUT if needed. +if [[ -z "${SYNC_BARRIER_TIMEOUT:-}" ]]; then + case "${MODEL_NAME}" in + *DeepSeek-V4*) SYNC_BARRIER_TIMEOUT=3000 ;; + *) SYNC_BARRIER_TIMEOUT=1800 ;; + esac +fi +echo "SYNC_BARRIER_TIMEOUT=${SYNC_BARRIER_TIMEOUT}s (model=${MODEL_NAME:-unset})" + # ============================================================================= # Optional KV cache offloading (HiCache) — enabled when # KV_OFFLOADING != none AND KV_OFFLOAD_BACKEND == hicache. @@ -500,7 +510,7 @@ if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then } # HiCache capacity via --hicache-ratio (scales with GPU KV pool). - HICACHE_RATIO="${HICACHE_RATIO:-16}" + HICACHE_RATIO="${HICACHE_RATIO:-5}" build_hicache_flags() { echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache --hicache-ratio ${HICACHE_RATIO} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" @@ -709,7 +719,7 @@ if [ "$NODE_RANK" -eq 0 ]; then --node-ips ${IPADDRS} \ --node-ports 8000 \ --wait-for-all-ports \ - --timeout 2400" + --timeout ${SYNC_BARRIER_TIMEOUT}" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BARRIER_CMD" @@ -782,7 +792,7 @@ if [ "$NODE_RANK" -eq 0 ]; then --node-ports 30000 \ --wait-for-all-health \ --health-endpoint /readiness \ - --timeout 3000" + --timeout ${SYNC_BARRIER_TIMEOUT}" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $HEALTH_BARRIER_CMD" @@ -1091,7 +1101,7 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then --node-ips ${NODE0_ADDR} \ --node-ports 30000 \ --wait-for-all-ports \ - --timeout 3000" + --timeout ${SYNC_BARRIER_TIMEOUT}" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BARRIER_CMD" @@ -1174,7 +1184,7 @@ else --node-ips ${NODE0_ADDR} \ --node-ports 30000 \ --wait-for-all-ports \ - --timeout 1800" + --timeout ${SYNC_BARRIER_TIMEOUT}" if [[ "$DRY_RUN" -eq 1 ]]; then echo "DRY RUN: $BARRIER_CMD" From 6b48abe979c0bb0dfa884eb4fe86c4d4a92071ba Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 10:58:57 +0000 Subject: [PATCH 27/37] fix: forward KV offload backend metadata to agentic client Pass KV_OFFLOAD_BACKEND_METADATA into server containers via job.slurm and forward it to sibling client containers in server_sglang client.env so process_agentic_result can write aggregate JSON. Derive KV_OFFLOAD_BACKEND from metadata in the DSv4 agentic recipe when unset. Co-authored-by: Cursor --- .../agentic/dsv4_fp4_mi355x_sglang-disagg.sh | 26 ++++++++++++++++--- benchmarks/multi_node/amd_utils/job.slurm | 1 + .../multi_node/amd_utils/server_sglang.sh | 2 +- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh index e5dde8d120..b2fc792d37 100755 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh @@ -70,12 +70,30 @@ export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}" # ── KV cache offloading (HiCache) ── # KV_OFFLOADING=none | dram (passed from YAML; default none for disagg). -# KV_OFFLOAD_BACKEND selects the backend when offloading is on; this recipe -# only implements HiCache, so "hicache" is the only supported value. -# HICACHE_TIER: L2 -> GPU + CPU-DRAM host pool. L3 -> + Mooncake store. +# KV_OFFLOAD_BACKEND selects the backend when offloading is on. CI supplies +# KV_OFFLOAD_BACKEND_METADATA (e.g. {"name":"hicache"}); derive the backend +# name from it when KV_OFFLOAD_BACKEND is unset. This recipe only implements +# HiCache, so local runs without metadata still default to "hicache". export KV_OFFLOADING="${KV_OFFLOADING:-none}" if [[ "$KV_OFFLOADING" != "none" ]]; then - export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-hicache}" + if [[ -z "${KV_OFFLOAD_BACKEND:-}" ]]; then + if [[ -n "${KV_OFFLOAD_BACKEND_METADATA:-}" ]]; then + KV_OFFLOAD_BACKEND=$(KV_OFFLOAD_BACKEND_METADATA="$KV_OFFLOAD_BACKEND_METADATA" python3 -c " +import json, os, sys +try: + meta = json.loads(os.environ['KV_OFFLOAD_BACKEND_METADATA']) +except json.JSONDecodeError as exc: + sys.exit(f'KV_OFFLOAD_BACKEND_METADATA must be valid JSON: {exc}') +name = meta.get('name') +if not isinstance(name, str) or not name: + sys.exit(\"KV_OFFLOAD_BACKEND_METADATA must contain a non-empty 'name'\") +print(name) +") || exit 1 + else + KV_OFFLOAD_BACKEND=hicache + fi + fi + export KV_OFFLOAD_BACKEND fi # HiCache/Mooncake tunables only matter when KV offloading is enabled. if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 1950353268..0912d7eaee 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -431,6 +431,7 @@ DOCKER_ENV_COMMON=( -e IS_AGENTIC=\${IS_AGENTIC:-0} -e KV_OFFLOADING=\${KV_OFFLOADING:-none} -e KV_OFFLOAD_BACKEND=\${KV_OFFLOAD_BACKEND:-} + -e KV_OFFLOAD_BACKEND_METADATA=\${KV_OFFLOAD_BACKEND_METADATA:-} -e TOTAL_CPU_DRAM_GB=\${TOTAL_CPU_DRAM_GB:-} -e ENABLE_METRICS=\${ENABLE_METRICS:-0} -e PREFILL_ROUTER_POLICY=\${PREFILL_ROUTER_POLICY:-random} diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index a5cda8cc46..231545fa7b 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -885,7 +885,7 @@ if [ "$NODE_RANK" -eq 0 ]; then TP EP_SIZE DP_ATTENTION DCP_SIZE PCP_SIZE \ PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_HARDWARE \ DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_HARDWARE \ - KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB KV_P2P_TRANSFER \ + KV_OFFLOADING KV_OFFLOAD_BACKEND KV_OFFLOAD_BACKEND_METADATA TOTAL_CPU_DRAM_GB KV_P2P_TRANSFER \ WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ AIPERF_TRAJECTORY_START_MIN_RATIO AIPERF_TRAJECTORY_START_MAX_RATIO \ From dfcc261da5eefae8504398b13f912fbc7846bdce Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 11:28:02 +0000 Subject: [PATCH 28/37] revert: drop metadata-derived KV_OFFLOAD_BACKEND from DSv4 recipe CI already sets KV_OFFLOAD_BACKEND from the workflow; keep the job.slurm and server_sglang metadata forwarding fixes without the redundant recipe logic. Co-authored-by: Cursor --- .../agentic/dsv4_fp4_mi355x_sglang-disagg.sh | 26 +++---------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh index b2fc792d37..e5dde8d120 100755 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh @@ -70,30 +70,12 @@ export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}" # ── KV cache offloading (HiCache) ── # KV_OFFLOADING=none | dram (passed from YAML; default none for disagg). -# KV_OFFLOAD_BACKEND selects the backend when offloading is on. CI supplies -# KV_OFFLOAD_BACKEND_METADATA (e.g. {"name":"hicache"}); derive the backend -# name from it when KV_OFFLOAD_BACKEND is unset. This recipe only implements -# HiCache, so local runs without metadata still default to "hicache". +# KV_OFFLOAD_BACKEND selects the backend when offloading is on; this recipe +# only implements HiCache, so "hicache" is the only supported value. +# HICACHE_TIER: L2 -> GPU + CPU-DRAM host pool. L3 -> + Mooncake store. export KV_OFFLOADING="${KV_OFFLOADING:-none}" if [[ "$KV_OFFLOADING" != "none" ]]; then - if [[ -z "${KV_OFFLOAD_BACKEND:-}" ]]; then - if [[ -n "${KV_OFFLOAD_BACKEND_METADATA:-}" ]]; then - KV_OFFLOAD_BACKEND=$(KV_OFFLOAD_BACKEND_METADATA="$KV_OFFLOAD_BACKEND_METADATA" python3 -c " -import json, os, sys -try: - meta = json.loads(os.environ['KV_OFFLOAD_BACKEND_METADATA']) -except json.JSONDecodeError as exc: - sys.exit(f'KV_OFFLOAD_BACKEND_METADATA must be valid JSON: {exc}') -name = meta.get('name') -if not isinstance(name, str) or not name: - sys.exit(\"KV_OFFLOAD_BACKEND_METADATA must contain a non-empty 'name'\") -print(name) -") || exit 1 - else - KV_OFFLOAD_BACKEND=hicache - fi - fi - export KV_OFFLOAD_BACKEND + export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-hicache}" fi # HiCache/Mooncake tunables only matter when KV offloading is enabled. if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then From aa95a53577f5b113ea527008be39cb4956efae45 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 11:54:45 +0000 Subject: [PATCH 29/37] fix: quote KV_OFFLOAD_BACKEND_METADATA in docker run env forwarding toJson() pretty-prints with embedded newlines, so the unquoted expansion in job.slurm's DOCKER_ENV_COMMON was word-split across the docker run command, corrupting its argument list ("docker: invalid reference format", exit 125). Quote the value so the whole JSON stays one token. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/job.slurm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 0912d7eaee..0ff5a2b169 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -431,7 +431,7 @@ DOCKER_ENV_COMMON=( -e IS_AGENTIC=\${IS_AGENTIC:-0} -e KV_OFFLOADING=\${KV_OFFLOADING:-none} -e KV_OFFLOAD_BACKEND=\${KV_OFFLOAD_BACKEND:-} - -e KV_OFFLOAD_BACKEND_METADATA=\${KV_OFFLOAD_BACKEND_METADATA:-} + -e KV_OFFLOAD_BACKEND_METADATA=\"\${KV_OFFLOAD_BACKEND_METADATA:-}\" -e TOTAL_CPU_DRAM_GB=\${TOTAL_CPU_DRAM_GB:-} -e ENABLE_METRICS=\${ENABLE_METRICS:-0} -e PREFILL_ROUTER_POLICY=\${PREFILL_ROUTER_POLICY:-random} From 81168337ec3d5d304668fcc8d4a428ea60943a49 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 12:33:58 +0000 Subject: [PATCH 30/37] Fix client.env generation to flatten multi-line env values docker run --env-file requires strict single-line KEY=VALUE entries. KV_OFFLOAD_BACKEND_METADATA carries pretty-printed multi-line JSON, which split into invalid env-file lines (e.g. '"name": "hicache",') and aborted the sibling client container launch with "docker: invalid env file ... contains whitespaces". Strip embedded newlines before writing each value so the JSON stays valid but single-line. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/server_sglang.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 231545fa7b..c5451e98ee 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -890,7 +890,17 @@ if [ "$NODE_RANK" -eq 0 ]; then AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ AIPERF_TRAJECTORY_START_MIN_RATIO AIPERF_TRAJECTORY_START_MAX_RATIO \ AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES ROUTER_PORT TQDM_MININTERVAL; do - if [[ -n "${!_v+x}" ]]; then printf '%s=%s\n' "$_v" "${!_v}"; fi + if [[ -n "${!_v+x}" ]]; then + _val="${!_v}" + # docker --env-file requires one KEY=VALUE per line with no + # embedded newlines; KV_OFFLOAD_BACKEND_METADATA carries + # pretty-printed multi-line JSON, which otherwise splits + # into unparseable lines (e.g. '"name": "hicache",') and + # aborts the client container launch. Flattening newlines + # keeps the JSON valid (whitespace between tokens is + # insignificant) while collapsing it to a single line. + printf '%s=%s\n' "$_v" "${_val//$'\n'/}" + fi done echo "INFMAX_CONTAINER_WORKSPACE=/workspace" # Do NOT pin AGENTIC_OUTPUT_DIR: it must default to /workspace (the From 315760c1c562a6acac4bb501158267fbe21c1239 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 12:43:51 +0000 Subject: [PATCH 31/37] Re-serialize KV_OFFLOAD_BACKEND_METADATA as compact JSON instead of stripping newlines Naive newline-stripping could silently corrupt a value if a JSON string ever contained a literal embedded newline (merging tokens across the line break). Round-tripping through json.loads/json.dumps guarantees a correct compact single-line representation for client.env, and fails loudly with a clear error if the value isn't valid JSON. Co-authored-by: Cursor --- .../multi_node/amd_utils/server_sglang.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index c5451e98ee..9b53098de5 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -896,10 +896,19 @@ if [ "$NODE_RANK" -eq 0 ]; then # embedded newlines; KV_OFFLOAD_BACKEND_METADATA carries # pretty-printed multi-line JSON, which otherwise splits # into unparseable lines (e.g. '"name": "hicache",') and - # aborts the client container launch. Flattening newlines - # keeps the JSON valid (whitespace between tokens is - # insignificant) while collapsing it to a single line. - printf '%s=%s\n' "$_v" "${_val//$'\n'/}" + # aborts the client container launch. Re-serialize it to + # compact single-line JSON (round-tripping through + # json.loads/json.dumps) instead of naively stripping + # newlines, so this stays correct even if a value ever + # contained a literal newline inside a string. + if [[ "$_v" == "KV_OFFLOAD_BACKEND_METADATA" ]]; then + _val="$(python3 -c 'import json, sys +print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { + echo "KV_OFFLOAD_BACKEND_METADATA must contain valid JSON" >&2 + exit 1 + } + fi + printf '%s=%s\n' "$_v" "$_val" fi done echo "INFMAX_CONTAINER_WORKSPACE=/workspace" From ceee75480af4fc645607badbbeaf5150ea6c9efa Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 14 Jul 2026 12:51:16 +0000 Subject: [PATCH 32/37] Fix regression: skip JSON round-trip for empty KV_OFFLOAD_BACKEND_METADATA job.slurm always forwards KV_OFFLOAD_BACKEND_METADATA into the server container, even as "" when KV offloading is disabled (the common default case). The prior commit unconditionally ran it through json.loads/json.dumps, which raised JSONDecodeError on that empty string and hard-failed every agentic client-container launch without KV offloading. Only attempt the JSON round-trip when the value is non-empty and not "null", matching how optional_kv_offload_backend_metadata() already treats those as "no metadata" rather than invalid JSON. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/server_sglang.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 9b53098de5..72b1cfb972 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -900,8 +900,14 @@ if [ "$NODE_RANK" -eq 0 ]; then # compact single-line JSON (round-tripping through # json.loads/json.dumps) instead of naively stripping # newlines, so this stays correct even if a value ever - # contained a literal newline inside a string. - if [[ "$_v" == "KV_OFFLOAD_BACKEND_METADATA" ]]; then + # contained a literal newline inside a string. Empty/ + # "none"/"null" is the normal case when KV offloading is + # disabled (job.slurm always sets this var, even to ""), + # and must pass through untouched -- matching how + # optional_kv_offload_backend_metadata() in + # process_agentic_result.py treats those as "no metadata" + # rather than invalid JSON. + if [[ "$_v" == "KV_OFFLOAD_BACKEND_METADATA" && -n "$_val" && "$_val" != "null" ]]; then _val="$(python3 -c 'import json, sys print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { echo "KV_OFFLOAD_BACKEND_METADATA must contain valid JSON" >&2 From 5fdef46ffdaf181a6b3657ba6e833150e8655010 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 14 Jul 2026 09:33:25 -0500 Subject: [PATCH 33/37] fix(changelog): append PR entry after main --- perf-changelog.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index eb6cfca865..0dc49b63ed 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4751,14 +4751,6 @@ - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 -- config-keys: - - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache - description: - - "Add DeepSeek-V4-Pro FP4 MI355X SGLang-disagg agentic-coding benchmark with HiCache DRAM KV offloading." - - "Add the multi-node agentic runtime plumbing: node-0 sibling benchmark-client container, per-worker metrics scrape, per-concurrency cache flushing, trace replay client, and one allocation per concurrency." - - "Image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2170 - - config-keys: - dsv4-fp4-mi355x-sglang description: @@ -4773,3 +4765,11 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache + description: + - "Add DeepSeek-V4-Pro FP4 MI355X SGLang-disagg agentic-coding benchmark with HiCache DRAM KV offloading." + - "Add the multi-node agentic runtime plumbing: node-0 sibling benchmark-client container, per-worker metrics scrape, per-concurrency cache flushing, trace replay client, and one allocation per concurrency." + - "Image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2170 From d8cbb70c46e98c3db3cfb228133ae1cbebc61471 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 14 Jul 2026 09:31:23 -0500 Subject: [PATCH 34/37] fix(changelog): omit null optional metadata --- utils/process_changelog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/process_changelog.py b/utils/process_changelog.py index 311c8c9e22..c293dc2f80 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -270,7 +270,7 @@ def main(): # Validate final results structure validated = ChangelogMatrixEntry.model_validate(final_results) - print(validated.model_dump_json(by_alias=True)) + print(validated.model_dump_json(by_alias=True, exclude_none=True)) if __name__ == "__main__": From e6251738a17d3a323f4b8694ba979976afb9a5ed Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 14 Jul 2026 16:00:33 -0500 Subject: [PATCH 35/37] fix(agentx): set DeepSeek reasoning effort high --- benchmarks/multi_node/amd_utils/env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index 2a3005c644..baa917b449 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -325,7 +325,7 @@ else # DSv4 model kernel routing (mirrors the single-node / manual PD recipe) export SGLANG_DEFAULT_THINKING=1 - export SGLANG_DSV4_REASONING_EFFORT=max + export SGLANG_DSV4_REASONING_EFFORT=high export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false export SGLANG_USE_AITER=1 export SGLANG_USE_ROCM700A=0 From b6563376a1f63e359065075005813cd61965e19d Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Wed, 15 Jul 2026 05:51:59 +0000 Subject: [PATCH 36/37] Remove stale patches/README.md documenting an already-retired mori_conn.py overlay mori_conn.py and the job.slurm auto-apply block it documented were already deleted upstream (retire MoRI conn.py PD-disagg overlay, aca9ceefb) once the real sglang fix landed in published images. A later unrelated commit (183a987ad) accidentally recreated this README from scratch while only intending to drop a bullet about a different, already-removed patch (decode_tp_queue_agree.patch), leaving stale docs pointing at a file and job.slurm block that no longer exist. Co-authored-by: Cursor --- .../multi_node/amd_utils/patches/README.md | 94 ------------------- 1 file changed, 94 deletions(-) delete mode 100644 benchmarks/multi_node/amd_utils/patches/README.md diff --git a/benchmarks/multi_node/amd_utils/patches/README.md b/benchmarks/multi_node/amd_utils/patches/README.md deleted file mode 100644 index 765d571b27..0000000000 --- a/benchmarks/multi_node/amd_utils/patches/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# In-tree patches for the MoRI / MoRIIO PD-disagg path - -This directory carries small overlays that fix up the engine source inside -the docker container at runtime. They are needed because some published -images ship known bugs in the (MoRI / MoRIIO) disaggregation backend that -block our benchmark + accuracy configs — so we can keep reusing the -**stock image** instead of rebuilding a patched one. - -- `mori_conn.py` — single-file overlay (bind-mounted) for the **sglang** - MoRI backend. - -> Note: the vLLM MoRIIO `minimax-m3` overlay (`moriio/`) was retired once the -> upstream fixes (vLLM #46039 / #46290 / #46332) shipped in the ROCm nightly -> image; `minimaxm3-fp8-mi355x-vllm-disagg` now runs the stock nightly directly. - -The `mori_conn.py` overlay is wired through the `EXTRA_DOCKER_MOUNTS` env -var that `job.slurm` consumes (an opt-in `${EXTRA_DOCKER_MOUNTS:-}` after -the existing `-v` block). The local-test driver scripts under -`scripts/sglang_disagg/` pre-set this env var to the path of the relevant -overlay; CI runners that need the patch can do the same. - -## `mori_conn.py` - -Overlays -`/sgl-workspace/sglang/python/sglang/srt/disaggregation/mori/conn.py`. - -Source: forked from the file shipped in -`lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523` -(sglang [v0.5.12.post1](https://github.com/sgl-project/sglang/tree/v0.5.12.post1)). -Four logical edits, all confined to `MoriKVReceiver.send_state`, -`MoriKVReceiver._register_kv_args`, and -`MoriKVReceiver._send_swa_dsa_state`: - -1. **Sender flatten** — handle the framework's nested - `state_item_lens: List[List[int]]` instead of crashing in the - naked `struct.pack("I", item_len)` (the legacy `List[int]` - assumption). Idempotent for legacy flat callers. -2. **`state_type` legacy fallback** — when the legacy singular - `kv_args.state_type` is `'none'` but `state_mem_descs` is non-empty, - read `kv_args.state_types[0]` (the modern plural API that Mooncake - and NIXL already use). Routes `MAMBA → _send_mamba_state` and - `DSA/SWA → _send_swa_dsa_state` correctly. -3. **Consumer normalization** — flatten `state_item_lens` and - `state_dim_per_tensor` to flat `List[int]` once at the entry of - `send_state`, so the existing per-tensor index arithmetic - (`state_item_lens[i]`) and length checks - (`len(state_item_lens) == len(state_mem_descs)`) keep working. -4. **DSA index rank+length normalization** — inside - `_send_swa_dsa_state`, before the `group_concurrent_contiguous` - call, ravel both `src_state_indices` and `dst_state_indices` to 1-D - and re-truncate to common length. Upstream's existing truncation - only slices the outer axis, leaving 2-D `(1, N)` arrays unchanged - and triggering an `np.diff` broadcasting error - (`shapes (1,12) (0,)`) for GLM-5 (single-DSA-component) prefill - traffic. See - `scripts/sglang_disagg/docs_glm5/01-bug-analysis.md` for the full - write-up. - -Verified passing GSM8K = 0.978 ± 0.004 on Qwen3.5-397B-A17B-FP8 1P+1D -TP=8 dp-attn=false (matches and slightly exceeds upstream -[PR #22665](https://github.com/sgl-project/sglang/pull/22665)'s -reported 0.970 GSM8K on the bf16 baseline). GLM-5 (DSA) verification -in progress under -`scripts/sglang_disagg/docs_glm5/02-fix-and-verification.md`. - -This is a stop-gap. The proper upstream fix is to migrate MoRI to the -plural `state_types: List[StateType]` API (full design + diff in -`scripts/sglang_disagg/docs/03-upstream-pr-proposal.md`). - -## How to enable - -```bash -export EXTRA_DOCKER_MOUNTS="-v $DI_REPO_DIR/benchmarks/multi_node/amd_utils/patches/mori_conn.py:/sgl-workspace/sglang/python/sglang/srt/disaggregation/mori/conn.py:ro" -``` - -`$DI_REPO_DIR` is the InferenceX checkout root that `job.slurm` -already mounts into the container at `/workspace`. - -When this env var is unset (CI default for runs that don't need the -patch), `${EXTRA_DOCKER_MOUNTS:-}` expands to the empty string and -container behavior is byte-identical to the unpatched path. - -## When to use which patch - -| Image / version | Need `mori_conn.py` overlay? | -|---|---| -| `lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523` | yes (Qwen3.5-MoE-FP8, GLM-5, any hybrid model on this image) | -| `lmsysorg/sglang-rocm:v0.5.10.post1-rocm720-mi35x-*` (used by `dsr1-fp4-*-disagg`) | not validated; same code path likely affected — try with the overlay if you hit the same `struct.error` | -| `rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-*` (used by `dsr1-fp8-*-disagg`, `glm5-*-disagg`) | predates [PR #22665](https://github.com/sgl-project/sglang/pull/22665); different code paths; **do not** apply this overlay | - -When upstream merges the proper fix (see -`scripts/sglang_disagg/docs/03-upstream-pr-proposal.md`) and that -fix lands in a published image, retire this overlay and the -`EXTRA_DOCKER_MOUNTS` knob can stay (still useful for future patches). From 9a54424376fc77d9aaee588ab8eff2519866e4bb Mon Sep 17 00:00:00 2001 From: Duyi-Wang Date: Thu, 16 Jul 2026 04:52:59 +0000 Subject: [PATCH 37/37] feat(agentx): enforce agentic DRAM offload budget for multinode disagg Multinode agentic entries hardcoded total-cpu-dram-gb=0, so the DRAM KV-offload pool was never bounded at runtime (HiCache fell back to a relative --hicache-ratio). Compute a real per-node budget from the prefill worker (the only KV offloader today) and wire it through to a --hicache-size cap. - generate_sweep_configs: fold multinode into agentic_dram_offload_gb; budget the prefill worker's per-node GPU footprint via worker_gpus_per_node, which rejects node-misaligned topologies instead of silently truncating. - validation: add total-cpu-dram-gb to MultiNodeAgenticMatrixEntry. - run-sweep: forward total-cpu-dram-gb to the multi-node agentic dispatch. - server_sglang.sh: size prefill HiCache via absolute --hicache-size (budget / ranks-per-node / host-pools), falling back to --hicache-ratio when no budget is provided; drop the dead HICACHE_TOTAL_CPU_DRAM_GB default. --- .github/workflows/run-sweep.yml | 1 + .../multi_node/amd_utils/server_sglang.sh | 23 ++++- utils/matrix_logic/generate_sweep_configs.py | 89 +++++++++++++---- .../test_generate_sweep_configs.py | 95 ++++++++++++++++++- utils/matrix_logic/validation.py | 1 + 5 files changed, 183 insertions(+), 26 deletions(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index fdb34e1e4d..819bbfb4ca 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -581,6 +581,7 @@ jobs: kv-offloading: ${{ matrix.config.kv-offloading }} kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} + total-cpu-dram-gb: ${{ matrix.config['total-cpu-dram-gb'] }} duration: ${{ matrix.config.duration }} run-eval: false scenario-type: agentic-coding diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 72b1cfb972..e26a0ab3e1 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -459,7 +459,6 @@ echo "SYNC_BARRIER_TIMEOUT=${SYNC_BARRIER_TIMEOUT}s (model=${MODEL_NAME:-unset}) KV_OFFLOADING="${KV_OFFLOADING:-none}" KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-}" if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then - HICACHE_TOTAL_CPU_DRAM_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-2000}" HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-wait_complete}" @@ -509,11 +508,29 @@ if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then echo "--hicache-storage-backend mooncake --hicache-storage-backend-extra-config '${extra}' --enable-metrics --enable-cache-report" } - # HiCache capacity via --hicache-ratio (scales with GPU KV pool). + # HiCache capacity. Prefer an absolute per-rank pool derived from the + # per-node DRAM budget computed by the sweep generator (enforcement); fall + # back to --hicache-ratio (relative to the GPU KV pool) when no budget is + # provided, keeping configs that predate the budget unchanged. HICACHE_RATIO="${HICACHE_RATIO:-5}" + HICACHE_SIZING_FLAGS="--hicache-ratio ${HICACHE_RATIO}" + if [[ -n "${TOTAL_CPU_DRAM_GB:-}" && "${TOTAL_CPU_DRAM_GB}" -gt 0 ]]; then + # TOTAL_CPU_DRAM_GB is the prefill worker's per-node budget (only prefill + # offloads KV to CPU DRAM today); --hicache-size is per rank per host + # pool. A prefill server may span nodes (PREFILL_TP_SIZE is its total + # ranks), so divide by the ranks that land on one node. + prefill_ranks_per_node=$(( PREFILL_TP_SIZE < GPUS_PER_NODE ? PREFILL_TP_SIZE : GPUS_PER_NODE )) + prefill_hicache_size_gb=$(( TOTAL_CPU_DRAM_GB / prefill_ranks_per_node / HICACHE_HOST_POOL_COUNT )) + if (( prefill_hicache_size_gb < 1 )); then + echo "Error: TOTAL_CPU_DRAM_GB=${TOTAL_CPU_DRAM_GB} / ranks_per_node=${prefill_ranks_per_node} / host_pools=${HICACHE_HOST_POOL_COUNT} rounds below 1 GB" >&2 + exit 1 + fi + HICACHE_SIZING_FLAGS="--hicache-size ${prefill_hicache_size_gb}" + echo "[HiCache] prefill CPU pool capped at ${prefill_hicache_size_gb} GB/rank (budget ${TOTAL_CPU_DRAM_GB} GB / ranks_per_node ${prefill_ranks_per_node} / host_pools ${HICACHE_HOST_POOL_COUNT})" + fi build_hicache_flags() { - echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache --hicache-ratio ${HICACHE_RATIO} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" + echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache ${HICACHE_SIZING_FLAGS} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" } # HiCache requires RadixAttention; strip any --disable-radix-cache. diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index d260549789..0bc8b56607 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -100,11 +100,57 @@ def with_worker_parallelism_defaults(worker: dict) -> dict: } +def worker_gpus_per_node(worker: dict, gpus_per_node: int) -> int: + """Return GPUs a single worker replica occupies on one node. + + The DRAM offload budget is sized per server process (per replica), matching + the single-node path: a replica claims the node-DRAM fraction that mirrors + its GPU footprint on the node. Topologies that do not tile a node cleanly + are rejected rather than silently truncated, keeping parity with the + single-node "must fit the node" rule: + + * A replica larger than one node (tp*pp*pcp > gpus-per-node) must fill whole + nodes, i.e. be an exact multiple of gpus-per-node; each of its nodes is + then fully occupied (fraction 1). + * A replica within one node must divide it evenly so co-located replicas of + the same role tile the node without overlap. + """ + gpus_per_replica = ( + worker[Fields.TP.value] + * worker.get(Fields.PP.value, 1) + * worker.get(Fields.PCP_SIZE.value, 1) + ) + if gpus_per_replica > gpus_per_node: + if gpus_per_replica % gpus_per_node != 0: + raise ValueError( + f"worker {Fields.TP.value}*{Fields.PP.value}*{Fields.PCP_SIZE.value}" + f"={gpus_per_replica} spans multiple nodes but is not a multiple " + f"of {Fields.GPUS_PER_NODE.value}={gpus_per_node}" + ) + return gpus_per_node + if gpus_per_node % gpus_per_replica != 0: + raise ValueError( + f"worker {Fields.TP.value}*{Fields.PP.value}*{Fields.PCP_SIZE.value}" + f"={gpus_per_replica} does not divide " + f"{Fields.GPUS_PER_NODE.value}={gpus_per_node} evenly" + ) + return gpus_per_replica + + def agentic_dram_offload_gb( agentic_config: dict, benchmark: dict, runner: str, runner_data: dict ) -> int: - """Return the aggregate DRAM offload budget for a single-node entry.""" - kv_offloading = benchmark[Fields.KV_OFFLOADING.value] + """Return the DRAM offload budget (GB) for one agentic server node. + + The budget scales the node's (MAX-capped) available CPU DRAM by the + utilization and by the fraction of the node's GPUs in use: + + * Single-node entries use the TP/PP/PCP topology, which must fit one node. + * Disaggregated multinode entries use the prefill worker's per-node GPU + footprint, since only prefill offloads KV to CPU DRAM today (decode can be + budgeted separately if it ever gains its own pool). + """ + kv_offloading = benchmark.get(Fields.KV_OFFLOADING.value, "none") if kv_offloading != "dram": return 0 @@ -114,15 +160,20 @@ def agentic_dram_offload_gb( ) utilization = Decimal(str(agentic_config[Fields.DRAM_UTILIZATION.value])) gpus_per_node = runner_gpus_per_node(runner, runner_data) - gpu_count = effective_gpu_count(benchmark) - if gpu_count > gpus_per_node: - raise ValueError( - f"tp={benchmark[Fields.TP.value]} with " - f"{Fields.PP.value}={benchmark.get(Fields.PP.value, 1)} and " - f"{Fields.PCP_SIZE.value}={benchmark.get(Fields.PCP_SIZE.value, 1)} " - f"requires {gpu_count} GPUs and exceeds " - f"{Fields.GPUS_PER_NODE.value}={gpus_per_node} for runner '{runner}'" - ) + + if Fields.PREFILL.value in benchmark: + gpu_count = worker_gpus_per_node( + benchmark[Fields.PREFILL.value], gpus_per_node) + else: + gpu_count = effective_gpu_count(benchmark) + if gpu_count > gpus_per_node: + raise ValueError( + f"tp={benchmark[Fields.TP.value]} with " + f"{Fields.PP.value}={benchmark.get(Fields.PP.value, 1)} and " + f"{Fields.PCP_SIZE.value}={benchmark.get(Fields.PCP_SIZE.value, 1)} " + f"requires {gpu_count} GPUs and exceeds " + f"{Fields.GPUS_PER_NODE.value}={gpus_per_node} for runner '{runner}'" + ) proportional_bytes = ( Decimal(available_mib) * BYTES_PER_MIB * utilization * gpu_count / gpus_per_node @@ -649,11 +700,8 @@ def generate_full_sweep(args, all_config_data, runner_data): spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "none") kv_offloading = bmk[Fields.KV_OFFLOADING.value] kv_offload_backend = bmk.get(Fields.KV_OFFLOAD_BACKEND.value) - total_cpu_dram_gb = ( - 0 - if is_multinode - else agentic_dram_offload_gb(agentic_config, bmk, runner, runner_data) - ) + total_cpu_dram_gb = agentic_dram_offload_gb( + agentic_config, bmk, runner, runner_data) # Get concurrency values conc_list = bmk.get(Fields.CONC_LIST.value) @@ -704,6 +752,7 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.DECODE.value: decode, Fields.CONC.value: conc_batch, Fields.KV_OFFLOADING.value: kv_offloading, + Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb, Fields.DURATION.value: duration, Fields.EXP_NAME.value: ( f"{model_code}_p{prefill[Fields.NUM_WORKER.value]}x{prefill[Fields.TP.value]}" @@ -955,11 +1004,8 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "none") kv_offloading = bmk[Fields.KV_OFFLOADING.value] kv_offload_backend = bmk.get(Fields.KV_OFFLOAD_BACKEND.value) - total_cpu_dram_gb = ( - 0 - if is_multinode - else agentic_dram_offload_gb(agentic_config, bmk, runner, runner_data) - ) + total_cpu_dram_gb = agentic_dram_offload_gb( + agentic_config, bmk, runner, runner_data) conc_list = bmk.get(Fields.CONC_LIST.value) if conc_list: @@ -1004,6 +1050,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.DECODE.value: decode, Fields.CONC.value: conc_batch, Fields.KV_OFFLOADING.value: kv_offloading, + Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb, Fields.DURATION.value: duration, Fields.EXP_NAME.value: ( f"{model_code}_p{prefill[Fields.NUM_WORKER.value]}x{prefill[Fields.TP.value]}" diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index a4baa81ac9..f8ee4f58ad 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -128,6 +128,7 @@ def sample_runner_config(): "cluster:b200-dgxc": {"available-cpu-dram-mib": 3774874, "gpus-per-node": 8}, "cluster:b300-nv": {"available-cpu-dram-mib": 2964436, "gpus-per-node": 8}, "cluster:mi300x-amds": {"available-cpu-dram-mib": 2321924, "gpus-per-node": 8}, + "cluster:mi355x-amds": {"available-cpu-dram-mib": 3095781, "gpus-per-node": 8}, "cluster:gb200-nv": {"available-cpu-dram-mib": 860160, "gpus-per-node": 4}, }, } @@ -2187,7 +2188,7 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): assert result[0]["decode"]["dcp-size"] == 2 assert result[0]["decode"]["pcp-size"] == 1 - def test_multinode_agentic_preserves_kv_offload_fields(self): + def test_multinode_agentic_preserves_kv_offload_fields(self, sample_runner_config): config = { "dsv4-agentic-hicache": { "image": "sglang-rocm", @@ -2201,6 +2202,7 @@ def test_multinode_agentic_preserves_kv_offload_fields(self): "kv-p2p-transfer": "mori", "scenarios": { "agentic-coding": [{ + "dram-utilization": 0.80, "search-space": [{ "conc-list": [16], "kv-offloading": "dram", @@ -2220,12 +2222,101 @@ def test_multinode_agentic_preserves_kv_offload_fields(self): runner_node_filter=None, ) - result = generate_test_config_sweep(args, config) + result = generate_test_config_sweep(args, config, sample_runner_config) assert len(result) == 1 assert result[0]["kv-offloading"] == "dram" assert result[0]["kv-offload-backend"] == {"name": "hicache"} assert result[0]["exp-name"] == "dsv4_p1x8_d1x8_conc16_kvdram-hicache" + # Budget tracks the prefill worker (the only KV-offloader): tp=8 fills + # the 8-GPU node -> full utilization share of the (MAX-capped) available + # DRAM: 2861022 MiB * 0.80. + assert result[0]["total-cpu-dram-gb"] == 2399 + + def test_multinode_agentic_budget_ignores_decode_topology( + self, sample_runner_config + ): + """Only prefill offloads today, so decode's topology does not shrink it.""" + config = { + "dsv4-agentic-hicache-asym": { + "image": "sglang-rocm", + "model": "deepseek-ai/DeepSeek-V4-Pro", + "model-prefix": "dsv4", + "precision": "fp4", + "framework": "sglang-disagg", + "runner": "cluster:mi355x-amds", + "multinode": True, + "disagg": True, + "kv-p2p-transfer": "mori", + "scenarios": { + "agentic-coding": [{ + "dram-utilization": 0.80, + "search-space": [{ + "conc-list": [16], + "kv-offloading": "dram", + "kv-offload-backend": {"name": "hicache"}, + # prefill fills the node (8 GPUs); decode uses half. + "prefill": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + "decode": {"num-worker": 1, "tp": 4, "ep": 1, "dp-attn": False}, + }], + }], + }, + }, + } + args = argparse.Namespace( + config_keys=["dsv4-agentic-hicache-asym"], + seq_lens=None, + conc=None, + scenario_type=["agentic-coding"], + runner_node_filter=None, + ) + + result = generate_test_config_sweep(args, config, sample_runner_config) + + assert len(result) == 1 + # prefill 8/8 -> full budget, regardless of decode tp=4. + assert result[0]["total-cpu-dram-gb"] == 2399 + + def test_multinode_agentic_rejects_node_misaligned_prefill( + self, sample_runner_config + ): + """A prefill worker whose GPU footprint does not tile the node is rejected.""" + config = { + "dsv4-agentic-hicache-misaligned": { + "image": "sglang-rocm", + "model": "deepseek-ai/DeepSeek-V4-Pro", + "model-prefix": "dsv4", + "precision": "fp4", + "framework": "sglang-disagg", + "runner": "cluster:mi355x-amds", + "multinode": True, + "disagg": True, + "kv-p2p-transfer": "mori", + "scenarios": { + "agentic-coding": [{ + "dram-utilization": 0.80, + "search-space": [{ + "conc-list": [16], + "kv-offloading": "dram", + "kv-offload-backend": {"name": "hicache"}, + # tp=6 does not divide an 8-GPU node evenly. + "prefill": {"num-worker": 1, "tp": 6, "ep": 1, "dp-attn": False}, + "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, + }], + }], + }, + }, + } + args = argparse.Namespace( + config_keys=["dsv4-agentic-hicache-misaligned"], + seq_lens=None, + conc=None, + scenario_type=["agentic-coding"], + runner_node_filter=None, + ) + + with pytest.raises(ValueError, match="does not divide"): + generate_test_config_sweep(args, config, sample_runner_config) # ============================================================================= diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 6b7c671289..ab9b8f4270 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -325,6 +325,7 @@ class MultiNodeAgenticMatrixEntry(BaseModel): kv_p2p_transfer: Optional[str] = Field( default=None, alias=Fields.KV_P2P_TRANSFER.value, min_length=1 ) + total_cpu_dram_gb: int = Field(alias=Fields.TOTAL_CPU_DRAM_GB.value, ge=0) duration: int = Field(alias=Fields.DURATION.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) disagg: bool