diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh new file mode 100755 index 000000000..ce3c2ad88 --- /dev/null +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +set -eo pipefail +set -x + +# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on MI355X using SGLang. +# +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# KV_OFFLOADING=dram requires one of these. +# KV_OFFLOAD_BACKEND=hicache. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +# ROCR/HIP visibility under slurm cgroups. +if [ -n "$ROCR_VISIBLE_DEVICES" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "$MODEL_PATH" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +rocm-smi || true +amd-smi || true + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +CACHE_ARGS=() +if agentic_kv_offload_enabled; then + # HiCache config — https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4 + case "$KV_OFFLOAD_BACKEND" in + hicache) + HICACHE_RATIO=4 + HICACHE_WRITE_POLICY="write_through" + HICACHE_IO_BACKEND="direct" + HICACHE_MEM_LAYOUT="page_first_direct" + CACHE_ARGS=( + --enable-hierarchical-cache + --hicache-ratio "$HICACHE_RATIO" + --hicache-write-policy "$HICACHE_WRITE_POLICY" + --hicache-io-backend "$HICACHE_IO_BACKEND" + --hicache-mem-layout "$HICACHE_MEM_LAYOUT" + ) + echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected: hicache)" >&2 + exit 1 + ;; + esac +fi +# ---- Client config ---------------------------------------------------------- +export AIPERF_HTTP_TCP_USER_TIMEOUT=1000000 + +# ---- LLM server config ---------------------------------------------------------- +USE_SGLANG_ROUTER=false +SGLANG_BACKEND_PORT="$PORT" +ROUTER_LOG="$RESULT_DIR/router.log" +MEM_FRACTION_STATIC=0.90 +CHUNKED_PREFILL_SIZE=8192 +PARALLEL_ARGS=(--tensor-parallel-size "$TP") +if [ "$DP_ATTENTION" = "true" ]; then + USE_SGLANG_ROUTER=true + # DPA + MTP needs additional runtime headroom for speculative decode and + # communication buffers beyond SGLang's static KV pool. + MEM_FRACTION_STATIC=0.75 + export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true + SGLANG_BACKEND_PORT=$((PORT + 1)) + SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) + SGLANG_ROUTER_CMD=(python3 -m sglang_router.launch_router) + + export SGLANG_SHARED_EXPERT_TP1=1 + export SGLANG_DP_SHARED_EXPERT_LOCAL=1 + export SGLANG_DP_USE_GATHERV=1 + export SGLANG_DP_USE_REDUCE_SCATTER=1 + + # SGLang divides the configured chunk across DP schedulers. Use a 16K + # per-scheduler chunk so long agentic prefill tails drain within the + # standard 600-second warmup grace period. + CHUNKED_PREFILL_SIZE=$((16384 * TP)) + PARALLEL_ARGS+=( + --dp "$TP" + --enable-dp-attention + ) +fi + +if [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS+=(--ep-size "$EP_SIZE") +fi + +# SGLang treats max-running-requests as a global DPA limit and partitions it +# internally. CUDA graph capture is per scheduler, so only its batch size is +# divided across DP ranks. +MAX_RUNNING_REQUESTS=$CONC +CUDA_GRAPH_MAX_BS=$CONC +if [ "$DP_ATTENTION" = "true" ]; then + CUDA_GRAPH_MAX_BS=$(( (CUDA_GRAPH_MAX_BS + TP - 1) / TP )) +fi +[ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 + +# Simulated acceptance-length (AL) settings. +export SGLANG_DEFAULT_THINKING=1 +export SGLANG_DSV4_REASONING_EFFORT=high +export SGLANG_SIMULATE_ACC_LEN=2.49 +export SGLANG_SIMULATE_ACC_METHOD=match-expected +export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token + +export SGLANG_USE_ROCM700A=0 +export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton +export AITER_BF16_FP8_MOE_BOUND=0 + +# sglang kv cache, when hicache is disabled +# https://github.com/sgl-project/sglang/pull/30339 +#export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 +export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 + +METRICS_ARGS=(--enable-metrics) +SPEC_ARGS=( + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 +) + +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$SGLANG_BACKEND_PORT" + --trust-remote-code + "${PARALLEL_ARGS[@]}" + --attention-backend compressed + --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static "$MEM_FRACTION_STATIC" + --swa-full-tokens-ratio 0.10 + --page-size 256 + --kv-cache-dtype fp8_e4m3 + --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" + --disable-shared-experts-fusion + --tool-call-parser deepseekv4 + --reasoning-parser deepseek-v4 + --chat-template "$(dirname "$0")/../chat_templates/deepseek_v4_thinking.jinja" + --watchdog-timeout 1800 + "${METRICS_ARGS[@]}" + "${SPEC_ARGS[@]}" + "${CACHE_ARGS[@]}" +) + +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" + +{ + echo "=== SGLANG_* env vars at launch ===" + env | grep -E '^SGLANG_' | sort + echo "===================================" +} | tee "$SERVER_LOG" + +echo "Starting SGLang server for MI355X..." +"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "$USE_SGLANG_ROUTER" = "true" ]; then + echo "Starting SGLang router on port $PORT for $TP DP ranks..." + "${SGLANG_ROUTER_CMD[@]}" \ + --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ + --policy consistent_hashing \ + --request-id-headers x-correlation-id \ + --dp-aware \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ + --connect-timeout-secs 900 \ + --request-timeout-secs 14400 \ + --disable-health-check \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + echo "Router PID: $ROUTER_PID" + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +# ---- Run benchmark ---------------------------------------------------------- +build_replay_cmd "$RESULT_DIR" +REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" + +run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 3e3170dac..21257334f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2415,6 +2415,21 @@ dsv4-fp4-mi355x-sglang-agentic: - { tp: 8, kv-offloading: none, conc-list: [16, 32, 64] } - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [64, 128, 256] } +dsv4-fp4-mi355x-sglang-agentic-hicache-mtp: + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260714 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8], spec-decoding: mtp } + - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 48, 64], spec-decoding: mtp } + # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 # MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fe3610186..35a616d5c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4826,6 +4826,48 @@ - "Recipes ported from NVIDIA/srt-slurm tag v1.0.26 (recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/{ISL1K_OSL1K,ISL8K_OSL1K}/STP; 13 ISL1K + 10 ISL8K), checked into benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/... and overlaid onto the srt-slurm checkout at runtime. Prefill tp=4/ep=4 (dep4)." - "launch_gb200-nv.sh: added glm5-fp4 case to dynamo-trt branch (MODEL_PATH=/mnt/lustre01/slurm-shared/glm-model/GLM-5-NVFP4, SRT_SLURM_MODEL_PREFIX=nvidia/GLM-5-NVFP4), pinned srt-slurm clone to v1.0.26 with cp -rT overlay of the local recipes/trtllm/glm5 recipes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1803 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-hicache-mtp + description: + - "Add DSv4 FP4 MI355X SGLang agentic MTP recipe with HiCache support" + - "Image: lmsysorg/sglang-rocm:v0.5.15-rocm700-mi35x-20260713" + - "EAGLE speculative decoding: 3 steps, topk 1, 4 draft tokens" + - "Sweep TP8 +/- DPA +/- HiCache with MTP across conc 1-96" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2200 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-hicache-mtp + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260714" + - "Fix HSA out-of-resources crashes during DPA agentic MTP warmup" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2200 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-hicache-mtp + description: + - "Scale SGLang request and CUDA-graph limits per DP-attention engine while preserving aggregate AgentX fan-out headroom" + - "Prevent per-rank HSA out-of-resources crashes during accelerated warmup" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2200 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-hicache-mtp + description: + - "Keep max-running-requests global for SGLang DP attention and scale only per-scheduler CUDA graph capture" + - "Reserve DPA MTP runtime headroom with mem-fraction-static 0.80" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2200 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-hicache-mtp + description: + - "Increase DPA MTP runtime headroom to mem-fraction-static 0.75 after c64 exhausted VRAM at 0.80 during regular warmup" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2200 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-hicache-mtp + description: + - "Increase DPA per-scheduler chunked prefill from 8K to 16K so long agentic prefill tails drain within the standard 600-second warmup grace period" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2200 - config-keys: - glm5-fp8-gb200-dynamo-sglang-mtp @@ -4859,6 +4901,13 @@ - "Add GB300 DeepSeek-V4 dynamo-sglang agentic recipes (agg-tp4 + dep8 disagg pareto)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2157 +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-hicache-mtp + description: + - "Disable the SGLang prefill delayer so DP-attention MTP c64 warmup drains within the standard 600-second grace period" + - "Restore DP-attention concurrency 64 to the full sweep" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2200 + - config-keys: - qwen3.5-fp4-mi355x-sglang - qwen3.5-fp4-mi355x-sglang-mtp diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 8cb92b7a1..d7007aaf0 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -208,11 +208,7 @@ else export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" set -x - # Exclude known-bad mi355x compute nodes (KLAUD_DEBUG §5.1 / §5.2): - # mia1-p01-g09: pyxis broken (persistently fails to create container filesystem) - # mia1-p01-g11: docker.sock permissions denied (cluster-cleanup step fails) - # Both have been root-caused via #1431/#1432/#1440/#1441/#1443 sweep failures. - salloc --partition=$PARTITION --exclude=mia1-p01-g09,mia1-p01-g11 --gres=gpu:$GPU_COUNT --exclusive --cpus-per-task=128 --time=500 --no-shell --job-name="$RUNNER_NAME" + salloc --partition=$PARTITION --gres=gpu:$GPU_COUNT --exclusive --cpus-per-task=128 --time=500 --no-shell --job-name="$RUNNER_NAME" JOB_ID=$(squeue --name="$RUNNER_NAME" -h -o %A | head -n1) srun --jobid=$JOB_ID bash -c "docker stop \$(docker ps -a -q)"