diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index b2219144f1..f98da238cf 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -240,7 +240,7 @@ jobs: ${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }} ${{ inputs.kv-offloading != '' && inputs.kv-offloading != 'none' && format('{0} KV offload', inputs.kv-offloading) || '' }} ${{ inputs.kv-offload-backend != '' && inputs.kv-offload-backend != 'none' && inputs.kv-offload-backend != 'default' && inputs.kv-offload-backend || '' }} - c${{ inputs.conc != '' && inputs.conc || join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }} + c${{ join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }} steps: - name: Slurm cleanup (pre-run) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index b983982af8..fdb34e1e4d 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -556,7 +556,7 @@ jobs: precision: ${{ matrix.config.precision }} router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} - conc-list: '[${{ matrix.config.conc }}]' + conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} prefill-hardware: ${{ matrix.config.prefill.hardware }} @@ -577,7 +577,7 @@ jobs: decode-ep: ${{ matrix.config.decode.ep }} decode-dp-attn: ${{ matrix.config.decode.dp-attn }} decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - conc: ${{ matrix.config.conc }} + conc: ${{ matrix.config.conc[0] }} kv-offloading: ${{ matrix.config.kv-offloading }} kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml new file mode 100644 index 0000000000..3ff9e31924 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml @@ -0,0 +1,154 @@ +name: "agg-gb300-tp4-mtp-kvoffload" + +# Agentic-coding SGLang aggregated recipe for DeepSeek-V4-Pro on GB300 +# (single aggregated worker, TP4, MTP + hierarchical-cache KV offload). +# +# Uses the flat single-variant schema the agentic CI flow expects. Concurrency is NOT +# a recipe field here: the GHA matrix fans out one job per concurrency from +# the master-config conc-list, exporting CONC into agentic_srt.sh. Modeled on +# the flat vllm/deepseek-v4/agentic recipes (agentic infra + benchmark wiring) +# and the flat sglang/deepseek-v4/8k1k recipes (sglang backend schema). + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + # Dedicated etcd/nats node — matches the vllm agentic recipes; under the + # dynamo router + multiple frontends the infra services need their own node. + etcd_nats_dedicated_node: true + # cc-traces prompts (~125K tokens) serialize to ~2.8MB; the 1 MiB NATS + # default drops them. 32 MiB matches the agentic vllm recipes (~10x headroom + # over the largest observed payload); schema recommends 24+ for long ISL. + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + # The sglang image is PEP 668 externally-managed; the runtime dynamo + # install (dynamo_wheels.py / source build) runs pip and fails with + # "externally-managed-environment" without this. Lets pip install into + # the system env. Applies to both dynamo.hash (source) and dynamo.wheel. + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + aggregated_environment: + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. + PIP_BREAK_SYSTEM_PACKAGES: "1" + # from submission for B300 + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + # for kvcache offload + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + stream-interval: 50 + watchdog-timeout: 1000000 + mem-fraction-static: 0.90 + chunked-prefill-size: 8192 # can only support 11000 for TP + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 128 + cuda-graph-max-bs: 128 + + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 4 + ep-size: 1 + # kv-cache offload + enable-hierarchical-cache: true + hicache-ratio: 6 + hicache-write-policy: write_through + hicache-io-backend: kernel + # mtp section + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user into the container as a non-root + # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to + # uid 0 inside the container. srt-slurm renders empty-string values as + # flag-only srun args. + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # Aggregated: one worker serves both prefill and decode, so GPU accounting + # is the single-worker form num_gpus = TP (not the disagg prefill+decode + # sum). The multinode post-processing path assumes disagg and would divide + # throughput by prefill_gpus + decode_gpus; force the single-node code path + # in process_agentic_result.py instead. TP must match sglang_config tp-size. + IS_MULTINODE: "false" + TP: "4" + # Enable dynamo conv-aware routing (per-session affinity so multi-turn + # KV cache is reused across turns). Previously opted out because the + # frontend 400'd on aiperf's nvext.session_control actions; re-enabled + # to test with the current build. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + # Container-side path of the aiperf mmap dataset cache; host-side mount + # wired via launch_gb300-*.sh srtslurm.yaml default_mounts. Without it, + # aiperf re-tokenizes + re-writes the dataset mmap on every run. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also via default_mounts) so the trace dataset + # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE. + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml new file mode 100644 index 0000000000..715b0c631e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml @@ -0,0 +1,212 @@ +name: "disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (12P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 1024. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 12 + decode_nodes: 4 + prefill_workers: 6 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + decode_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + sglang_config: + prefill: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 1024 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml new file mode 100644 index 0000000000..dc91c255ce --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml @@ -0,0 +1,211 @@ +name: "disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (2P x DEP8 / 1D x TP4, MTP + hierarchical-cache KV offload), tuned for concurrency 8. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + decode_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '2048' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + sglang_config: + prefill: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 32768 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 4 + dp-size: 1 + ep-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + moe-runner-backend: flashinfer_mxfp4 + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 128 + cuda-graph-max-bs: 128 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml new file mode 100644 index 0000000000..6df695b914 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -0,0 +1,212 @@ +name: "disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (2P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 256. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + decode_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + sglang_config: + prefill: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml new file mode 100644 index 0000000000..107a787ba9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -0,0 +1,212 @@ +name: "disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 +# (4P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 256. +# +# DEP8-prefill variant aligned with the measured Pareto point: prefill uses +# tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. +# Concurrency is exported into agentic_srt.sh +# from the master-config conc-list. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 4 + prefill_workers: 2 + decode_workers: 1 + + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_DSV4_MHC_PREWARM: '1' + + decode_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_DSV4_MHC_PREWARM: '1' + + sglang_config: + prefill: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml new file mode 100644 index 0000000000..27a4e42d22 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml @@ -0,0 +1,212 @@ +name: "disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (6P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 512. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 6 + decode_nodes: 4 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + decode_environment: + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_SIMULATE_ACC_LEN: '2.49' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + + sglang_config: + prefill: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 512 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e4e0a7a75f..33e9118b7a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -15215,6 +15215,135 @@ glm5-fp4-gb300-dynamo-trt-mtp: ep: 16 dp-attn: true +dsv4-fp4-gb300-dynamo-sglang-agentic-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.3.0.dev1" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [1, 4, 8, 16] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml" + # Aggregated worker: prefill's GPUs also serve decode, so decode has no + # separate allocation (num-worker: 0). Deployment is recipe-driven + # (agg_workers: 1); this only makes GPU accounting count 4, not 4+4. + decode: + num-worker: 0 + tp: 4 + ep: 1 + dp-attn: false + +dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.3.0.dev1" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [8] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [256] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [256] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [512] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [1024] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 6 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true qwen3.5-fp8-gb300-dynamo-sglang: image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6ad6a689ca..e8e9078c2e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4852,3 +4852,11 @@ - "run_benchmark_serving uses --dsv4 chat-template encoding, required for meaningful MTP acceptance rate; routes to dsv4_fp8_mi325x_mtp.sh via the launcher's spec-decoding=mtp suffix" - "256GB MI325X keeps the default eval concurrency (no KV cap needed)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2195 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Add GB300 DeepSeek-V4 dynamo-sglang agentic recipes (agg-tp4 + dep8 disagg pareto)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2157 + diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 1381bd5db7..d450cdb86d 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -20,6 +20,17 @@ export AIPERF_MMAP_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/ai-perf-cach export HF_HUB_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/hf-hub-cache" mkdir -p "$HF_HUB_CACHE_HOST_PATH" +# Persistent dynamo source-build cache. srtctl's hash-pinned dynamo install +# (_hash_cached_source_install) caches the built wheel + src tarball at +# /configs/dynamo-wheels/ with a .complete sentinel; on a warm cache the +# install is just `pip install` from the cache (no apt, no root). In CI /configs +# is the per-job srt-slurm checkout (cold every job → cold build needs apt + +# root, which the non-root server containers can't do), so persist and share +# the cache across jobs by bind-mounting this host dir at /configs/dynamo-wheels. +# Seed it once with a --container-remap-root build. 777 for multi-user runners. +export DYNAMO_WHEELS_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/dynamo-wheels" +mkdir -p "$DYNAMO_WHEELS_CACHE_HOST_PATH" + export MODEL_PATH=$MODEL if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then @@ -127,7 +138,18 @@ SRT_REPO_DIR="${GITHUB_WORKSPACE}/srt-slurm-${GITHUB_RUN_ID:-manual}-${GITHUB_RU SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" -if [[ "$IS_AGENTIC" == "1" ]]; then +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then + # DSv4 GB300 sglang agentic: NVIDIA/srt-slurm v1.0.10 has the nginx + # client_max_body_size fix (>1 MiB agentic warmup bodies), the + # session-affinity frontend, and the BenchmarkType.CUSTOM / extra_mount + # schema these recipes need. + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.10 + mkdir -p recipes/sglang/deepseek-v4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ + recipes/sglang/deepseek-v4/agentic +elif [[ "$IS_AGENTIC" == "1" ]]; then # Agentic multi-node uses cquil11/srt-slurm-nv@cam/no-preflight-flag, # a thin branch off NVIDIA/srt-slurm@127597c that adds one CLI flag # (`srtctl apply --no-preflight`) — needed because: @@ -282,6 +304,9 @@ srtctl_root: "${SRTCTL_ROOT}" default_mounts: "${AIPERF_MMAP_CACHE_HOST_PATH}": "/aiperf_mmap_cache" "${HF_HUB_CACHE_HOST_PATH}": "/hf_hub_cache" + # Warm dynamo source-build cache (nested over the auto /configs mount) so the + # hash-pinned install is a cache hit (pip-only, no apt/root) on every job. + "${DYNAMO_WHEELS_CACHE_HOST_PATH}": "/configs/dynamo-wheels" # Model path aliases model_paths: diff --git a/utils/agentic/aggregation/process_agentic_result.py b/utils/agentic/aggregation/process_agentic_result.py index 6b4acbc131..5d13fa0f1b 100644 --- a/utils/agentic/aggregation/process_agentic_result.py +++ b/utils/agentic/aggregation/process_agentic_result.py @@ -148,6 +148,16 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: num_prefill_gpu = prefill_num_workers * prefill_tp * prefill_pp * prefill_pcp_size num_decode_gpu = decode_num_workers * decode_tp * decode_pp * decode_pcp_size num_gpus = num_prefill_gpu + num_decode_gpu + # Aggregated configs set decode num-worker 0 (prefill+decode co-located on one + # worker), so there are no separate decode GPUs. Mirror process_result.py and drop + # the decode-side parallelism, so TP/EP and the per-GPU throughput denominator + # reflect the single aggregated worker instead of double-counting its GPUs. + if num_decode_gpu <= 0: + decode_tp = 0 + decode_ep = 0 + decode_pp = 1 + decode_dcp_size = 1 + decode_pcp_size = 1 tp = prefill_tp + decode_tp ep = max(prefill_ep, decode_ep) dp_attention = ( diff --git a/utils/process_changelog.py b/utils/process_changelog.py index 311c8c9e22..c293dc2f80 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -270,7 +270,7 @@ def main(): # Validate final results structure validated = ChangelogMatrixEntry.model_validate(final_results) - print(validated.model_dump_json(by_alias=True)) + print(validated.model_dump_json(by_alias=True, exclude_none=True)) if __name__ == "__main__":