diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh new file mode 100755 index 000000000..cdcc3742a --- /dev/null +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh @@ -0,0 +1,385 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for Kimi-K2.7 FP4 on MI355X using vLLM. +# KV_OFFLOADING=none -> GPU KV only +# KV_OFFLOADING=dram KV_OFFLOAD_BACKEND=lmcache -> LMCache MP server + connector +# KV_OFFLOADING=dram KV_OFFLOAD_BACKEND=mooncake -> Mooncake embedded store + connector +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE + + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# ROCR/HIP visibility for vLLM 0.14+ +if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +rocm-smi || true +amd-smi || true + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# Install amd-quark for MXFP4 (manual install due to ROCm vLLM bug) +pip install amd-quark + +# Disable AITER RMSNorm for TP < 8 due to accuracy issues +if [ "${TP}" -lt 8 ]; then + export VLLM_ROCM_USE_AITER_RMSNORM=0 +fi +# Workaround for MEC FW <177 RCCL memory reclaim issue +version=$(rocm-smi --showfw 2>/dev/null | grep MEC | head -n 1 | awk '{print $NF}') +if [[ "$version" == "" || ${version:-0} -lt 177 ]]; then + export HSA_NO_SCRATCH_RECLAIM=1 +fi + +export VLLM_ROCM_USE_AITER=1 +# amd/Kimi-K2.7-Code-MXFP4 has 64 KV heads; the ROCm AITER-MLA kernel only +# supports 16 or 128 heads, so it MUST be disabled for this model. With AITER-MLA +# off, ROCm serves MLA via TRITON_MLA, which does NOT support block_size=1 -- so +# this recipe passes no --block-size and lets vLLM pick a TRITON_MLA-valid +# default (matches AMD's reference serve command for this model). The old +# --block-size=1 only existed to force the (now-disabled) AITER-MLA path. +export VLLM_ROCM_USE_AITER_MLA=0 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +# Avoid intermittent symm_mem all-reduce rendezvous hang at engine init on +# MI35x nodes (see KIMIK27_CONC64_LMCACHE_RUNBOOK error #4). +export VLLM_ALLREDUCE_USE_SYMM_MEM="${VLLM_ALLREDUCE_USE_SYMM_MEM:-0}" + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" +MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +mkdir -p "$RESULT_DIR" + +OFFLOAD_ARGS=() +PREFIX_CACHE_ARGS=() + +# ---- Offload service cleanup ------------------------------------------------ +LMCACHE_PID="" +MOONCAKE_MASTER_PID="" +ROUTER_PID="" + +cleanup_offload_services() { + if [[ -n "$ROUTER_PID" ]] && kill -0 "$ROUTER_PID" 2>/dev/null; then + kill "$ROUTER_PID" 2>/dev/null || true + wait "$ROUTER_PID" 2>/dev/null || true + fi + if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then + kill "$LMCACHE_PID" 2>/dev/null || true + wait "$LMCACHE_PID" 2>/dev/null || true + fi + if [[ -n "$MOONCAKE_MASTER_PID" ]] && kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then + kill "$MOONCAKE_MASTER_PID" 2>/dev/null || true + wait "$MOONCAKE_MASTER_PID" 2>/dev/null || true + fi +} +trap cleanup_offload_services EXIT + +wait_for_lmcache_ready() { + { set +x; } 2>/dev/null + local attempts="${LMCACHE_READY_ATTEMPTS:-120}" + local tail_pid="" + + while [ ! -f "$LMCACHE_LOG" ]; do + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before creating log file. Exiting." >&2 + exit 1 + fi + sleep 1 + done + + tail -f -n +1 "$LMCACHE_LOG" & + tail_pid=$! + + for ((i = 1; i <= attempts; i++)); do + if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + return 0 + fi + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before becoming healthy. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 + fi + sleep 1 + done + + echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 +} + +if [[ "$KV_OFFLOADING" == "none" ]]; then + OFFLOAD_MODE="none" +else + OFFLOAD_MODE="${KV_OFFLOAD_BACKEND:?KV_OFFLOAD_BACKEND required when KV_OFFLOADING=dram}" +fi + +case "$OFFLOAD_MODE" in + none) + # GPU-only KV baseline: keep on-GPU prefix caching ON (no DRAM offload) + # so it's apples-to-apples vs the lmcache cell. + PREFIX_CACHE_ARGS=(--enable-prefix-caching) + ;; + lmcache) + unset VLLM_USE_SIMPLE_KV_OFFLOAD + + # Build LMCache against ROCm if the connector isn't importable. Clone to + # a container-local dir (NOT bind-mounted /workspace) so the next job's + # checkout `clean: true` won't trip over root-owned build artifacts. + if ! python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null 2>&1; then + LMCACHE_SRC_DIR="${LMCACHE_SRC_DIR:-/opt/lmcache-src}" + LMCACHE_GIT_REF="${LMCACHE_GIT_REF:-aaf7c0d3}" + rm -rf "$LMCACHE_SRC_DIR" + git clone https://github.com/LMCache/LMCache.git "$LMCACHE_SRC_DIR" + ( cd "$LMCACHE_SRC_DIR" + git checkout "$LMCACHE_GIT_REF" + pip install -r requirements/build.txt + CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation ) + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + fi + + LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" + LMCACHE_PORT="${LMCACHE_PORT:-5555}" + LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" + LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" + # LMCache L1 is SHM-backed: if L1 > /dev/shm free it silently disables + # SHM and falls back to the pickle path (crashes at load). Cap L1 to 90% + # of /dev/shm free so SHM stays enabled. + SHM_FREE_GB=$(df -BG --output=avail /dev/shm 2>/dev/null | tail -1 | tr -dc '0-9') + SHM_CAP_GB=$(( SHM_FREE_GB * 90 / 100 )) + LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$TOTAL_CPU_DRAM_GB}" + if [ -n "$SHM_CAP_GB" ] && [ "$SHM_CAP_GB" -gt 0 ] && [ "$LMCACHE_L1_SIZE_GB" -gt "$SHM_CAP_GB" ]; then + echo "Capping LMCACHE_L1_SIZE_GB $LMCACHE_L1_SIZE_GB -> $SHM_CAP_GB to fit /dev/shm (${SHM_FREE_GB}G free)" + LMCACHE_L1_SIZE_GB="$SHM_CAP_GB" + fi + LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" + LMCACHE_L1_READ_TTL_SECONDS="${LMCACHE_L1_READ_TTL_SECONDS:-7200}" + LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" + LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$((TP * 2))}" + export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" + export LMCACHE_BLOCKING_TIMEOUT_SECS=60 + + echo "Starting LMCache MP server..." + LMCACHE_CMD=( + lmcache server + --host "$LMCACHE_HOST" + --port "$LMCACHE_PORT" + --http-host "$LMCACHE_HOST" + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$LMCACHE_L1_SIZE_GB" + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" + --l1-read-ttl-seconds "$LMCACHE_L1_READ_TTL_SECONDS" + --chunk-size "$LMCACHE_CHUNK_SIZE" + --max-workers "$LMCACHE_MAX_WORKERS" + --eviction-policy LRU + ) + printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" + printf '\n' >> "$RESULT_DIR/lmcache_command.txt" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + echo "LMCache server PID: $LMCACHE_PID" + wait_for_lmcache_ready + + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT}}" + ) + ;; + mooncake) + unset VLLM_USE_SIMPLE_KV_OFFLOAD + + # Mooncake embedded mode contributes one global segment per GPU rank to + # a shared distributed store, so pre-divide the aggregate host-memory + # budget across TP ranks. Mirrors the MI355X DSv4 vLLM recipe. + MOONCAKE_PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) + + # No prebuilt ROCm wheel: build the Mooncake transfer engine from source + # if the store module isn't importable. Clone to a container-local dir + # (NOT bind-mounted /workspace) so the next job's checkout `clean: true` + # won't trip over root-owned build artifacts. + # + # A bare `make install` only lays down the C++ libs, so the launcher can + # import `mooncake` but the vLLM worker SUBPROCESSES cannot (fresh + # site-packages) -> "Please install mooncake ..." at KV-cache init. Build + # the wheel via Mooncake's own scripts/build_wheel.sh (auditwheel bundles + # every .so into a self-contained package) and pip install it so the + # `mooncake` package lands in site-packages and is importable everywhere. + if ! python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null 2>&1; then + MOONCAKE_SRC_DIR="${MOONCAKE_SRC_DIR:-/opt/mooncake-src}" + MOONCAKE_GIT_REF="${MOONCAKE_GIT_REF:-main}" + MOONCAKE_PYVER="$(python3 -c 'import sys;print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + pip install --quiet build auditwheel patchelf + rm -rf "$MOONCAKE_SRC_DIR" + git clone https://github.com/kvcache-ai/Mooncake.git "$MOONCAKE_SRC_DIR" + ( cd "$MOONCAKE_SRC_DIR" + git checkout "$MOONCAKE_GIT_REF" + bash dependencies.sh + mkdir -p build && cd build + cmake .. -DWITH_STORE=ON + make -j + make install + cd .. + # Produces a self-contained wheel under mooncake-wheel/dist/. + bash scripts/build_wheel.sh "$MOONCAKE_PYVER" + pip install --force-reinstall mooncake-wheel/dist/*.whl + + # CMake installs the Python package into the user site, which + # shadows the wheel installed by pip. Ensure the CLI wrapper can + # find the native master binary in the package it imports. + MOONCAKE_PACKAGE_DIR=$(python3 -c 'import pathlib, mooncake; print(pathlib.Path(mooncake.__file__).parent)') + install -m 0755 build/mooncake-store/src/mooncake_master \ + "$MOONCAKE_PACKAGE_DIR/mooncake_master" ) + python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + test -x "$(python3 -c 'import pathlib, mooncake; print(pathlib.Path(mooncake.__file__).parent / "mooncake_master")')" + fi + + MOONCAKE_MASTER_PORT=$((PORT + 12000)) + MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" + cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & + MOONCAKE_MASTER_PID=$! + echo "Mooncake master PID: $MOONCAKE_MASTER_PID" + sleep 10 + if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then + echo "Mooncake master died during startup. Log follows:" >&2 + cat "$MOONCAKE_MASTER_LOG" >&2 || true + exit 1 + fi + + PREFIX_CACHE_ARGS=(--enable-prefix-caching) + OFFLOAD_ARGS=( + --kv-transfer-config + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$OFFLOAD_MODE' (expected: lmcache, mooncake)" >&2 + exit 1 + ;; +esac + +EP_ARGS=() +if [ "$EP_SIZE" -gt 1 ]; then + EP_ARGS=(--enable-expert-parallel) +fi + +# Parallelism layout: +# DP_ATTENTION=false -> pure TP: attention TP-sharded across all $TP GPUs in a +# single engine (low TPOT, capacity-capped at high CONC). +# DP_ATTENTION=true -> DEP: per-DP-rank attention (TP1 x DP=$TP) with experts +# EP-sharded across the DP ranks (requires EP_SIZE>1). Grows KV capacity + +# decode width with concurrency. Mirrors dsv4_fp4_b300_vllm.sh. +PARALLEL_ARGS=(--tensor-parallel-size="$TP") +USE_VLLM_ROUTER=false +VLLM_BACKEND_PORT="$PORT" +if [ "$DP_ATTENTION" = "true" ]; then + PARALLEL_ARGS=(--tensor-parallel-size=1 --data-parallel-size="$TP") + USE_VLLM_ROUTER=true + VLLM_BACKEND_PORT=$((PORT + 1)) + VLLM_ROUTER_METRICS_PORT=$((PORT + 10000)) + # vllm-router expands the one HTTP backend into one logical worker per DP + # rank and sends X-data-parallel-rank per request; consistent_hash pins each + # conversation to a rank so its prefix cache stays warm. aiperf's stable + # X-Correlation-ID is aliased to the router's X-Session-ID for that affinity. + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 + pip install --quiet "vllm-router==0.1.14" +fi + +echo "Starting vllm server..." +export PYTHONNOUSERSITE=1 + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$VLLM_BACKEND_PORT" + "${PARALLEL_ARGS[@]}" + "${EP_ARGS[@]}" + --gpu-memory-utilization 0.90 + --trust-remote-code + --max-num-seqs "$CONC" + --mm-encoder-tp-mode data + "${PREFIX_CACHE_ARGS[@]}" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +# In DEP mode, front the DP engine with vllm-router on $PORT so the agentic +# client (which targets $PORT) load-balances across DP ranks with prefix +# affinity. Pure-TP serves the client directly on $PORT. +if [ "$USE_VLLM_ROUTER" = "true" ]; then + ROUTER_LOG="$RESULT_DIR/router.log" + echo "Starting vLLM router on port $PORT for $TP DP ranks..." + vllm-router \ + --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ + --policy consistent_hash \ + --intra-node-data-parallel-size "$TP" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \ + --request-timeout-secs 14400 \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + echo "Router PID: $ROUTER_PID" + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +# ---- Run benchmark ---------------------------------------------------------- +build_replay_cmd "$RESULT_DIR" + +run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4b1ea4aed..92c1c7fab 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -814,6 +814,69 @@ kimik2.5-fp4-mi355x-vllm-agentic: - { tp: 4, kv-offloading: none, conc-list: [16, 24, 32, 40] } - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [16, 24, 32, 40] } +kimik2.7-fp4-mi355x-vllm-agentic: + image: vllm/vllm-openai-rocm:v0.24.0 + model: amd/Kimi-K2.7-Code-MXFP4 + model-prefix: kimik2.7 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + # TP8, 3600s. Compares three KV tiers on the agentic trace. At TP8 the + # GPU-only (none) path collapses under load (KV-capacity thrash), so it is + # only swept at low conc [4,8] where it is viable; the dram offload backends + # (LMCache, Mooncake) carry the high-conc [16,32,48] range where they hold + # throughput by keeping the GPU KV pool from thrashing. + - dram-utilization: 0.80 + duration: 3600 + search-space: + - { tp: 8, kv-offloading: none, conc-list: [4, 8] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "aaf7c0d3" }, conc-list: [16, 32, 48] } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "source-main" }, conc-list: [16, 32, 48] } + + +# Perf-tuning experiment key (same recipe as the record sweep above, via +# model-prefix kimik2.7). Goal: get a throughput curve that scales with +# concurrency instead of collapsing. Compares pure-TP8 (current, capacity-capped) +# vs DEP (data-parallel attention TP1xDP8 + expert parallelism), both with LMCache. +# Dispatched at 900s via e2e-tests duration override to verify the perf delta +# before committing a full 3600s run. Mooncake is split into a separate key +# (-tune-mc) so a fast Mooncake bringup failure can't fail-fast-cancel these cells. +kimik2.7-fp4-mi355x-vllm-agentic-tune: + image: vllm/vllm-openai-rocm:v0.24.0 + model: amd/Kimi-K2.7-Code-MXFP4 + model-prefix: kimik2.7 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + duration: 900 + search-space: + - { tp: 8, dp-attn: false, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "aaf7c0d3" }, conc-list: [16, 32, 48] } + - { tp: 8, dp-attn: true, ep: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "aaf7c0d3" }, conc-list: [16, 32, 48] } + +# Mooncake-under-DEP, isolated so its failure can't cancel the LMCache scaling run. +kimik2.7-fp4-mi355x-vllm-agentic-tune-mc: + image: vllm/vllm-openai-rocm:v0.24.0 + model: amd/Kimi-K2.7-Code-MXFP4 + model-prefix: kimik2.7 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + duration: 900 + search-space: + - { tp: 8, dp-attn: true, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "source-main" }, conc-list: [16, 32, 48] } + + kimik2.5-fp4-mi355x-atom: image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.4_202607091539 model: amd/Kimi-K2.5-MXFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e05c2ca1c..5b78b8d76 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4765,3 +4765,20 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - kimik2.7-fp4-mi355x-vllm-agentic + description: + - "Add Kimi-K2.7-Code-MXFP4 agentic-coding config on MI355X (runner: cluster:mi355x-amds) via vLLM v0.24.0, TP4" + - "Sweep KV-offloading none (conc [1,4,8]) vs dram/LMCache (conc [16,32]); LMCache MP server + LMCacheMPConnector, dram-utilization 0.80" + - "Recipe benchmarks/single_node/agentic/kimik2.7_fp4_mi355x.sh clones LMCache to container-local /opt/lmcache-src (pinned) so CI checkout never trips over root-owned build artifacts" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2126 + +- config-keys: + - kimik2.7-fp4-mi355x-vllm-agentic + description: + - "Move the agentic-coding sweep from TP4 to TP8 and compare three KV tiers on the agentic trace: GPU-only (none) at conc [4,8], and dram/LMCache + dram/Mooncake at conc [16,32,48] (3600s trace replay)" + - "TP4 was GPU-KV-capacity bound: the ~121 GiB pool held only ~12 resident ~90k-token contexts, so throughput plateaued (~160 tok/s) and requests preempted/thrashed from conc16 up. GPU-only still collapses under load at TP8, so it is swept only at low conc; the dram offload backends carry the high-conc range" + - "Add a Mooncake KV-offload backend to the recipe (embedded MooncakeStoreConnector store). Built from source for ROCm via Mooncake's build_wheel.sh and pip-installed so the mooncake package lands in site-packages and is importable from the vLLM worker subprocesses (a bare make install left workers unable to import it)" + - "Adds optional per-scenario 'duration' override to the agentic-coding config schema (defaults to 3600s when unset)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2126 diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index babc328a8..b1e58298c 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -627,7 +627,7 @@ def generate_full_sweep(args, all_config_data, runner_data): for agentic_config in agentic_configs: bmk_space = agentic_config[Fields.SEARCH_SPACE.value] - duration = DEFAULT_AGENTIC_DURATION_SECONDS + duration = agentic_config.get(Fields.DURATION.value) or DEFAULT_AGENTIC_DURATION_SECONDS for bmk in bmk_space: if is_multinode: @@ -932,7 +932,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): # ---- Agentic-coding scenarios ---- agentic_configs = val[Fields.SCENARIOS.value].get(Fields.AGENTIC_CODING.value, []) if (scenario_filter is None or 'agentic-coding' in scenario_filter) else [] for agentic_config in agentic_configs: - duration = DEFAULT_AGENTIC_DURATION_SECONDS + duration = agentic_config.get(Fields.DURATION.value) or DEFAULT_AGENTIC_DURATION_SECONDS bmk_space = agentic_config[Fields.SEARCH_SPACE.value] for bmk in bmk_space: diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 9925adb62..7534e5b36 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -574,10 +574,33 @@ def test_available_cpu_dram_is_not_a_master_config_field(self): }], }) - def test_duration_is_not_a_master_config_field(self): + def test_duration_is_an_optional_master_config_field(self): + # Per-scenario duration override (falls back to the default 3600s in the + # sweep generator when unset). Used for shorter agentic smoke sweeps. + cfg = AgenticCodingConfig(**{ + "duration": 1800, + "search-space": [{ + "tp": 8, + "kv-offloading": "none", + "conc-list": [16], + }], + }) + assert cfg.duration == 1800 + + def test_duration_defaults_to_none_when_unset(self): + cfg = AgenticCodingConfig(**{ + "search-space": [{ + "tp": 8, + "kv-offloading": "none", + "conc-list": [16], + }], + }) + assert cfg.duration is None + + def test_duration_must_be_positive(self): with pytest.raises(Exception, match="duration"): AgenticCodingConfig(**{ - "duration": 1800, + "duration": 0, "search-space": [{ "tp": 8, "kv-offloading": "none", diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 6b7c67128..d0728b5fa 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -633,6 +633,11 @@ class AgenticCodingConfig(BaseModel): dram_utilization: Optional[float] = Field( default=None, alias=Fields.DRAM_UTILIZATION.value, gt=0, le=1 ) + # Per-scenario trace-replay duration override (seconds). Falls back to + # DEFAULT_AGENTIC_DURATION_SECONDS in the sweep generator when unset. + duration: Optional[int] = Field( + default=None, alias=Fields.DURATION.value, gt=0 + ) @model_validator(mode='after') def validate_dram_offload_capacity(self):