From b55c07cb1f28cf17444db29eef415e4c1cb28011 Mon Sep 17 00:00:00 2001 From: Sirra Date: Sat, 12 Sep 2026 16:04:26 +0530 Subject: [PATCH 01/10] [AMD] [AGENTX] KIMI Perf Tuning Signed-off-by: Sirra --- .../agentic/kimik3_fp4_mi355x_mtp.sh | 429 +++++++----------- configs/amd-master.yaml | 9 +- perf-changelog.yaml | 8 + 3 files changed, 164 insertions(+), 282 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index f43c6ed13a..8bce2bea52 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -1,70 +1,20 @@ #!/usr/bin/env bash set -euo pipefail set -x - -# Agentic trace replay benchmark for Kimi-K3 MXFP4 on MI355X / MI350X (gfx950) -# using vLLM. -# -# The server command is the AMD reference `vllm serve` for this model, i.e. the -# upstream vLLM recipe's amd block (vllm-project/recipes, -# https://recipes.vllm.ai/moonshotai/Kimi-K3) as run in practice: -# -# --trust-remote-code --moe-backend auto --tensor-parallel-size 8 -# --load-format auto --gpu-memory-utilization 0.95 --mm-encoder-tp-mode data -# --max-num-seqs 128 --max-num-batched-tokens 4096 --enable-auto-tool-choice -# --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 -# -# with env VLLM_ROCM_USE_AITER=1 SAFETENSORS_FAST_GPU=1 AITER_SITUV2_A8W4=1 -# AITER_BF16_FP8_MOE_BOUND=0 VLLM_USE_BREAKABLE_CUDAGRAPH=0. -# -# K3 is a 2.8T-parameter natively-multimodal MoE (896 routed experts, 16/token -# plus shared) on Kimi Delta Attention, gated MLA and Attention Residuals, with -# a 1M-token native context. -# -# TP=8 ONLY. The MXFP4 checkpoint is 1.561 TB decimal (1.420 TiB, 96 -# safetensors), ~195 GB/GPU across 8 GPUs of the 288 GB part; TP=4 would need -# ~390 GB/GPU and cannot load. Upstream strategy_min_gpus agrees (single_node_tp -# and multi_node_tep both 8, DEP 16+), which is why there is no DP-attention arm. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, -# EP_SIZE -# -# Perf-search knobs. Each defaults to the reference command's value, so an -# otherwise-unset run reproduces the reference exactly: -# GPU_MEM_UTIL 0.95 (reference) -# MAX_NUM_BATCHED_TOKENS 8192 (default) -# AITER_A8W4 1 (reference; 0 = aiter a16w4 MoE path) -# LANGUAGE_MODEL_ONLY true -# KV_CACHE_DTYPE fp8 (default for every arm; =auto for a bf16 A/B) -# KV_BLOCK_SIZE unset (unset -> vLLM sizes the page; 128 under fp8) -# MAX_MODEL_LEN 1M -# SPEC_DECODE true (this is the _mtp DSpark recipe; =false for a no-spec A/B) -# SPEC_NUM_TOKENS 2 (DSpark draft length; validated by the _mtp config) - source "$(dirname "$0")/../../benchmark_lib.sh" - wait_for_amd_gpu_clean +export EVAL_ONLY="${EVAL_ONLY:-false}" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi +DP_SIZE=1 +export DP_SIZE +TOTAL_RANKS=$(( TP * DP_SIZE )) -if [ "$TP" -ne 8 ]; then - echo "Error: Kimi-K3 MXFP4 is a 1.56 TB checkpoint and only fits at TP=8 on" >&2 - echo " 288 GB gfx950 parts (~195 GB/GPU). Got TP=$TP." >&2 - exit 1 -fi - -# ROCR/HIP visibility for vLLM 0.14+ if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" fi -# `hf download` creates the target dir if missing and is itself idempotent. The -# 1.56 TB checkpoint is normally pre-staged, so these calls are a no-op there. if [[ -n "${MODEL_PATH:-}" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" @@ -75,260 +25,127 @@ else fi rocm-smi || true -amd-smi || true - -# ---- Resolve traces and install deps ---------------------------------------- resolve_trace_source install_agentic_deps -# ---- Reference env block ---------------------------------------------------- export VLLM_ROCM_AITER_MLA_ASM_PADDING=asm export VLLM_ROCM_USE_AITER=1 -export SAFETENSORS_FAST_GPU=1 +export VLLM_ROCM_USE_AITER_MLA=1 +export VLLM_ROCM_USE_AITER_MOE=1 export VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION="${VLLM_ROCM_QUICK_REDUCE_QUANTIZATION:-NONE}" + export AITER_SITUV2_A8W4=1 +export AITER_FLYDSL_STAGE2_FP8="${AITER_FLYDSL_STAGE2_FP8:-1}" export AITER_BF16_FP8_MOE_BOUND=0 +export AITER_DISABLE_FMHA_OPUS=1 +export SAFETENSORS_FAST_GPU=1 +export GPU_ARCHS=gfx950 +export HSA_NO_SCRATCH_RECLAIM=1 export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 - -# Workaround for MEC FW <177 RCCL memory reclaim issue (shared with the other -# gfx950 recipes in this tree). -mec_version=$(rocm-smi --showfw 2>/dev/null | grep MEC | head -n 1 | awk '{print $NF}') -if [[ "$mec_version" == "" || ${mec_version:-0} -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -# 2.8T of weights off a shared/NFS mount takes far longer than the default. -export VLLM_ENGINE_READY_TIMEOUT_S="${VLLM_ENGINE_READY_TIMEOUT_S:-7200}" - -# Long agentic turns against a 1M context: keep the client from timing out -# mid-request while the server is prefill-bound. +export VLLM_K3_KDA_SAFE_STAGES=1 +export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1 +export VLLM_ENGINE_READY_TIMEOUT_S=7200 +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600 export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 +export PYTHONNOUSERSITE=1 +export PYTHONHASHSEED=42 -# ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" - SERVER_PID="" -LMCACHE_PID="" cleanup_agentic_services() { local exit_code=$? trap - EXIT INT TERM set +e stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - stop_background_process_tree "$LMCACHE_PID" "LMCache server" exit "$exit_code" } trap cleanup_agentic_services EXIT trap 'exit 130' INT trap 'exit 143' TERM -# ---- KV offload ------------------------------------------------------------- -# TOTAL_CPU_DRAM_GB is the aggregate host-DRAM budget the matrix generator -# derives from dram-utilization and the runner's available-cpu-dram-mib, capped -# at the 3,095,781 MiB (3 TB decimal) agentic limit. Per -# benchmarks/single_node/agentic/README.md it must be consumed as given and -# never replaced with a model-specific constant. -OFFLOAD_ARGS=() - -if agentic_kv_offload_enabled; then -case "${KV_OFFLOAD_BACKEND:-}" in - vllm-simple) - require_agentic_kv_offload_backend "$KV_OFFLOAD_BACKEND" - CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / TP )) - # Identical prefixes must hash to identical block keys across ranks. - export PYTHONHASHSEED=42 - SIMPLE_LAZY_OFFLOAD="${SIMPLE_LAZY_OFFLOAD:-false}" - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":$CPU_BYTES_PER_RANK,\"lazy_offload\":$SIMPLE_LAZY_OFFLOAD}}" - ) - echo "SimpleCPUOffloadConnector: ${CPU_BYTES_PER_RANK} B/rank x ${TP} ranks, lazy_offload=$SIMPLE_LAZY_OFFLOAD" - ;; - lmcache) - require_agentic_kv_offload_backend "$KV_OFFLOAD_BACKEND" - - LMCACHE_VERSION=0.5.5.dev114+rocm7.2 - LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/nightly-rocm" - - agentic_pip_install --quiet --no-cache-dir --no-deps \ - "sortedcontainers==2.4.0" \ - "opentelemetry-exporter-prometheus==0.61b0" \ - "cupy-rocm-7-0==14.1.1" \ - "lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX" - - # LMCache 0.5.5's transfer-channel layer eagerly imports the Mooncake - # backend (mooncake_te_impl.py -> `from mooncake.engine import - # TransferEngine`), whose native .so resolves all of its DT_NEEDED libs at - # import. The vLLM ROCm image ships none of them, so the import sanity - # check below (and the LMCache server) would otherwise fail with - # "ImportError: lib*.so: cannot open shared object file" (first libglog, - # then libjsoncpp, ...). Provision Mooncake's full runtime lib set from the - # distro before importing. apt-get install is idempotent, so run it - # whenever any of the libs is still missing rather than gating on one. - LMCACHE_NATIVE_LIBS=(libglog.so.0 libjsoncpp.so.25 libibverbs.so.1 librdmacm.so.1 libnuma.so.1) - for lib in "${LMCACHE_NATIVE_LIBS[@]}"; do - if ! ldconfig -p | grep -q "$lib"; then - apt-get update - apt-get install -y \ - libgoogle-glog0v5 libjsoncpp25 libibverbs1 librdmacm1 libnuma1 - break - fi - done - python3 -c \ - "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ - >/dev/null - - # One MP server for the node, per the Kimi-K3 recipe - # (docs.lmcache.ai/recipes/kimi_k3.html), with --chunk-size sized for - # THIS stack rather than the recipe's CUDA-path 768: the connector - # requires the chunk to be a multiple of every engine KV group's - # tokens_per_block, and the hybrid KDA/MLA layout here registers - # attention groups at 1536 ("Setting attention block size to 1536", - # run 31644990546) plus a KDA state group at 3072 (run 31645828378), - # so 3072 is the minimum valid chunk. The multi-group layout also - # requires one object group per sliding-window size: - # --separate-object-groups. - LMCACHE_PORT=6555 - LMCACHE_HTTP_PORT=8090 - LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" - - LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB" - - # DCP shards decode KV across the TP ranks, so the LMCache GPU transfer - # pool needs one worker per rank; a non-DCP arm only needs a single worker. - # The DCP KV interleave also needs the larger 12288 chunk; a non-DCP arm - # uses the 3072 minimum (one KDA state group). - if [ "${DCP_SIZE:-1}" -gt 1 ]; then - LMCACHE_MAX_GPU_WORKERS=8 - LMCACHE_CHUNK_SIZE=12288 - else - LMCACHE_MAX_GPU_WORKERS=1 - LMCACHE_CHUNK_SIZE=3072 - fi - - LMCACHE_CMD=( - lmcache server - --host 127.0.0.1 - --port "$LMCACHE_PORT" - --http-host 127.0.0.1 - --http-port "$LMCACHE_HTTP_PORT" - --l1-size-gb "$LMCACHE_L1_SIZE_GB" - --l1-init-size-gb 10 - --chunk-size "$LMCACHE_CHUNK_SIZE" - --separate-object-groups - --enable-extra-logging - --extra-logging-interval 30 - --max-cpu-workers 8 - --max-gpu-workers "$LMCACHE_MAX_GPU_WORKERS" - --eviction-policy LRU - --supported-transfer-mode lmcache_driven - --shm-name "" - ) - append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" - "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & - LMCACHE_PID=$! - wait_for_ready \ - --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \ - --log "$LMCACHE_LOG" \ - --pid "$LMCACHE_PID" \ - --sleep-interval 1 \ - --timeout 600 - - # 100k-330k-token agentic prefixes make single retrieves large; use the - # same MQ timeout headroom as the MiniMax-M3 arm. - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}" - ) - ;; - *) - echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected vllm-simple or lmcache)" >&2 - exit 1 - ;; -esac -fi - -# ---- LLM server ------------------------------------------------------------ - -# ---- Parallelism ------------------------------------------------------------ -EP_ARGS=() -if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# ---- Speculative / Util------------------------------------------------------ +SPEC_ARGS=() +SPEC_ROWS=1 +KDA_ARGS=() case "$CONC" in - # No KV offload; the working set fits in HBM. - 1) - SYNTHETIC_ACCEPT_LEN=3.75 - SPEC_NUM_TOKENS=6 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=16384 - ;; - 4|8|10|12|14) - SYNTHETIC_ACCEPT_LEN=3.00 - SPEC_NUM_TOKENS=3 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=8192 - ;; - 44|48|52) - SPEC_NUM_TOKENS=0 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=8192 + 1|2|4|8) + DCP_SIZE="${DCP_SIZE:-1}" + SPEC_NUM_TOKENS="${SPEC_NUM_TOKENS:-4}" + case "$SPEC_NUM_TOKENS" in + 1) SYNTHETIC_ACCEPT_LEN=1.85 ;; + 2) SYNTHETIC_ACCEPT_LEN=2.51 ;; + 3) SYNTHETIC_ACCEPT_LEN=3.00 ;; + 4) SYNTHETIC_ACCEPT_LEN=3.36 ;; + 5) SYNTHETIC_ACCEPT_LEN=3.62 ;; + 6) SYNTHETIC_ACCEPT_LEN=3.75 ;; + 7) SYNTHETIC_ACCEPT_LEN=3.84 ;; + 8) SYNTHETIC_ACCEPT_LEN=4.00 ;; + *) echo "[spec] no golden AL for k=$SPEC_NUM_TOKENS" >&2; exit 1 ;; + esac + DRAFT_KV_DTYPE="${DRAFT_KV_DTYPE:-fp8}" + SPEC_BASE="\"model\":\"Inferact/Kimi-K3-DSpark\",\"num_speculative_tokens\":$SPEC_NUM_TOKENS,\"method\":\"dspark\",\"attention_backend\":\"TRITON_MLA\",\"kv_cache_dtype\":\"$DRAFT_KV_DTYPE\",\"draft_sample_method\":\"probabilistic\"" + if [ "${EVAL_ONLY:-false}" = "true" ]; then + SPEC_ARGS=(--speculative-config "{$SPEC_BASE,\"rejection_sample_method\": \"block\"}") + echo "MTP: k=$SPEC_NUM_TOKENS LIVE block rejection (accuracy gate) draft_kv=$DRAFT_KV_DTYPE" + else + SPEC_ARGS=(--speculative-config "{$SPEC_BASE,\"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}") + echo "MTP: k=$SPEC_NUM_TOKENS synthetic_accept=$SYNTHETIC_ACCEPT_LEN draft_kv=$DRAFT_KV_DTYPE" + fi + SPEC_ROWS=$(( SPEC_NUM_TOKENS + 1 )) + KDA_ARGS=(--additional-config '{"kda_prefill_backend":"triton"}') + MAX_NUM_SEQS="${MAX_NUM_SEQS:-$(( CONC > 2 ? CONC : 2 ))}" + MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-16384}" ;; *) - SPEC_NUM_TOKENS=0 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=8192 + DCP_SIZE="${DCP_SIZE:-8}" + MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-24576}" + if [ "$CONC" -lt 72 ]; then MAX_NUM_SEQS="${MAX_NUM_SEQS:-$(( CONC * 14 / 10 ))}" + elif [ "$CONC" -eq 72 ]; then MAX_NUM_SEQS="${MAX_NUM_SEQS:-96}" + else MAX_NUM_SEQS="${MAX_NUM_SEQS:-112}"; fi ;; esac +export DCP_SIZE -SPEC_ARGS=() -if [ "$SPEC_NUM_TOKENS" -gt 0 ]; then -if [ "${EVAL_ONLY:-false}" = "true" ]; then - SPEC_ARGS=( - --speculative-config - "{\"model\":\"Inferact/Kimi-K3-DSpark\",\"num_speculative_tokens\":$SPEC_NUM_TOKENS,\"method\":\"dspark\",\"attention_backend\":\"TRITON_MLA\",\"kv_cache_dtype\":\"fp8\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\": \"block\"}" - ) -else - SPEC_ARGS=( - --speculative-config - "{\"model\":\"Inferact/Kimi-K3-DSpark\",\"num_speculative_tokens\":$SPEC_NUM_TOKENS,\"method\":\"dspark\",\"attention_backend\":\"TRITON_MLA\",\"kv_cache_dtype\":\"fp8\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" - ) +NUMA_ARGS=() +if [ "${K3_NUMA_BIND:-0}" = "1" ]; then + if ! command -v numactl >/dev/null 2>&1; then + apt-get update -qq >/dev/null 2>&1 && apt-get install -y -qq numactl >/dev/null 2>&1 || true fi + command -v numactl >/dev/null 2>&1 || { echo "[numa] FATAL: numactl unavailable, --numa-bind would silently no-op" >&2; exit 1; } + numactl --cpunodebind=0 --membind=0 true 2>/dev/null || { + numactl --cpunodebind=0 true 2>/dev/null && echo "[numa] WARN: membind rejected, CPU binding only" >&2 \ + || { echo "[numa] FATAL: numactl present but binding rejected, --numa-bind would silently no-op" >&2; exit 1; } + } + export VLLM_WORKER_MULTIPROC_METHOD=spawn + NUMA_ARGS=(--numa-bind) + echo "[numa] $(numactl --show | tr '\n' ' ')" fi -# ---- HIP graph ------------------------------------------------------------ -MAX_NUM_SEQS=$((2 * CONC)) -MAX_CUDAGRAPH_CAPTURE_SIZE=$((MAX_NUM_SEQS * (1 + SPEC_NUM_TOKENS))) -CUDAGRAPH_CAPTURE_SIZES="$(seq -s, 2 "$MAX_CUDAGRAPH_CAPTURE_SIZE")" -COMPILATION_CONFIG_ARGS=(--compilation-config "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"max_cudagraph_capture_size\":$MAX_CUDAGRAPH_CAPTURE_SIZE,\"custom_ops\":[\"+fused_rms_norm_gated\"],\"cudagraph_capture_sizes\":[$CUDAGRAPH_CAPTURE_SIZES]}") - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 -export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS="${VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS:-1200}" +GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}" +CUDAGRAPH_MODE="${CUDAGRAPH_MODE:-FULL_DECODE_ONLY}" +LADDER=$(( MAX_NUM_SEQS * SPEC_ROWS )) +CUDAGRAPH_CAPTURE_SIZES=$(seq -s, 1 "$LADDER") +COMPILATION_CONFIG_ARGS=(--compilation-config "{\"mode\":3,\"cudagraph_mode\":\"$CUDAGRAPH_MODE\",\"max_cudagraph_capture_size\":$LADDER,\"custom_ops\":[\"+fused_rms_norm_gated\"],\"cudagraph_capture_sizes\":[$CUDAGRAPH_CAPTURE_SIZES]}") -# ---- DCP ------------------------------------------------------------ -# DCP shards decode KV across the TP ranks, so it must divide TP. -DCP_SIZE="${DCP_SIZE:-8}" -if [ $((TP % DCP_SIZE)) -ne 0 ]; then - echo "Error: TP='$TP' must be divisible by DCP_SIZE='$DCP_SIZE'" >&2 - exit 1 -fi -CP_ARGS=() -ATTN_BE_ARGS=() +CP_ARGS=(--attention-backend ROCM_AITER_MLA) if [ "$DCP_SIZE" -gt 1 ]; then - CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a) - ATTN_BE_ARGS+=(--attention-backend ROCM_AITER_MLA) + CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a --cp-kv-cache-interleave-size 1) fi -export VLLM_USE_DIRECT_DCP_A2A=0 -export VLLM_USE_DIRECT_DCP_Q_GATHER=0 -export VLLM_USE_DIRECT_DCP_KV_GATHER=0 -{ set +x; } 2>/dev/null +OFFLOAD_ARGS=() +if agentic_kv_offload_enabled; then + CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / TOTAL_RANKS )) + OFFLOAD_ARGS=(--kv-transfer-config "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":$CPU_BYTES_PER_RANK,\"lazy_offload\":false}}") +fi + +EP_ARGS=() +if [ "${EP_SIZE:-1}" -gt 1 ]; then EP_ARGS=(--enable-expert-parallel); fi + +echo "[cfg] conc=$CONC dcp=$DCP_SIZE gmu=$GPU_MEM_UTIL mns=$MAX_NUM_SEQS ladder=1..$LADDER spec_rows=$SPEC_ROWS chunk=$MAX_BATCHED_TOKENS cudagraph=$CUDAGRAPH_MODE offload=${KV_OFFLOADING:-none}" + VLLM_CMD=( vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 @@ -336,36 +153,94 @@ VLLM_CMD=( --trust-remote-code --moe-backend auto --tensor-parallel-size "$TP" - "${EP_ARGS[@]}" --load-format fastsafetensors --gpu-memory-utilization "$GPU_MEM_UTIL" --language-model-only --max-num-seqs "$MAX_NUM_SEQS" + --max-num-batched-tokens "$MAX_BATCHED_TOKENS" + --max-model-len 1048576 + --kv-cache-dtype fp8 --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 - --max-model-len 1048576 --enable-prefix-caching - --kv-cache-dtype "fp8" - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --enable-prompt-tokens-details + --no-async-scheduling --attention-config '{"mla_prefill_backend":"ROCM_AITER_FA"}' - "${ATTN_BE_ARGS[@]}" - "${COMPILATION_CONFIG_ARGS[@]}" - "${SPEC_ARGS[@]}" "${OFFLOAD_ARGS[@]}" "${CP_ARGS[@]}" + "${EP_ARGS[@]}" + "${SPEC_ARGS[@]}" + "${NUMA_ARGS[@]}" + "${KDA_ARGS[@]}" + "${COMPILATION_CONFIG_ARGS[@]}" ) + printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" + "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! echo "Server PID: $SERVER_PID" +python3 - <<'CCDPY' > /tmp/ccdmap.txt 2>/dev/null || true +import subprocess, re, os, glob +def expand(s): + v=[] + for part in s.split(','): + if '-' in part: + a,b=part.split('-'); v+=list(range(int(a),int(b)+1)) + else: v.append(int(part)) + return v +def l3_domains(): + seen,out=set(),[] + for c in sorted(int(re.search(r'cpu(\d+)$',x).group(1)) for x in glob.glob('/sys/devices/system/cpu/cpu[0-9]*')): + f=f'/sys/devices/system/cpu/cpu{c}/cache/index3/shared_cpu_list' + if not os.path.exists(f): continue + d=open(f).read().strip() + if d not in seen: seen.add(d); out.append(d) + return out +def node_of(cpus): + for n in glob.glob('/sys/devices/system/node/node[0-9]*'): + nid=int(re.search(r'node(\d+)$',n).group(1)) + if cpus[0] in expand(open(f'{n}/cpulist').read().strip()): return nid + return -1 +topo="" +try: topo=subprocess.run(["rocm-smi","--showtoponuma"],capture_output=True,text=True).stdout +except Exception: pass +gpu_node={int(m.group(1)):int(m.group(2)) for m in re.finditer(r"GPU\[(\d+)\].*?Numa Node:\s*(\d+)",topo)} +if not gpu_node: raise SystemExit +by={} +for d in l3_domains(): by.setdefault(node_of(expand(d)),[]).append(d) +for n in by: by[n].sort(key=lambda d: expand(d)[0]) +for n in sorted(by): + for i,g in enumerate(sorted(k for k,v in gpu_node.items() if v==n)): + if i < len(by[n]): print(f"{g} {by[n][i]}") +CCDPY + +PIN_CCD="${PIN_CCD:-1}" +pin_workers_to_ccd() { + [ "$PIN_CCD" = "1" ] || return 0 + [ -s /tmp/ccdmap.txt ] || return 0 + local pinned=0 + while read -r _g _cpus; do + for _p in $(pgrep -f "VLLM::Worker_TP${_g}([^0-9]|$)" 2>/dev/null); do + for _t in /proc/$_p/task/*; do + taskset -pc "$_cpus" "${_t##*/}" >/dev/null 2>&1 && pinned=$((pinned+1)) || true + done + done + done < /tmp/ccdmap.txt + echo "[pin-ccd] pinned $pinned threads" +} + + wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" -if [ "${EVAL_ONLY}" = "true" ]; then +pin_workers_to_ccd || true + +if [ "${EVAL_ONLY:-false}" = "true" ]; then run_eval --port "$PORT" else build_replay_cmd "$RESULT_DIR" run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi +fi \ No newline at end of file diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 3d9a3dccaf..d98e851c9d 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -625,7 +625,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: - "DECODE_MTP_SIZE=2" kimik3-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb + image: vllm/vllm-openai-rocm:nightly-e7edf17cea217e52701f913cd8491fcacf2d9490 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds @@ -634,11 +634,10 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: multinode: false scenarios: agentic-coding: - - dram-utilization: 0.60 + - dram-utilization: 0.65 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1] , spec-decoding: mtp} - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5.dev114+rocm7.2" }, conc-list: [4, 8, 10, 12, 14], spec-decoding: mtp } - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5.dev114+rocm7.2" }, conc-list: [44, 48], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [72], spec-decoding: mtp } # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1f90deb80e..6a1af70464 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7408,3 +7408,11 @@ - "Enable required power for the seven existing B200 Kimi-K3 recipes, use the pinned AgentX producer and shared collector, fix the exporter import URI, and record DCP8 with offload disabled to match the serving commands." - "启用 Kimi-K3 B200 实测功耗并修正 DCP 与 Offload 元数据。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3043 + +- config-keys: + - kimik3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Image : vllm/vllm-openai-rocm:nightly-e7edf17cea217e52701f913cd8491fcacf2d9490" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD From a9a63ad7af5bbf3eb9df95ee411c1ea26fce058e Mon Sep 17 00:00:00 2001 From: Sirra Date: Sat, 12 Sep 2026 16:06:40 +0530 Subject: [PATCH 02/10] [AMD] [AGENTX] KIMI Perf Tuning Signed-off-by: Sirra --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6a1af70464..7480df09e1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7415,4 +7415,4 @@ - agentic-coding description: - "Image : vllm/vllm-openai-rocm:nightly-e7edf17cea217e52701f913cd8491fcacf2d9490" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3059 From 6cd867667d23a1536a41d1448b63f5aec2044624 Mon Sep 17 00:00:00 2001 From: Sirra Date: Sat, 12 Sep 2026 16:08:10 +0530 Subject: [PATCH 03/10] [AMD] [AGENTX] KIMI Perf Tuning Signed-off-by: Sirra --- benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index 8bce2bea52..9e7625e6e7 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -123,7 +123,7 @@ if [ "${K3_NUMA_BIND:-0}" = "1" ]; then echo "[numa] $(numactl --show | tr '\n' ' ')" fi -GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}" +GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.89}" CUDAGRAPH_MODE="${CUDAGRAPH_MODE:-FULL_DECODE_ONLY}" LADDER=$(( MAX_NUM_SEQS * SPEC_ROWS )) From 62b97387623900b84cee0562e7df3057b970e948 Mon Sep 17 00:00:00 2001 From: Sirra Date: Sat, 12 Sep 2026 16:15:15 +0530 Subject: [PATCH 04/10] [AMD] [AGENTX] KIMI Perf Tuning Signed-off-by: Sirra --- .../agentic/kimik3_fp4_mi355x_mtp.sh | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index 9e7625e6e7..6bc8a695ed 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -108,22 +108,7 @@ case "$CONC" in esac export DCP_SIZE -NUMA_ARGS=() -if [ "${K3_NUMA_BIND:-0}" = "1" ]; then - if ! command -v numactl >/dev/null 2>&1; then - apt-get update -qq >/dev/null 2>&1 && apt-get install -y -qq numactl >/dev/null 2>&1 || true - fi - command -v numactl >/dev/null 2>&1 || { echo "[numa] FATAL: numactl unavailable, --numa-bind would silently no-op" >&2; exit 1; } - numactl --cpunodebind=0 --membind=0 true 2>/dev/null || { - numactl --cpunodebind=0 true 2>/dev/null && echo "[numa] WARN: membind rejected, CPU binding only" >&2 \ - || { echo "[numa] FATAL: numactl present but binding rejected, --numa-bind would silently no-op" >&2; exit 1; } - } - export VLLM_WORKER_MULTIPROC_METHOD=spawn - NUMA_ARGS=(--numa-bind) - echo "[numa] $(numactl --show | tr '\n' ' ')" -fi - -GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.89}" +GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}" CUDAGRAPH_MODE="${CUDAGRAPH_MODE:-FULL_DECODE_ONLY}" LADDER=$(( MAX_NUM_SEQS * SPEC_ROWS )) @@ -171,7 +156,6 @@ VLLM_CMD=( "${CP_ARGS[@]}" "${EP_ARGS[@]}" "${SPEC_ARGS[@]}" - "${NUMA_ARGS[@]}" "${KDA_ARGS[@]}" "${COMPILATION_CONFIG_ARGS[@]}" ) @@ -243,4 +227,4 @@ if [ "${EVAL_ONLY:-false}" = "true" ]; then else build_replay_cmd "$RESULT_DIR" run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi \ No newline at end of file +fi From f0b36c9ad7c18250737b710c8ca8cd80e2c2bca7 Mon Sep 17 00:00:00 2001 From: Sirra Date: Sat, 12 Sep 2026 16:17:37 +0530 Subject: [PATCH 05/10] [AMD] [AGENTX] KIMI Perf Tuning Signed-off-by: Sirra --- benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index 6bc8a695ed..b0ce95bcdb 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -108,7 +108,7 @@ case "$CONC" in esac export DCP_SIZE -GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}" +GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.89}" CUDAGRAPH_MODE="${CUDAGRAPH_MODE:-FULL_DECODE_ONLY}" LADDER=$(( MAX_NUM_SEQS * SPEC_ROWS )) From d349d8299cdd5aacb072d654e6dd2b7f7e9de43e Mon Sep 17 00:00:00 2001 From: Sirra Date: Sat, 12 Sep 2026 21:51:17 +0530 Subject: [PATCH 06/10] [AMD] [AGENTX] KIMI Perf Tuning Signed-off-by: Sirra --- configs/amd-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d98e851c9d..ec659450bc 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -636,8 +636,8 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.65 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1], spec-decoding: mtp } - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [72], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4, 8], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [16, 32, 64, 70, 72, 74, 80], spec-decoding: mtp } # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in From 7343641b3de6402ce11de1ed29a8d96b3504b66f Mon Sep 17 00:00:00 2001 From: Sirra Date: Sun, 13 Sep 2026 07:10:01 +0530 Subject: [PATCH 07/10] [AMD] [AGENTX] KIMI Perf Tuning Signed-off-by: Sirra --- configs/amd-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ec659450bc..3f86ddd8e2 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -636,8 +636,8 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.65 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4, 8], spec-decoding: mtp } - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [16, 32, 64, 70, 72, 74, 80], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 16, 32, 64, 70, 72, 74, 80], spec-decoding: mtp } # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in From 51ce68383e559d1c6e648acce8f654a42f195caa Mon Sep 17 00:00:00 2001 From: Sirra Date: Sun, 13 Sep 2026 07:55:39 +0530 Subject: [PATCH 08/10] [AMD] [AGENTX] KIMI Perf Tuning Signed-off-by: Sirra --- benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index b0ce95bcdb..60ccd3aa33 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -96,11 +96,13 @@ case "$CONC" in SPEC_ROWS=$(( SPEC_NUM_TOKENS + 1 )) KDA_ARGS=(--additional-config '{"kda_prefill_backend":"triton"}') MAX_NUM_SEQS="${MAX_NUM_SEQS:-$(( CONC > 2 ? CONC : 2 ))}" - MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-16384}" + if [ "$CONC" -eq 1 ]; then MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-16384}" + else MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"; fi ;; *) DCP_SIZE="${DCP_SIZE:-8}" - MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-24576}" + if [ "$CONC" -gt 64 ]; then MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-24576}" + else MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"; fi if [ "$CONC" -lt 72 ]; then MAX_NUM_SEQS="${MAX_NUM_SEQS:-$(( CONC * 14 / 10 ))}" elif [ "$CONC" -eq 72 ]; then MAX_NUM_SEQS="${MAX_NUM_SEQS:-96}" else MAX_NUM_SEQS="${MAX_NUM_SEQS:-112}"; fi From 11bb0dce7015d1770d76b6c5ff43353612dba1fe Mon Sep 17 00:00:00 2001 From: Sirra Date: Sun, 13 Sep 2026 07:57:30 +0530 Subject: [PATCH 09/10] [AMD] [AGENTX] KIMI Perf Tuning Signed-off-by: Sirra --- configs/amd-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 3f86ddd8e2..b98865c812 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -636,8 +636,8 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.65 search-space: - - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1], spec-decoding: mtp } - - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [8, 16, 32, 64, 70, 72, 74, 80], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 1, kv-offloading: none, conc-list: [1, 4], spec-decoding: mtp } + - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [12, 16, 32, 64, 70, 72, 74, 80], spec-decoding: mtp } # Kimi-K3 MXFP4 agentic-coding benchmark on MI355X via ATOM with DSpark # speculative decoding. Acceptance is pinned to the committed golden curve in From d35988f737c4ce0b9c3a58e175d6dabacc422f57 Mon Sep 17 00:00:00 2001 From: Sirra Date: Sun, 13 Sep 2026 08:09:04 +0530 Subject: [PATCH 10/10] [AMD] [AGENTX] KIMI Perf Tuning Signed-off-by: Sirra --- benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh index 60ccd3aa33..51293867b5 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh @@ -70,9 +70,10 @@ SPEC_ARGS=() SPEC_ROWS=1 KDA_ARGS=() case "$CONC" in - 1|2|4|8) + 1|2|4|8|10|12|14) DCP_SIZE="${DCP_SIZE:-1}" - SPEC_NUM_TOKENS="${SPEC_NUM_TOKENS:-4}" + if [ "$CONC" -eq 1 ]; then SPEC_NUM_TOKENS="${SPEC_NUM_TOKENS:-4}" + else SPEC_NUM_TOKENS="${SPEC_NUM_TOKENS:-3}"; fi case "$SPEC_NUM_TOKENS" in 1) SYNTHETIC_ACCEPT_LEN=1.85 ;; 2) SYNTHETIC_ACCEPT_LEN=2.51 ;;