diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh index e9e3d5eb5..9a09392e9 100644 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -14,7 +14,7 @@ check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} +SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-60} if [[ -n "${SLURM_JOB_ID:-}" ]]; then echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" @@ -90,7 +90,7 @@ export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 export SGLANG_MAMBA_SSM_DTYPE=bfloat16 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT8 +export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 export SGLANG_TIMEOUT_KEEP_ALIVE=1800 if [ "${EVAL_ONLY:-false}" != "true" ]; then @@ -113,7 +113,7 @@ SGLANG_CMD=( --watchdog-timeout 1200 --page-size 16 --kv-cache-dtype fp8_e4m3 - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" --max-running-requests "$MAX_RUNNING_REQUESTS" --max-prefill-tokens 16384 --chunked-prefill-size 16384 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 26010d293..4958de065 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -361,7 +361,7 @@ qwen3.5-fp4-mi355x-sglang-mtp: - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } qwen3.5-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260908 + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260911 model: amd/Qwen3.5-397B-A17B-MXFP4 model-prefix: qwen3.5 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c69451c2e..b65157b98 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7561,3 +7561,13 @@ - "Replace the local 2*CONC max_num_seqs limit with an explicit reproducibility cap of 128 (the effective vllm serve default on MI355X is 1024). The old cap sat below AgentX subagent fan-out and produced queue times up to 16.2 seconds at concurrency 1; 128 removed that scheduling queue. The derived CUDA-graph capture ceiling is 1024 for each configured concurrency." - "将本地 2*CONC 的 max_num_seqs 上限改为显式固定的可复现值 128(MI355X 上 vllm serve 的实际默认值为 1024)。旧上限低于 AgentX 子代理扇出,并发 1 时排队时间最高达 16.2 秒;128 消除了该调度排队。所有已配置并发点由此得到的 CUDA graph 捕获上限均为 1024。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3058 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260911." + - "Rename --cuda-graph-max-bs to --cuda-graph-max-bs-decode for sglang 0.5.19." + - "Prefill allreduce INT4; scheduler-recv-interval default 60." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2972