perf: MI355X DSv4.1 Flash graph capture sized to concurrency, sweep extended to 128 / MI355X DSv4.1 Flash 按并发设定 graph capture 规模并将并发扫描扩展至 128

benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_vllm_mtp.sh

@@ -3,7 +3,8 @@ set -eo pipefail
 
 # DeepSeek-V4.1-Flash on MI355X: native DSpark and GPU-resident KV.
 # Follow upstream AMD defaults for Engram; storage behavior needs verification.
-# Image: vllm/vllm-openai-rocm:deepseekv41-flash-0909 (configured in amd-master.yaml); GPU validation is pending.
+# Image: vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657
+# (configured in amd-master.yaml); GPU validation is pending.
 # https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml
 source "$(dirname "$0")/../../benchmark_lib.sh"
 check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
@@ -23,6 +24,14 @@ if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then
 fi
 export VLLM_ROCM_USE_AITER=1
 export VLLM_ROCM_USE_AITER_MOE=1
+# AITER's Triton MoE GEMM warns on every call that Gluon is unavailable and it
+# is falling back to Triton. Gluon supports only gfx1250, so on gfx950 that is
+# a fixed property rather than a condition worth reporting, and it was 98% of
+# the lines in a gsm8k server log (411k of 417k, 30 MiB of 32 MiB). Every
+# warning aiter.ops.triton emits is about Gluon availability, so raising the
+# threshold loses nothing actionable here. Log hygiene only: the emits cost
+# 0.03% of wall time per worker, so this is not a throughput change.
+export AITER_TRITON_LOG_LEVEL=ERROR
 # DeepseekV41ForCausalLM is not torch-compiled upstream, so the default
 # cudagraph_mode=FULL_AND_PIECEWISE aborts at engine init with "piecewise CUDA
 # graphs unavailable" (run 34566727564). The model is built for the breakable
@@ -33,17 +42,35 @@ export OMP_NUM_THREADS=1
 export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126
 resolve_trace_source
 install_agentic_deps
+
+# Gate strictly (<=1%, ~2.9 GB on the 288 GB part) rather than the default 10%
+# (~28.8 GB). vLLM refuses to start unless free memory covers
+# gpu-memory-utilization * total, which is 259.19 of 287.98 GiB at 0.9, so the
+# default gate leaves nothing to spare: six of the eight points in run
+# 34736531646 aborted in init_device with 31-43 GiB of a prior job still
+# resident on the device. vLLM then sizes the KV pool from device-wide free
+# memory, so residual admitted here also moves the pool between otherwise
+# identical reruns.
+wait_for_amd_gpu_clean 1
+
 mkdir -p "$RESULT_DIR"
 SERVER_LOG="$RESULT_DIR/server.log"
 export VLLM_ENGINE_READY_TIMEOUT_S=3600
 export VLLM_USE_RUST_FRONTEND=1
 export PYTHONUNBUFFERED=1
 
-# Match the sibling's scheduler headroom for AgentX subagent fan-out.
-MAX_NUM_SEQS=$((2 * CONC))
+# Upstream default. The previous 2*CONC cap sat below AgentX's subagent
+# fan-out, so at CONC=1 the engine admitted 2 requests and left the rest
+# queued on scheduling capacity.
+MAX_NUM_SEQS=128
 NUM_SPEC_TOKENS=5
+# Size graph capture to the point being measured: each of the CONC lanes submits
+# one real token plus NUM_SPEC_TOKENS drafts per step, so that product bounds
+# the decode batch. Deriving from MAX_NUM_SEQS instead pinned every point at
+# 1024, spending 38.1 GiB per GPU on shapes the workload cannot reach and
+# leaving a 33.6M-token KV pool.
 CAPTURE_SIZE=1
-while (( CAPTURE_SIZE < MAX_NUM_SEQS * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 )); do
+while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 )); do
     CAPTURE_SIZE=$((CAPTURE_SIZE * 2))
 done
 
@@ -70,7 +97,14 @@ VLLM_CMD=(
     --tokenizer-mode deepseek_v41
     --tool-call-parser deepseek_v41 --enable-auto-tool-choice
     --reasoning-parser deepseek_v41
-    --moe-backend aiter_triton_mxfp4_bf16
+    # aiter, not aiter_triton_mxfp4_bf16: the plain name opens vLLM's full
+    # priority list and the CK kernel at its head wins. Despite the BF16
+    # backend name and this checkpoint's activation_scheme=dynamic, CK
+    # quantizes activations to FP8 internally and dispatches the a8w4 experts
+    # (mfma_moe1_silu_mul_afp8_wfp4_bf16 / mfma_moe2_afp8_wfp4_bf16) that the
+    # DSV4-Pro MI355X recipe already gets. Pinning the Triton name instead
+    # forced the W4A16 _moe_gemm_a16w4 kernel.
+    --moe-backend aiter
     --gpu-memory-utilization 0.9
     --speculative-config "$SPEC_CONFIG"
     --max-model-len 1048576

configs/amd-master.yaml

@@ -1753,9 +1753,11 @@ dsv4-fp4-mi355x-sglang-agentic-mtp:
       - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: mtp }
       - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128, 256], spec-decoding: mtp }
 
-# Official upstream ROCm image from the vLLM recipe; MI355X runtime validation is pending.
+# Upstream ROCm nightly rather than the deepseekv41-flash-0909 release tag: that
+# tag predates vllm-project/vllm#56503, which moves the mHC delayed pre block off
+# the eager Torch reference and onto AITER. MI355X runtime validation is pending.
 dsv41flash-fp4-mi355x-vllm-agentic-dspark:
-  image: vllm/vllm-openai-rocm:deepseekv41-flash-0909
+  image: vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657
   model: deepseek-ai/DeepSeek-V4.1-Flash
   model-prefix: dsv41flash
   runner: cluster:mi355x-amds
@@ -1767,4 +1769,4 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark:
     - dram-utilization: 0.60
       search-space:
       # Follow upstream AMD Engram defaults; omit the CUDA-only config flag.
-      - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] }
+      - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] }

docs/configuration-procedures.md

@@ -409,8 +409,8 @@ A configuration is ready for sweep only when the executable files agree, the exa
 
 ## DeepSeek-V4.1-Flash on MI355X
 
-The draft `dsv41flash-fp4-mi355x-vllm-agentic-dspark` recipe extends [#2958](https://github.com/SemiAnalysisAI/InferenceX/pull/2958) to MI355X AgentX: TP4, concurrency 1–32, native five-token DSpark. Throughput uses the [committed golden AL](../golden_al_distribution/dsv41flash_dspark.yaml) of 3.51 for thinking on and five draft tokens, with synthetic rejection sampling and adaptive verification disabled. Accuracy evals retain real block rejection but, unlike the CUDA arms, also keep adaptive verification disabled: it trims verification requests on device, which the ROCm `DeepseekV4IndexerBackend` does not support, and the engine refused to start with it enabled ([run 34651830283](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34651830283)). FP4 describes the MXFP4 experts; the checkpoint also contains MXFP8 weights.
+The draft `dsv41flash-fp4-mi355x-vllm-agentic-dspark` recipe extends [#2958](https://github.com/SemiAnalysisAI/InferenceX/pull/2958) to MI355X AgentX: TP4, concurrency 1–128, native five-token DSpark. CUDA graph capture is sized to the point being measured, `CONC * (1 + NUM_SPEC_TOKENS)` rounded up to a power of two. Throughput uses the [committed golden AL](../golden_al_distribution/dsv41flash_dspark.yaml) of 3.51 for thinking on and five draft tokens, with synthetic rejection sampling and adaptive verification disabled. Accuracy evals retain real block rejection but, unlike the CUDA arms, also keep adaptive verification disabled: it trims verification requests on device, which the ROCm `DeepseekV4IndexerBackend` does not support, and the engine refused to start with it enabled ([run 34651830283](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34651830283)). FP4 describes the MXFP4 experts; the checkpoint also contains MXFP8 weights.
 
-Follow the AMD overrides in [upstream recipe #946](https://github.com/vllm-project/recipes/pull/946): `VLLM_ROCM_USE_AITER=1`, `VLLM_ROCM_USE_AITER_MOE=1`, and `--moe-backend aiter_triton_mxfp4_bf16`. The recipe pins `semianalysis_cc_traces_weka_062126` (the unfiltered corpus) via `WEKA_LOADER_OVERRIDE`. KV stays GPU-resident; Engram follows upstream AMD defaults. Do not copy the NVIDIA `--engram-config` option: upstream currently rejects it on ROCm. The MI355X launcher uses the shared HF cache and mounts this model's repository at `/ix`, and exports `INFMAX_CONTAINER_WORKSPACE=/ix` so AgentX dependencies and outputs resolve inside that mount.
+Follow the AMD overrides in [upstream recipe #946](https://github.com/vllm-project/recipes/pull/946): `VLLM_ROCM_USE_AITER=1`, `VLLM_ROCM_USE_AITER_MOE=1`, and `--moe-backend aiter` (the upstream recipe names the Triton backend explicitly; `aiter` lets vLLM pick the CK a8w4 experts instead, matching the DSV4-Pro MI355X recipe). The recipe pins `semianalysis_cc_traces_weka_062126` (the unfiltered corpus) via `WEKA_LOADER_OVERRIDE`. KV stays GPU-resident; Engram follows upstream AMD defaults. Do not copy the NVIDIA `--engram-config` option: `EngramConfig.verify_model_config` gates on `current_platform.is_cuda()`, which is false on ROCm, so passing it raises `ValueError` and the engine refuses to start. Omitting it is not a downgrade. `VllmConfig._resolve_and_verify_engram_config` returns before validation when the flag is absent, and the Engram module then defaults to `cpu_offload=True`, so the tables still land in pinned host memory — 23.60 GiB per rank per layer, logged at `engram.py:690` on every MI355X run. The MI355X launcher uses the shared HF cache and mounts this model's repository at `/ix`, and exports `INFMAX_CONTAINER_WORKSPACE=/ix` so AgentX dependencies and outputs resolve inside that mount.
 
-**GPU validation is pending:** The recipe uses `vllm/vllm-openai-rocm:deepseekv41-flash-0909`, the image referenced by the [upstream vLLM recipe](https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash?hardware=mi355x). The tag was published to Docker Hub on 2026-09-11 after AMD verification; it returned HTTP 404 in run 34466680355 before publication. Engram behavior and serving compatibility still require GPU validation. Follow the [AgentX procedure](./eval-agentx-procedures.md#7-run-agentx-fast-feedback-versus-canonical-evidence) for runtime evidence; local generation and registry metadata are not GPU proof.
+**GPU validation is pending:** The recipe uses `vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657` (digest `sha256:960228cf…`, published 2026-09-12). The `deepseekv41-flash-0909` tag referenced by the [upstream vLLM recipe](https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash?hardware=mi355x) predates [vllm-project/vllm#56503](https://github.com/vllm-project/vllm/pull/56503), which moves the mHC delayed pre block off the eager Torch reference and onto AITER; that block is 85% of the decoder's kernel launches on this model, so the release tag leaves most of the decode cost on the table. The upstream recipe is being moved to the same nightly in [vllm-project/recipes#962](https://github.com/vllm-project/recipes/pull/962). Engram behavior and serving compatibility still require GPU validation. Follow the [AgentX procedure](./eval-agentx-procedures.md#7-run-agentx-fast-feedback-versus-canonical-evidence) for runtime evidence; local generation and registry metadata are not GPU proof.

docs/configuration-procedures_zh.md

@@ -386,8 +386,8 @@ python -m pytest utils/matrix_logic/ -v
 
 ## MI355X 上的 DeepSeek-V4.1-Flash
 
-草案配方 `dsv41flash-fp4-mi355x-vllm-agentic-dspark` 将 [#2958](https://github.com/SemiAnalysisAI/InferenceX/pull/2958) 扩展至 MI355X AgentX：TP4、并发 1–32、原生五 token DSpark。吞吐测试使用[已提交的黄金 AL](../golden_al_distribution/dsv41flash_dspark.yaml)：thinking 开启、五个草稿 token 对应 3.51，采用合成拒绝采样并关闭自适应验证。准确率 eval 保留真实块拒绝采样，但与 CUDA 分支不同，同样关闭自适应验证：它会在设备端裁剪验证请求，而 ROCm 的 `DeepseekV4IndexerBackend` 不支持该操作，启用后引擎拒绝启动（[运行 34651830283](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34651830283)）。FP4 表示 MXFP4 专家权重；检查点还包含 MXFP8 权重。
+草案配方 `dsv41flash-fp4-mi355x-vllm-agentic-dspark` 将 [#2958](https://github.com/SemiAnalysisAI/InferenceX/pull/2958) 扩展至 MI355X AgentX：TP4、并发 1–128、原生五 token DSpark。CUDA graph 捕获规模按被测点设定，即 `CONC * (1 + NUM_SPEC_TOKENS)` 向上取到 2 的幂。吞吐测试使用[已提交的黄金 AL](../golden_al_distribution/dsv41flash_dspark.yaml)：thinking 开启、五个草稿 token 对应 3.51，采用合成拒绝采样并关闭自适应验证。准确率 eval 保留真实块拒绝采样，但与 CUDA 分支不同，同样关闭自适应验证：它会在设备端裁剪验证请求，而 ROCm 的 `DeepseekV4IndexerBackend` 不支持该操作，启用后引擎拒绝启动（[运行 34651830283](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34651830283)）。FP4 表示 MXFP4 专家权重；检查点还包含 MXFP8 权重。
 
-遵循[上游配方 #946](https://github.com/vllm-project/recipes/pull/946) 的 AMD 设置：`VLLM_ROCM_USE_AITER=1`、`VLLM_ROCM_USE_AITER_MOE=1` 和 `--moe-backend aiter_triton_mxfp4_bf16`。配方通过 `WEKA_LOADER_OVERRIDE` 固定使用完整语料 `semianalysis_cc_traces_weka_062126`。KV 驻留 GPU；Engram 沿用上游 AMD 默认设置。不要复制 NVIDIA 的 `--engram-config` 选项：上游目前在 ROCm 上拒绝该选项。MI355X launcher 使用共享 HF 缓存，并将此模型的仓库挂载至 `/ix`，同时导出 `INFMAX_CONTAINER_WORKSPACE=/ix`，确保 AgentX 依赖与输出路径位于该挂载中。
+遵循[上游配方 #946](https://github.com/vllm-project/recipes/pull/946) 的 AMD 设置：`VLLM_ROCM_USE_AITER=1`、`VLLM_ROCM_USE_AITER_MOE=1` 和 `--moe-backend aiter`（上游配方显式指定 Triton 后端；使用 `aiter` 可让 vLLM 选择 CK a8w4 专家内核，与 DSV4-Pro MI355X 配方一致）。配方通过 `WEKA_LOADER_OVERRIDE` 固定使用完整语料 `semianalysis_cc_traces_weka_062126`。KV 驻留 GPU；Engram 沿用上游 AMD 默认设置。不要复制 NVIDIA 的 `--engram-config` 选项：`EngramConfig.verify_model_config` 以 `current_platform.is_cuda()` 作为判据，而 ROCm 上该判据为 false，因此传入该选项会抛出 `ValueError`，引擎拒绝启动。省略它并不会降级：缺少该选项时 `VllmConfig._resolve_and_verify_engram_config` 会在校验前返回，Engram 模块随后默认取 `cpu_offload=True`，表仍然放在锁页主机内存中——每 rank 每层 23.60 GiB，每次 MI355X 运行都会在 `engram.py:690` 记录。MI355X launcher 使用共享 HF 缓存，并将此模型的仓库挂载至 `/ix`，同时导出 `INFMAX_CONTAINER_WORKSPACE=/ix`，确保 AgentX 依赖与输出路径位于该挂载中。
 
-**GPU 验证尚待完成：** 配方使用 `vllm/vllm-openai-rocm:deepseekv41-flash-0909`，即[上游 vLLM 配方](https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash?hardware=mi355x)引用的镜像。该标签经 AMD 验证后于 2026-09-11 发布到 Docker Hub；发布前在运行 34466680355 中返回 HTTP 404。Engram 行为与服务兼容性仍需 GPU 验证。请按 [AgentX 流程](./eval-agentx-procedures_zh.md) 获取运行时证据；本地矩阵生成和镜像元数据不等于 GPU 验证。
+**GPU 验证尚待完成：** 配方使用 `vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657`（摘要 `sha256:960228cf…`，发布于 2026-09-12）。[上游 vLLM 配方](https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash?hardware=mi355x)引用的 `deepseekv41-flash-0909` 标签早于 [vllm-project/vllm#56503](https://github.com/vllm-project/vllm/pull/56503)，该 PR 将 mHC delayed pre 块从 eager Torch 参考实现切换到 AITER；该块占此模型解码器内核启动数的 85%，因此发布标签会浪费大部分解码开销。上游配方正在 [vllm-project/recipes#962](https://github.com/vllm-project/recipes/pull/962) 中切换到同一 nightly。Engram 行为与服务兼容性仍需 GPU 验证。请按 [AgentX 流程](./eval-agentx-procedures_zh.md) 获取运行时证据；本地矩阵生成和镜像元数据不等于 GPU 验证。