From 38dc8869c8f0940bdaded38d51f4d8c13e4a3cc0 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 27 Oct 2026 21:16:39 -0700 Subject: [PATCH 1/2] Store DSpark draft KV in FP8 on the GB300 SGLang AgentX curve Set speculative-draft-kv-cache-dtype fp8_e4m3, matching the vLLM DSpark drafter, so the bf16 draft KV no longer takes HBM and HiCache host pool space from target KV. The whole curve reruns. Co-Authored-By: Claude Opus 5.5 --- .../minimaxm31flash/sglang/gb300-fp4-mtp/agentic-tp2.yaml | 2 ++ .../minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml | 2 ++ inferencex-e2e/perf-changelog.yaml | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-tp2.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-tp2.yaml index ab1788f364..300703a0e5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-tp2.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic-tp2.yaml @@ -33,6 +33,8 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml index ebad930fc4..eec61fb43b 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm31flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -33,6 +33,8 @@ base: speculative-algorithm: DSPARK speculative-dspark-block-size: 7 speculative-use-block-verification: true + # FP8 draft KV, as vLLM's DSpark drafter uses; bf16 costs ~10 GB of HBM per rank. + speculative-draft-kv-cache-dtype: fp8_e4m3 cuda-graph-backend-prefill: breakable tokenizer-worker-num: 8 watchdog-timeout: 3600 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 907bd21213..91c1c0bc95 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11489,3 +11489,11 @@ - "Update the MI355X DeepSeek-V4-Pro-0813 SGLang MoRI disaggregated AgentX serving image from lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261019 to lmsysorg/sglang-rocm:v0.5.22-rocm10-mi35x-20261026, the latest published MI35x ROCm 10 nightly (digest sha256:edabad83728ab04e3eae39f315eb16261eeb64dbdb051112bd22fc88560b60c4)." - "Preserve all eight concurrency points (1, 16, 32, 48, 192, 256, 384, 512), prefill/decode topology, DSpark settings, UMBP offload, MoRI transfer, and benchmark client image. The workload binder supplies the serving image to every selected recipe variant." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4571 + +- config-keys: + - minimaxm31flash-fp4-gb300-sglang-agentic-dspark + description: + - "Store the DSpark draft KV cache in FP8 (speculative-draft-kv-cache-dtype fp8_e4m3) on every point except the batch-size-1 MonoKernel Low-Latency point, instead of the bf16 default, matching the vLLM DSpark drafter; this frees HBM and HiCache host pool space for target KV." + - "Rerun the whole curve so it is measured on one recipe revision." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + scenario-type: [agentic-coding] From 3b96f3383ade71707f456573909b65a84b0b74d1 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 27 Oct 2026 21:17:45 -0700 Subject: [PATCH 2/2] Set the changelog PR link Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 91c1c0bc95..4e4d88fd08 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11495,5 +11495,5 @@ description: - "Store the DSpark draft KV cache in FP8 (speculative-draft-kv-cache-dtype fp8_e4m3) on every point except the batch-size-1 MonoKernel Low-Latency point, instead of the bf16 default, matching the vLLM DSpark drafter; this frees HBM and HiCache host pool space for target KV." - "Rerun the whole curve so it is measured on one recipe revision." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4600 scenario-type: [agentic-coding]