From 30e37632ffa69db4ca628eea340c10195e1562b1 Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Mon, 12 Oct 2026 10:46:47 -0700 Subject: [PATCH 1/3] =?UTF-8?q?[NV]=20DeepSeek-V4.1-Flash=20FP4=20GB300=20?= =?UTF-8?q?SGLang=20AgentX:=20knob-tuned=20curve=20(stock=2020261006=20nig?= =?UTF-8?q?htly)=20/=20DeepSeek-V4.1-Flash=20FP4=20GB300=20SGLang=20AgentX?= =?UTF-8?q?=20=E5=8F=82=E6=95=B0=E8=B0=83=E4=BC=98=E6=9B=B2=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: hshrivastava-droid --- .../sglang/gb300-fp4-mtp/agentic-tuned.yaml | 495 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 17 + inferencex-e2e/perf-changelog.yaml | 9 + 3 files changed, 521 insertions(+) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml new file mode 100644 index 0000000000..53c791af41 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml @@ -0,0 +1,495 @@ +# DeepSeek-V4.1-Flash AgentX on GB300 with SGLang native DSpark: tuned variants per concurrency. +base: + schema: 2 + name: dsv41flash-fp4-gb300-sglang-agentic-tuned + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: lmsysorg/sglang:nightly-dev-cu13-20261006-f3fe7534@sha256:690d95001a04c7ae56cd97270c1c0d6ba3c498b0e6423ac7520472cefcf183a2 + precision: fp4 + resources: + gpu_type: gb300 + gpus_per_node: 4 + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + speculative-algorithm: DSPARK + speculative-dspark-block-size: 5 + reasoning-parser: auto + tool-call-parser: auto + watchdog-timeout: 3600 + enable-metrics: true + chunked-prefill-size: 16384 + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV41_REASONING_EFFORT: high + SGLANG_DSPARK_OPT_MARKOV_W2_BF16: 'True' + SGLANG_DSPARK_FOLDED_SAMPLING: '2' + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +override_tp2_c1: + roles: + agg: + args: + prefill-decode-interval: 16 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 2 + expert-parallel-size: 2 + max-running-requests: 2 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + gpus: 2 + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c2: + roles: + agg: + args: + prefill-decode-interval: 16 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 128 + max-running-requests: 4 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + gpus: 2 + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c4: + roles: + agg: + args: + prefill-decode-interval: 96 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 2 + expert-parallel-size: 1 + swa-prefix-tails: 256 + max-running-requests: 8 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_NUMA_BIND_V2: '0' + gpus: 2 + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c8: + roles: + agg: + args: + prefill-decode-interval: 96 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 512 + max-running-requests: 16 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_NUMA_BIND_V2: '0' + gpus: 2 + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c16: + roles: + agg: + args: + prefill-decode-interval: 96 + tokenizer-worker-num: 4 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 1024 + max-running-requests: 32 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_NUMA_BIND_V2: '0' + gpus: 2 + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c20: + roles: + agg: + args: + prefill-decode-interval: 96 + tokenizer-worker-num: 4 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 1280 + max-running-requests: 32 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_NUMA_BIND_V2: '0' + gpus: 2 + benchmark: + env: + CONC: '20' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp2_c24: + roles: + agg: + args: + prefill-decode-interval: 64 + tokenizer-worker-num: 4 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 1536 + max-running-requests: 64 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_NUMA_BIND_V2: '0' + gpus: 2 + benchmark: + env: + CONC: '24' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c32: + roles: + agg: + args: + prefill-decode-interval: 48 + tokenizer-worker-num: 4 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 2048 + max-running-requests: 64 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_NUMA_BIND_V2: '0' + gpus: 2 + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c40: + roles: + agg: + args: + prefill-decode-interval: 24 + tokenizer-worker-num: 4 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 2560 + max-running-requests: 64 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + gpus: 2 + benchmark: + env: + CONC: '40' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c56: + roles: + agg: + args: + prefill-decode-interval: 32 + tokenizer-worker-num: 4 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 3584 + max-running-requests: 64 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + gpus: 2 + benchmark: + env: + CONC: '56' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c64: + roles: + agg: + args: + prefill-decode-interval: 24 + schedule-policy: shortest-prefill-first + tokenizer-worker-num: 4 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 4096 + max-running-requests: 64 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + gpus: 2 + benchmark: + env: + CONC: '64' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c80: + roles: + agg: + args: + prefill-decode-interval: 24 + schedule-policy: shortest-prefill-first + tokenizer-worker-num: 4 + mem-fraction-static: 0.84 + cuda-graph-max-bs-decode: 80 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 4096 + max-running-requests: 80 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_NUMA_BIND_V2: '0' + gpus: 2 + benchmark: + env: + CONC: '80' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c96: + roles: + agg: + args: + prefill-decode-interval: 24 + schedule-policy: shortest-prefill-first + tokenizer-worker-num: 4 + mem-fraction-static: 0.84 + cuda-graph-max-bs-decode: 96 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 4096 + max-running-requests: 96 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_NUMA_BIND_V2: '0' + gpus: 2 + benchmark: + env: + CONC: '96' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp2_c112: + roles: + agg: + args: + prefill-decode-interval: 16 + schedule-policy: shortest-prefill-first + tokenizer-worker-num: 4 + mem-fraction-static: 0.82 + cuda-graph-max-bs-decode: 112 + tensor-parallel-size: 2 + expert-parallel-size: 2 + swa-prefix-tails: 4096 + max-running-requests: 112 + enable-hierarchical-cache: true + hicache-ratio: 1.0 + hicache-write-policy: write_back + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_NUMA_BIND_V2: '0' + gpus: 2 + benchmark: + env: + CONC: '112' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '360' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp4_c1: + roles: + agg: + args: + prefill-decode-interval: 16 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 4 + expert-parallel-size: 1 + max-running-requests: 2 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + SGLANG_RAGGED_VERIFY_MODE: static + gpus: 4 + benchmark: + env: + CONC: '1' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c2: + roles: + agg: + args: + prefill-decode-interval: 16 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 64 + tensor-parallel-size: 4 + expert-parallel-size: 1 + swa-prefix-tails: 128 + max-running-requests: 4 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + SGLANG_RAGGED_VERIFY_MODE: static + gpus: 4 + benchmark: + env: + CONC: '2' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c4: + roles: + agg: + args: + prefill-decode-interval: 0 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 128 + tensor-parallel-size: 4 + expert-parallel-size: 1 + swa-prefix-tails: 4096 + max-running-requests: 128 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_RAGGED_VERIFY_MODE: static + gpus: 4 + benchmark: + env: + CONC: '4' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c8: + roles: + agg: + args: + prefill-decode-interval: 0 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 128 + tensor-parallel-size: 4 + expert-parallel-size: 1 + swa-prefix-tails: 4096 + max-running-requests: 128 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_RAGGED_VERIFY_MODE: static + gpus: 4 + benchmark: + env: + CONC: '8' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_tp4_c16: + roles: + agg: + args: + prefill-decode-interval: 192 + tokenizer-worker-num: 4 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 128 + tensor-parallel-size: 4 + expert-parallel-size: 1 + swa-prefix-tails: 4096 + max-running-requests: 128 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_RAGGED_VERIFY_MODE: static + SGLANG_NUMA_BIND_V2: '0' + gpus: 4 + benchmark: + env: + CONC: '16' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index c32f89b47b..aa938fbaf1 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9240,6 +9240,23 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } +# Tuned variant of dsv41flash-fp4-gb300-sglang-agentic-dspark. +dsv41flash-fp4-gb300-sglang-agentic-dspark-tuned: + image: lmsysorg/sglang:nightly-dev-cu13-20261006-f3fe7534@sha256:690d95001a04c7ae56cd97270c1c0d6ba3c498b0e6423ac7520472cefcf183a2 + model: deepseek-ai/DeepSeek-V4.1-Flash + model-prefix: dsv41flash + runner: cluster:gb300-nv + precision: fp4 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 8, 16, 20, 24, 32, 40, 56, 64, 80, 96], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml } + - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [4], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml } + - { tp: 2, ep: 2, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [112], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml } dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-20260928-81f27fb3@sha256:d9e4917808cfaa4b3be033a0c85a3a73d71eb93c17accbfa2ac3e96f551f3a33 model: deepseek-ai/DeepSeek-V4.1-Flash diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 058145a6d1..8b57e2c914 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9611,3 +9611,12 @@ - "Concurrency 1 (dcp1 and dcp8) adds quantization-config linear fp8_per_channel: vLLM quantizes the target's linear layers to FP8 W8A8 at load, except kv_b_proj, in_proj_qkvgfab, f_b_proj, fused_qkv_a_proj, self_attn.g_proj and layer 0 mlp.down_proj, which stay BF16. The routed experts stay MXFP4." - "No change to the Inferact/Kimi-K3-DSpark draft's precision: its model config takes the speculative config's own quantization (unset) and never receives quantization-config, and get_draft_quant_config() returns None for K3DSparkModel, so its weights stay BF16 and its KV cache stays fp8." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3833 + +- config-keys: + - dsv41flash-fp4-gb300-sglang-agentic-dspark-tuned + scenario-type: + - agentic-coding + description: + - "Add a tuned DeepSeek-V4.1-Flash FP4 GB300 SGLang AgentX config on lmsysorg/sglang:nightly-dev-cu13-20261006-f3fe7534 with per-concurrency server settings for TP2 and TP4." + - "Enable folded DSpark draft sampling and add scheduler, tokenizer and expert-parallelism tuning, plus a HiCache DRAM tier at concurrency 112." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD From 2702de1ff7a2fca571cc74e0795904b1914bc06b Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Mon, 12 Oct 2026 10:48:52 -0700 Subject: [PATCH 2/3] Add PR link to perf changelog Co-Authored-By: Claude Sonnet 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 8b57e2c914..ab694b4023 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9619,4 +9619,4 @@ description: - "Add a tuned DeepSeek-V4.1-Flash FP4 GB300 SGLang AgentX config on lmsysorg/sglang:nightly-dev-cu13-20261006-f3fe7534 with per-concurrency server settings for TP2 and TP4." - "Enable folded DSpark draft sampling and add scheduler, tokenizer and expert-parallelism tuning, plus a HiCache DRAM tier at concurrency 112." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3929 From 6517a81b093ade5a67b05a91d1390c81a989990c Mon Sep 17 00:00:00 2001 From: hshrivastava-droid Date: Mon, 12 Oct 2026 15:12:46 -0700 Subject: [PATCH 3/3] =?UTF-8?q?Update=20GB300=20DeepSeek-V4.1-Flash=20SGLa?= =?UTF-8?q?ng=20AgentX=20tuned=20config=20/=20=E6=9B=B4=E6=96=B0=20GB300?= =?UTF-8?q?=20DeepSeek-V4.1-Flash=20SGLang=20AgentX=20=E8=B0=83=E4=BC=98?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- .../sglang/gb300-fp4-mtp/agentic-tuned.yaml | 360 +++++++----------- inferencex-e2e/configs/nvidia-master.yaml | 12 +- inferencex-e2e/perf-changelog.yaml | 5 +- 3 files changed, 149 insertions(+), 228 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml index 53c791af41..8710ec9ff9 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml @@ -1,16 +1,30 @@ -# DeepSeek-V4.1-Flash AgentX on GB300 with SGLang native DSpark: tuned variants per concurrency. +# DeepSeek-V4.1-Flash AgentX on GB300 with SGLang native DSpark: knob-tuned +# curve on the official nightly-dev-cu13-20261012-e6072935 image. Stock SGLang, +# no framework patches: every change is a server flag, environment variable, +# topology or the image. Twelve points, TP2 C8-C128 (DEP2 at C64/C128) and +# TP4 C1-C16, each the exact configuration measured on this image in one full +# 3600 s AgentX run (results in each variant's comment: p90 tok/s/user / +# tok/s/GPU / TTFT p90, and the published GB300 point at the same TP and +# concurrency where one exists). +# +# Knobs against the published dsv41flash-fp4-gb300-sglang-agentic-dspark +# curve: SGLANG_DSPARK_FOLDED_SAMPLING=2, 16K prefill chunks, a longer +# prefill-decode interval, tokenizer-worker-num 4, shortest-prefill-first +# scheduling at C64+, decoder SWA bounded replay at TP2, SGLANG_NUMA_BIND_V2=0 +# (Engram host-table huge pages), expert parallelism 1 at TP4, caps sized to the +# concurrency; DEP2 at C64/C128 (data-parallel attention, MegaMoE all-to-all, +# cache-aware DP-aware sglang-router, as in InferenceX #3859). base: schema: 2 name: dsv41flash-fp4-gb300-sglang-agentic-tuned model: path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: lmsysorg/sglang:nightly-dev-cu13-20261006-f3fe7534@sha256:690d95001a04c7ae56cd97270c1c0d6ba3c498b0e6423ac7520472cefcf183a2 + container: lmsysorg/sglang:nightly-dev-cu13-20261012-e6072935@sha256:b5c488694d74c13587af6057b90220498c4869571c8e1e3c43dd6f43ae721306 precision: fp4 resources: gpu_type: gb300 gpus_per_node: 4 frontend: - type: sglang enable_multiple_frontends: false observability: enabled: false @@ -50,67 +64,10 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' -override_tp2_c1: - roles: - agg: - args: - prefill-decode-interval: 16 - mem-fraction-static: 0.8 - cuda-graph-max-bs-decode: 64 - tensor-parallel-size: 2 - expert-parallel-size: 2 - max-running-requests: 2 - env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - gpus: 2 - benchmark: - env: - CONC: '1' - AGENTIC_WARMUP_GRACE_PERIOD: '1800' - -override_tp2_c2: - roles: - agg: - args: - prefill-decode-interval: 16 - mem-fraction-static: 0.8 - cuda-graph-max-bs-decode: 64 - tensor-parallel-size: 2 - expert-parallel-size: 2 - swa-prefix-tails: 128 - max-running-requests: 4 - env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - gpus: 2 - benchmark: - env: - CONC: '2' - AGENTIC_WARMUP_GRACE_PERIOD: '1800' - -override_tp2_c4: - roles: - agg: - args: - prefill-decode-interval: 96 - mem-fraction-static: 0.8 - cuda-graph-max-bs-decode: 64 - tensor-parallel-size: 2 - expert-parallel-size: 1 - swa-prefix-tails: 256 - max-running-requests: 8 - env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - SGLANG_NUMA_BIND_V2: '0' - gpus: 2 - benchmark: - env: - CONC: '4' - AGENTIC_WARMUP_GRACE_PERIOD: '1800' - +# c8-pdi96-numa0-i1012: 317.3 / 30,527 / 1.5 s; published 2026-10-09 227.3 / 29,970 / 1.5 s override_tp2_c8: + frontend: + type: sglang roles: agg: args: @@ -134,7 +91,10 @@ override_tp2_c8: CONC: '8' AGENTIC_WARMUP_GRACE_PERIOD: '1800' +# c16-pdi96-tok4-numa0-i1012-r2: 276.8 / 55,686 / 2.2 s; published 2026-10-09 149.9 / 53,617 / 3.5 s override_tp2_c16: + frontend: + type: sglang roles: agg: args: @@ -159,57 +119,10 @@ override_tp2_c16: CONC: '16' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_tp2_c20: - roles: - agg: - args: - prefill-decode-interval: 96 - tokenizer-worker-num: 4 - mem-fraction-static: 0.8 - cuda-graph-max-bs-decode: 64 - tensor-parallel-size: 2 - expert-parallel-size: 2 - swa-prefix-tails: 1280 - max-running-requests: 32 - enable-decoder-swa-bounded-replay: true - cuda-graph-backend-prefill: disabled - env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_NUMA_BIND_V2: '0' - gpus: 2 - benchmark: - env: - CONC: '20' - AGENTIC_WARMUP_GRACE_PERIOD: '1800' - -override_tp2_c24: - roles: - agg: - args: - prefill-decode-interval: 64 - tokenizer-worker-num: 4 - mem-fraction-static: 0.8 - cuda-graph-max-bs-decode: 64 - tensor-parallel-size: 2 - expert-parallel-size: 2 - swa-prefix-tails: 1536 - max-running-requests: 64 - enable-decoder-swa-bounded-replay: true - cuda-graph-backend-prefill: disabled - env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_NUMA_BIND_V2: '0' - gpus: 2 - benchmark: - env: - CONC: '24' - AGENTIC_WARMUP_GRACE_PERIOD: '3600' - +# c32-pdi48-tok4-numa0-i1012: 186.3 / 114,929 / 2.8 s; published 2026-10-09 112.5 / 94,503 / 10.3 s override_tp2_c32: + frontend: + type: sglang roles: agg: args: @@ -234,106 +147,53 @@ override_tp2_c32: CONC: '32' AGENTIC_WARMUP_GRACE_PERIOD: '3600' -override_tp2_c40: - roles: - agg: - args: - prefill-decode-interval: 24 - tokenizer-worker-num: 4 - mem-fraction-static: 0.8 - cuda-graph-max-bs-decode: 64 - tensor-parallel-size: 2 - expert-parallel-size: 2 - swa-prefix-tails: 2560 - max-running-requests: 64 - enable-decoder-swa-bounded-replay: true - cuda-graph-backend-prefill: disabled - env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - gpus: 2 - benchmark: - env: - CONC: '40' - AGENTIC_WARMUP_GRACE_PERIOD: '3600' - -override_tp2_c56: - roles: - agg: - args: - prefill-decode-interval: 32 - tokenizer-worker-num: 4 - mem-fraction-static: 0.8 - cuda-graph-max-bs-decode: 64 - tensor-parallel-size: 2 - expert-parallel-size: 2 - swa-prefix-tails: 3584 - max-running-requests: 64 - enable-decoder-swa-bounded-replay: true - cuda-graph-backend-prefill: disabled - env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - gpus: 2 - benchmark: - env: - CONC: '56' - AGENTIC_WARMUP_GRACE_PERIOD: '3600' - -override_tp2_c64: +# c64-dep2-megamoe-router-i1012: 116.1 / 203,195 / 2.3 s; no published point +override_dep2_c64: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true roles: agg: args: prefill-decode-interval: 24 - schedule-policy: shortest-prefill-first - tokenizer-worker-num: 4 mem-fraction-static: 0.8 cuda-graph-max-bs-decode: 64 tensor-parallel-size: 2 + data-parallel-size: 2 expert-parallel-size: 2 - swa-prefix-tails: 4096 - max-running-requests: 64 - enable-decoder-swa-bounded-replay: true - cuda-graph-backend-prefill: disabled - env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - gpus: 2 - benchmark: - env: - CONC: '64' - AGENTIC_WARMUP_GRACE_PERIOD: '3600' - -override_tp2_c80: - roles: - agg: - args: - prefill-decode-interval: 24 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + load-balance-method: total_requests + moe-a2a-backend: megamoe schedule-policy: shortest-prefill-first tokenizer-worker-num: 4 - mem-fraction-static: 0.84 - cuda-graph-max-bs-decode: 80 - tensor-parallel-size: 2 - expert-parallel-size: 2 swa-prefix-tails: 4096 - max-running-requests: 80 - enable-decoder-swa-bounded-replay: true - cuda-graph-backend-prefill: disabled + max-running-requests: 128 env: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_RAGGED_VERIFY_MODE: static SGLANG_NUMA_BIND_V2: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True gpus: 2 benchmark: env: - CONC: '80' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' AGENTIC_WARMUP_GRACE_PERIOD: '3600' + CONC: '64' +# c96-pdi24-spf-tok4-numa0-i1012: 104.1 / 232,789 / 2.5 s; no published point override_tp2_c96: + frontend: + type: sglang roles: agg: args: @@ -359,38 +219,53 @@ override_tp2_c96: CONC: '96' AGENTIC_WARMUP_GRACE_PERIOD: '3600' -override_tp2_c112: +# c128-dep2-megamoe-router-i1012: 84.2 / 278,104 / 3.6 s; no published point +override_dep2_c128: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true roles: agg: args: prefill-decode-interval: 16 - schedule-policy: shortest-prefill-first - tokenizer-worker-num: 4 - mem-fraction-static: 0.82 - cuda-graph-max-bs-decode: 112 + mem-fraction-static: 0.85 + cuda-graph-max-bs-decode: 128 tensor-parallel-size: 2 + data-parallel-size: 2 expert-parallel-size: 2 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + load-balance-method: total_requests + moe-a2a-backend: megamoe + schedule-policy: shortest-prefill-first + tokenizer-worker-num: 4 swa-prefix-tails: 4096 - max-running-requests: 112 - enable-hierarchical-cache: true - hicache-ratio: 1.0 - hicache-write-policy: write_back - enable-decoder-swa-bounded-replay: true - cuda-graph-backend-prefill: disabled + max-running-requests: 256 env: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_RAGGED_VERIFY_MODE: static SGLANG_NUMA_BIND_V2: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True gpus: 2 benchmark: env: - CONC: '112' - KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '360' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' AGENTIC_WARMUP_GRACE_PERIOD: '3600' + CONC: '128' +# c1-tp4-fold2-i1012: 624.9 / 7,358 / 1.3 s; published 2026-10-09 536.0 / 6,840 / 1.8 s override_tp4_c1: + frontend: + type: sglang roles: agg: args: @@ -409,31 +284,41 @@ override_tp4_c1: CONC: '1' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_tp4_c2: +# c5-tp4-stock-i1012: 527.3 / 10,470 / 1.4 s; no published point +override_tp4_c5: + frontend: + type: sglang roles: agg: args: - prefill-decode-interval: 16 + prefill-decode-interval: 192 + tokenizer-worker-num: 4 mem-fraction-static: 0.8 - cuda-graph-max-bs-decode: 64 + cuda-graph-max-bs-decode: 128 tensor-parallel-size: 4 expert-parallel-size: 1 - swa-prefix-tails: 128 - max-running-requests: 4 + swa-prefix-tails: 4096 + max-running-requests: 128 env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank SGLANG_RAGGED_VERIFY_MODE: static + SGLANG_NUMA_BIND_V2: '0' gpus: 4 benchmark: env: - CONC: '2' + CONC: '5' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_tp4_c4: +# c6-tp4-stock-i1012: 514.8 / 12,359 / 1.7 s; no published point +override_tp4_c6: + frontend: + type: sglang roles: agg: args: - prefill-decode-interval: 0 + prefill-decode-interval: 192 + tokenizer-worker-num: 4 mem-fraction-static: 0.8 cuda-graph-max-bs-decode: 128 tensor-parallel-size: 4 @@ -444,17 +329,22 @@ override_tp4_c4: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank SGLANG_RAGGED_VERIFY_MODE: static + SGLANG_NUMA_BIND_V2: '0' gpus: 4 benchmark: env: - CONC: '4' + CONC: '6' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_tp4_c8: +# c13-tp4-stock-i1012: 467.4 / 19,814 / 1.8 s; no published point +override_tp4_c13: + frontend: + type: sglang roles: agg: args: - prefill-decode-interval: 0 + prefill-decode-interval: 192 + tokenizer-worker-num: 4 mem-fraction-static: 0.8 cuda-graph-max-bs-decode: 128 tensor-parallel-size: 4 @@ -465,13 +355,43 @@ override_tp4_c8: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank SGLANG_RAGGED_VERIFY_MODE: static + SGLANG_NUMA_BIND_V2: '0' gpus: 4 benchmark: env: - CONC: '8' + CONC: '13' AGENTIC_WARMUP_GRACE_PERIOD: '1800' +# c14-tp4-stock-i1012: 453.8 / 22,381 / 1.9 s; no published point +override_tp4_c14: + frontend: + type: sglang + roles: + agg: + args: + prefill-decode-interval: 192 + tokenizer-worker-num: 4 + mem-fraction-static: 0.8 + cuda-graph-max-bs-decode: 128 + tensor-parallel-size: 4 + expert-parallel-size: 1 + swa-prefix-tails: 4096 + max-running-requests: 128 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '1' + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_RAGGED_VERIFY_MODE: static + SGLANG_NUMA_BIND_V2: '0' + gpus: 4 + benchmark: + env: + CONC: '14' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +# c16-tp4-stock-i1012: 436.0 / 28,997 / 2.6 s; published 2026-10-09 222.2 / 28,694 / 1.6 s override_tp4_c16: + frontend: + type: sglang roles: agg: args: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index aa938fbaf1..338c987c84 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9240,9 +9240,10 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } -# Tuned variant of dsv41flash-fp4-gb300-sglang-agentic-dspark. +# Knob-tuned sibling of dsv41flash-fp4-gb300-sglang-agentic-dspark on the official +# 20261012 SGLang nightly: stock SGLang, no framework patches. C64/C128 run DEP2 behind sglang-router. dsv41flash-fp4-gb300-sglang-agentic-dspark-tuned: - image: lmsysorg/sglang:nightly-dev-cu13-20261006-f3fe7534@sha256:690d95001a04c7ae56cd97270c1c0d6ba3c498b0e6423ac7520472cefcf183a2 + image: lmsysorg/sglang:nightly-dev-cu13-20261012-e6072935@sha256:b5c488694d74c13587af6057b90220498c4869571c8e1e3c43dd6f43ae721306 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv @@ -9253,10 +9254,9 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark-tuned: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 8, 16, 20, 24, 32, 40, 56, 64, 80, 96], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml } - - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [4], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml } - - { tp: 2, ep: 2, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [112], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml } - - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml } + - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 32, 96], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml } + - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 5, 6, 13, 14, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic-tuned.yaml } dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-20260928-81f27fb3@sha256:d9e4917808cfaa4b3be033a0c85a3a73d71eb93c17accbfa2ac3e96f551f3a33 model: deepseek-ai/DeepSeek-V4.1-Flash diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index ab694b4023..5a278f4be1 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9617,6 +9617,7 @@ scenario-type: - agentic-coding description: - - "Add a tuned DeepSeek-V4.1-Flash FP4 GB300 SGLang AgentX config on lmsysorg/sglang:nightly-dev-cu13-20261006-f3fe7534 with per-concurrency server settings for TP2 and TP4." - - "Enable folded DSpark draft sampling and add scheduler, tokenizer and expert-parallelism tuning, plus a HiCache DRAM tier at concurrency 112." + - "Add a knob-tuned sibling of the GB300 DeepSeek-V4.1-Flash SGLang AgentX curve on the official lmsysorg/sglang:nightly-dev-cu13-20261012-e6072935 (digest pinned, the same image as InferenceX #3859), stock SGLang without framework patches: twelve points measured on that image, TP2/EP2 at C8, C16, C32 and C96, DEP2 (TP2/DP2/EP2 with DP attention) at C64 and C128, and TP4/EP1 at C1, C5, C6, C13, C14 and C16; one variant per point in agentic-tuned.yaml, each the exact configuration measured in a full 3600 s run." + - "Every point: SGLANG_DSPARK_FOLDED_SAMPLING=2 and 16K prefill chunks. TP2 C8-C96: decoder SWA bounded replay (prefill graph backend disabled), expandable allocator segments, SGLANG_NUMA_BIND_V2=0 and a prefill-decode interval of 96 at C8/C16, 48 at C32 and 24 at C96; tokenizer-worker-num 4 from C16; at C96 shortest-prefill-first, request and decode-graph caps of 96 and mem-fraction-static 0.84. DEP2 C64/C128, as in InferenceX #3859: DP attention with DP LM head and local control broadcast, MegaMoE all-to-all, total_requests load balancing, shortest-prefill-first, tokenizer-worker-num 4, interval 24/16 and request caps 128/256, behind sglang-router (cache_aware, dp-aware, routing key from the AIPerf correlation id). TP4 C5-C16: EP1, interval 192, tokenizer-worker-num 4, caps 128, static ragged verify and SGLANG_NUMA_BIND_V2=0; TP4 C1: the published recipe with folded sampling and 16K chunks." + - "Canonical DSpark block 5 and golden acceptance 3.51 are unchanged. GSM8K passed on folded sampling (TP2 0.975, TP4 0.973), on bounded replay (0.977), on the DEP2 router config on this image (0.9742) and on stock TP4 on this image (0.9735)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3929