From 9cce59415e83945c0af9d36fb43d8ad03fcd67c3 Mon Sep 17 00:00:00 2001 From: "Po-Han Huang (NVIDIA)" <53919306+nvpohanh@users.noreply.github.com> Date: Fri, 16 Oct 2026 06:07:27 -0700 Subject: [PATCH 1/3] feat(agentx): GB300 DSV4.1-Flash Dynamo+SGLang TP4/TEP2 agg and DEP2/DEP16 disagg points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the GB300 DeepSeek-V4.1-Flash Dynamo + SGLang lane to the nightly-dev-cu13-20261015-a6a8c560 image and run the ai-dynamo 1.6.0.dev20261014 it ships. - Aggregated: replace TP4 c1 and TP2/EP2 c1 with TP4/EP1 c1, c4, c16 and TP2/EP2 c16, using the engine settings of the matching standalone SGLang points. The Dynamo frontend tokenizes with fastokens. - Disaggregated: add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg with 4x DEP2 prefill + 1x DEP16 decode at c1228 and 6x DEP2 prefill + 1x DEP16 decode at c2884, and remove the six TP2/EP2 1PxD points of dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg. 中文：将 GB300 DeepSeek-V4.1-Flash Dynamo + SGLang 配置迁移到 nightly-dev-cu13-20261015-a6a8c560 镜像，并直接使用镜像自带的 ai-dynamo 1.6.0.dev20261014。 - 聚合模式：用 TP4/EP1 c1、c4、c16 和 TP2/EP2 c16 取代原有的 TP4 c1 与 TP2/EP2 c1，引擎参数沿用对应的独立 SGLang 配置点；由 Dynamo 前端使用 fastokens 进行分词。 - 分离模式：新增 dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg， 包含 c1228（4 个 DEP2 prefill + 1 个 DEP16 decode）与 c2884（6 个 DEP2 prefill + 1 个 DEP16 decode）两个配置点，并移除 dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg 中六个 TP2/EP2 1PxD 配置点。 Co-Authored-By: Claude Opus 5.5 --- .../sglang/gb300-fp4/agentx/agg-variants.yaml | 127 +++++++---- .../gb300-fp4/agentx/disagg-variants.yaml | 202 +++++++++--------- inferencex-e2e/configs/nvidia-master.yaml | 113 ++++------ inferencex-e2e/perf-changelog.yaml | 13 ++ 4 files changed, 245 insertions(+), 210 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml index 4a43da219d..680fcbdfcd 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -1,14 +1,23 @@ - schema: 2 base: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg identity: model: {} + frameworks: + dynamo: "1.6.0.dev20261014" + sglang: "0.0.0.dev1+ga6a8c5601" + # The image ships ai-dynamo 1.6.0.dev20261014 (includes ai-dynamo/dynamo#16005). dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" + install: false + request_plane: tcp + environment: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + HF_DATASETS_CACHE: /hf_hub_cache/datasets + observability: + tachometer: + enabled: false slurm: time_limit: "4:00:00" # Cold weight loads from the shared HF cache plus graph capture take 15-20 min. @@ -18,7 +27,6 @@ base: resources: gpu_type: gb300 gpus_per_node: 4 - # Dynamo 1.5 discovers workers through etcd and also needs NATS. services: - name: etcd type: etcd @@ -28,15 +36,24 @@ base: type: nats placement: node: infra + - name: otel-collector + type: otel-collector + enabled: false frontend: type: dynamo enable_multiple_frontends: false env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" args: router-mode: kv - router-session-affinity-ttl-secs: "3600" + router-session-affinity-ttl-secs: "14400" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" @@ -45,81 +62,113 @@ base: agg: nodes: 1 workers: 1 - gpus: 2 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" PYTHONNOUSERSITE: "1" PYTHONUNBUFFERED: "1" - HF_HUB_CACHE: /hf_hub_cache - # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. SGLANG_TIMEOUT_KEEP_ALIVE: "900" - # AgentX measures thinking on, the regime of the golden acceptance curve. SGLANG_DEFAULT_THINKING: "1" SGLANG_DSV41_REASONING_EFFORT: high SGLANG_DSPARK_OPT_MARKOV_W2_BF16: "True" - # TP2 keeps the row-sharded Engram tables in host DRAM. - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + # AUTO leaves DSpark draft sampling eager for V4.1 TP shards, which leaves c1 + # decode launch-bound; fold it into the draft CUDA graph. + SGLANG_DSPARK_FOLDED_SAMPLING: "2" args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true - tensor-parallel-size: 2 - expert-parallel-size: 2 mem-fraction-static: 0.8 - # Keep active decode requests progressing while long prefixes are queued. - prefill-decode-interval: 16 + chunked-prefill-size: 16384 # Admission never exceeds the captured decode graph batch. cuda-graph-max-bs-decode: 64 - # DSpark is the checkpoint's bundled draft; the block size is its only knob. speculative-algorithm: DSPARK speculative-dspark-block-size: 5 reasoning-parser: auto tool-call-parser: auto - # Draft passes under long-context load outlast the 1800 s default watchdog. watchdog-timeout: 3600 enable-metrics: true - weight-loader-prefetch-checkpoints: true - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' sbatch_directives: mem: "0" exclusive: "" srun_options: container-remap-root: "" + # Engram host tables need the node's full memory in every worker step. + mem: "0" benchmark: env: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}' + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + +override_tp2_c16: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp2ep2-c16 + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + prefill-decode-interval: 96 + swa-prefix-tails: 1024 + max-running-requests: 32 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_NUMA_BIND_V2: "0" override_tp4_c1: - name: agg-gb300-tp4ep1-c1 + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c1 roles: agg: gpus: 4 args: tensor-parallel-size: 4 expert-parallel-size: 1 - chunked-prefill-size: 4096 - max-running-requests: 2 + prefill-decode-interval: 0 + # A subagent prefill waits instead of stalling the running decode. + max-running-requests: 1 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled env: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "0" SGLANG_RAGGED_VERIFY_MODE: static - benchmark: - env: - AGENTIC_WARMUP_GRACE_PERIOD: "1800" -override_tp2_c1: - name: agg-gb300-tp2ep2-c1 +override_tp4_c4: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c4 roles: agg: + gpus: 4 args: - chunked-prefill-size: 4096 - max-running-requests: 2 + tensor-parallel-size: 4 + expert-parallel-size: 1 + prefill-decode-interval: 0 + swa-prefix-tails: 4096 + max-running-requests: 128 + cuda-graph-max-bs-decode: 128 env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_RAGGED_VERIFY_MODE: static + +override_tp4_c16: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c16 + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + prefill-decode-interval: 192 + swa-prefix-tails: 4096 + max-running-requests: 128 + cuda-graph-max-bs-decode: 128 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_NUMA_BIND_V2: "0" SGLANG_RAGGED_VERIFY_MODE: static - benchmark: - env: - AGENTIC_WARMUP_GRACE_PERIOD: "1800" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml index c457c1f7c6..bfaeb4f493 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -1,16 +1,24 @@ - schema: 2 base: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg identity: model: {} + frameworks: + dynamo: "1.6.0.dev20261014" + sglang: "0.0.0.dev1+ga6a8c5601" + # The image ships ai-dynamo 1.6.0.dev20261014 (includes ai-dynamo/dynamo#16005). dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" + install: false + request_plane: tcp + environment: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache + HF_DATASETS_CACHE: /hf_hub_cache/datasets + observability: + tachometer: + enabled: false slurm: - # c96 AgentX runs take ~2.5 h including the 60 min profiling phase and the gate. time_limit: "6:00:00" health_check: max_attempts: 1440 @@ -18,8 +26,6 @@ base: resources: gpu_type: gb300 gpus_per_node: 4 - # Dynamo 1.5 discovers workers through etcd and also needs NATS; both run with the - # frontend on the prefill node (two nodes total for 1P1D). services: - name: etcd type: etcd @@ -29,54 +35,78 @@ base: type: nats placement: node: infra + - name: otel-collector + type: otel-collector + enabled: false frontend: type: dynamo enable_multiple_frontends: false + # One CPU node hosts the frontend, etcd, NATS and the AgentX client. + placement: + node: dedicated env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" args: router-mode: kv - # Session-affinity TTL as measured for the single-decode 1P1D cells; the - # multi-decode overrides below set it to 1 s (per-turn load-aware decode pick). - router-session-affinity-ttl-secs: "3600" + router-session-affinity-ttl-secs: "14400" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" engine: sglang roles: prefill: - nodes: 1 - workers: 1 gpus: 2 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - PYTHONNOUSERSITE: "1" PYTHONUNBUFFERED: "1" HF_HUB_CACHE: /hf_hub_cache SGLANG_TIMEOUT_KEEP_ALIVE: "900" SGLANG_DEFAULT_THINKING: "1" SGLANG_DSV41_REASONING_EFFORT: high - SGLANG_DSPARK_OPT_MARKOV_W2_BF16: "True" SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "32768" + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: "1" + SGLANG_RAGGED_VERIFY_MODE: static NCCL_MNNVL_ENABLE: "1" NCCL_CUMEM_ENABLE: "1" MC_FORCE_MNNVL: "1" + NIXL_UCX_SGL_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_MODULE_DIR: "/opt/sglang/lib/python3.12/site-packages/nixl_cu13.libs/ucx" args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true tensor-parallel-size: 2 + data-parallel-size: 2 expert-parallel-size: 2 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: megamoe + dsv4-attn-backend: flashmla disaggregation-transfer-backend: mooncake - # Long-context first turns (P90 ISL ~250k tokens): one 16k-token chunk per step. - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - max-running-requests: 64 - swa-prefix-tails: 4096 + chunked-prefill-size: 65536 + max-running-requests: 128 + swa-prefix-tails: 2048 + enable-decoder-swa-bounded-replay: true + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct mem-fraction-static: 0.85 speculative-algorithm: DSPARK speculative-dspark-block-size: 5 @@ -87,36 +117,50 @@ base: weight-loader-prefetch-checkpoints: true weight-loader-drop-cache-after-load: false model-loader-extra-config: '{"enable_multithread_load":true}' - # KV events for the Dynamo KV router; srtctl allocates the ZMQ ports per worker. + # Only prefill publishes KV events to the Dynamo KV router. kv_events: true decode: - nodes: 1 + nodes: 4 workers: 1 - gpus: 2 + gpus: 16 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - PYTHONNOUSERSITE: "1" PYTHONUNBUFFERED: "1" HF_HUB_CACHE: /hf_hub_cache SGLANG_TIMEOUT_KEEP_ALIVE: "900" SGLANG_DEFAULT_THINKING: "1" SGLANG_DSV41_REASONING_EFFORT: high - SGLANG_DSPARK_OPT_MARKOV_W2_BF16: "True" SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + # One request gateway process saturates a CPU core serving 16 DP ranks. + DYN_SGL_GATEWAY_WORKERS: "8" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + # Sync the per-step DP batch over NCCL instead of gloo. + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "16384" + SGLANG_RAGGED_VERIFY_MODE: static NCCL_MNNVL_ENABLE: "1" NCCL_CUMEM_ENABLE: "1" MC_FORCE_MNNVL: "1" + NIXL_UCX_SGL_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_MODULE_DIR: "/opt/sglang/lib/python3.12/site-packages/nixl_cu13.libs/ucx" args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true - tensor-parallel-size: 2 - expert-parallel-size: 2 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: megamoe + dsv4-attn-backend: flashmla disaggregation-transfer-backend: mooncake - max-running-requests: 128 - cuda-graph-max-bs-decode: 128 mem-fraction-static: 0.85 speculative-algorithm: DSPARK speculative-dspark-block-size: 5 @@ -127,90 +171,52 @@ base: weight-loader-prefetch-checkpoints: true weight-loader-drop-cache-after-load: false model-loader-extra-config: '{"enable_multithread_load":true}' - kv_events: true sbatch_directives: mem: "0" exclusive: "" srun_options: container-remap-root: "" + # Engram host tables need the node's full memory in every worker step. + mem: "0" benchmark: + placement: + node: dedicated + colocate_with_frontend: true env: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - -override_1p1d_c96: - name: disagg-gb300-1p1d-tp2ep2-c96 + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}' + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.10" + AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" -override_1p2d_c48: - name: disagg-gb300-1p2d-tp2ep2-c48 - frontend: - args: - router-session-affinity-ttl-secs: "1" - roles: - decode: - workers: 2 - args: - max-running-requests: 64 - cuda-graph-max-bs-decode: 64 - -override_1p2d_c64: - name: disagg-gb300-1p2d-tp2ep2-c64 - frontend: - args: - router-session-affinity-ttl-secs: "1" - roles: - decode: - workers: 2 - args: - max-running-requests: 64 - cuda-graph-max-bs-decode: 64 - -override_1p2d_spread_c16: - name: disagg-gb300-1p2d-spread-tp2ep2-c16 - frontend: - args: - router-session-affinity-ttl-secs: "1" - roles: - decode: - nodes: 2 - workers: 2 - args: - max-running-requests: 64 - cuda-graph-max-bs-decode: 64 - -override_1p1d_hicache_c160: - name: disagg-gb300-1p1d-hicache-tp2ep2-c160 +override_4p1d_dep2_dep16_c1228: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-4p1d-dep2-dep16-c1228 roles: prefill: + nodes: 2 + workers: 4 args: - enable-hierarchical-cache: true - hicache-ratio: 1 - hicache-write-policy: write_back - hicache-io-backend: direct + hicache-ratio: 1.02 decode: args: - max-running-requests: 192 - cuda-graph-max-bs-decode: 192 + max-running-requests: 384 + cuda-graph-max-bs-decode: 24 + disaggregation-decode-extra-slots: 36 -override_2p1d_hicache_c256: - name: disagg-gb300-2p1d-hicache-tp2ep2-c256 - frontend: - args: - router-session-affinity-ttl-secs: "1" +override_6p1d_dep2_dep16_c2884: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-6p1d-dep2-dep16-c2884 roles: prefill: - nodes: 2 - workers: 2 + nodes: 3 + workers: 6 args: - max-running-requests: 128 - enable-hierarchical-cache: true - hicache-ratio: 1 - hicache-write-policy: write_back - hicache-io-backend: direct + hicache-ratio: 1.1 decode: args: - max-running-requests: 256 - cuda-graph-max-bs-decode: 256 + max-running-requests: 640 + cuda-graph-max-bs-decode: 40 + disaggregation-decode-extra-slots: 80 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index ffd7a896bb..39c1c5a0d3 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8962,13 +8962,13 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 16], srt-recipe: agentic.yaml } dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-20260928-81f27fb3@sha256:d9e4917808cfaa4b3be033a0c85a3a73d71eb93c17accbfa2ac3e96f551f3a33 + image: lmsysorg/sglang:nightly-dev-cu13-20261015-a6a8c560@sha256:0a1ba33aa4e032b8c5bcebdc69c45664074c032074add4f7db01629ec1fcc381 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.5.0.dev20260914" } + router: { name: dynamo-router, version: "1.6.0.dev20261014" } multinode: true disagg: false srt-recipe-dir: dsv41flash/sglang/gb300-fp4/agentx @@ -8986,22 +8986,41 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false - spec-decoding: mtp - conc-list: [1] + conc-list: [4] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c4 + worker: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [16] num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp2_c1 + srt-recipe: agg-variants.yaml:override_tp4_c16 + worker: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [16] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp2_c16 worker: num-worker: 1 tp: 2 ep: 2 dp-attn: false -dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-20260928-81f27fb3@sha256:d9e4917808cfaa4b3be033a0c85a3a73d71eb93c17accbfa2ac3e96f551f3a33 + +dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20261015-a6a8c560@sha256:0a1ba33aa4e032b8c5bcebdc69c45664074c032074add4f7db01629ec1fcc381 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.5.0.dev20260914" } + router: { name: dynamo-router, version: "1.6.0.dev20261014" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -9011,83 +9030,31 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: - dram-utilization: 0.80 search-space: - spec-decoding: mtp - conc-list: [16] - srt-recipe: disagg-variants.yaml:override_1p2d_spread_c16 - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false - decode: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: false - - spec-decoding: mtp - conc-list: [48] - srt-recipe: disagg-variants.yaml:override_1p2d_c48 - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false - decode: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: false - - spec-decoding: mtp - conc-list: [64] - srt-recipe: disagg-variants.yaml:override_1p2d_c64 - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false - decode: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: false - - spec-decoding: mtp - conc-list: [96] - srt-recipe: disagg-variants.yaml:override_1p1d_c96 - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false - decode: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false - - spec-decoding: mtp - conc-list: [160] - srt-recipe: disagg-variants.yaml:override_1p1d_hicache_c160 + conc-list: [1228] + srt-recipe: disagg-variants.yaml:override_4p1d_dep2_dep16_c1228 prefill: - num-worker: 1 + num-worker: 4 tp: 2 ep: 2 - dp-attn: false + dp-attn: true decode: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false + tp: 16 + ep: 16 + dp-attn: true - spec-decoding: mtp - conc-list: [256] - srt-recipe: disagg-variants.yaml:override_2p1d_hicache_c256 + conc-list: [2884] + srt-recipe: disagg-variants.yaml:override_6p1d_dep2_dep16_c2884 prefill: - num-worker: 2 + num-worker: 6 tp: 2 ep: 2 - dp-attn: false + dp-attn: true decode: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false + tp: 16 + ep: 16 + dp-attn: true dsv41flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: image: vllm/vllm-openai:nightly-aarch64-dev-20261012-59e75f4 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index c5da284820..41afdb9153 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9995,3 +9995,16 @@ - "Synthetic acceptance follows the committed musespark_dflash golden curve (#4138, #4139): AL 4.05 at 7 draft tokens, thinking_on." - "The server loads the checkpoint and draft from the pre-seeded shared HF cache with HF_HUB_OFFLINE=1, so concurrent points make no Hub API calls." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4127 + +- config-keys: + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg + scenario-type: + - agentic-coding + description: + - "Run the GB300 DeepSeek-V4.1-Flash Dynamo + SGLang points on lmsysorg/sglang:nightly-dev-cu13-20261015-a6a8c560 (digest sha256:0a1ba33aa4e032b8c5bcebdc69c45664074c032074add4f7db01629ec1fcc381) with the ai-dynamo 1.6.0.dev20261014 it ships (dynamo.install: false, TCP request plane). Tachometer exporters and the OTel collector are off." + - "Aggregated: replace TP4 c1 and TP2/EP2 c1 with TP4/EP1 c1, c4 and c16 and TP2/EP2 c16 on one node each. Engine args and env follow the matching standalone SGLang points of dsv41flash-fp4-gb300-sglang-agentic-dspark (#4033); the Dynamo frontend tokenizes with fastokens, so SGLang's tokenizer-backend and tokenizer-worker-num are dropped." + - "Disaggregated: add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg with 4x DEP2 prefill (2 nodes) and one DEP16 decode (4 nodes) at c1228, and 6x DEP2 prefill (3 nodes) and one DEP16 decode at c2884. MegaMoE all-to-all, FlashMLA attention, DSpark block size 5, Mooncake KV transfer, prefill HiCache write-back (ratio 1.02 at c1228, 1.1 at c2884), 8 decode gateway workers and NCCL DP-batch sync. One CPU node hosts etcd, NATS, the frontend and the AgentX client." + - "Remove dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg (six TP2/EP2 1PxD points, c16 to c256, on the 20260928 image)." + - "The Dynamo frontend renders the chat template, so the AgentX client sends thinking=true, reasoning_effort=high and temperature 1 with every request." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 55413a80ac3740ef2eada67d83467d2765213abc Mon Sep 17 00:00:00 2001 From: "Po-Han Huang (NVIDIA)" <53919306+nvpohanh@users.noreply.github.com> Date: Fri, 16 Oct 2026 06:11:41 -0700 Subject: [PATCH 2/3] chore(changelog): set the pr-link for #4180 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将 perf-changelog 条目的 pr-link 设为 #4180。 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 41afdb9153..574fecfc64 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10007,4 +10007,4 @@ - "Disaggregated: add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg with 4x DEP2 prefill (2 nodes) and one DEP16 decode (4 nodes) at c1228, and 6x DEP2 prefill (3 nodes) and one DEP16 decode at c2884. MegaMoE all-to-all, FlashMLA attention, DSpark block size 5, Mooncake KV transfer, prefill HiCache write-back (ratio 1.02 at c1228, 1.1 at c2884), 8 decode gateway workers and NCCL DP-batch sync. One CPU node hosts etcd, NATS, the frontend and the AgentX client." - "Remove dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg (six TP2/EP2 1PxD points, c16 to c256, on the 20260928 image)." - "The Dynamo frontend renders the chat template, so the AgentX client sends thinking=true, reasoning_effort=high and temperature 1 with every request." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4180 From e39cdfc9c620038023ece0e2fbf4e4b5410f4cae Mon Sep 17 00:00:00 2001 From: "Po-Han Huang (NVIDIA)" <53919306+nvpohanh@users.noreply.github.com> Date: Fri, 16 Oct 2026 06:34:53 -0700 Subject: [PATCH 3/3] fix(agentx): drop no-op settings from GB300 DSV4.1-Flash Dynamo+SGLang recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Keep only HF_HUB_CACHE: gb300-nv serves this model from the Hub through the mounted shared cache, and Dynamo and SGLang both read HF_HUB_CACHE. - Drop env that restates image defaults (SGLANG_DSPARK_FOLDED_SAMPLING is deprecated; folded proposals, the BF16 markov W2 and static ragged verify are defaults) and env that only SGLang's own HTTP server reads, which Dynamo's token-based workers never start. - Drop client knobs equal to the agentx preset (warmup grace 1800 s, failed request threshold 0.10, TCP user timeout 900000 ms). - Drop the NIXL/UCX env from the disaggregated roles; they use Mooncake. - Declare the disaggregated prefill HiCache as DRAM KV offload so results record kv-offloading dram with the hicache backend. - Explain why Tachometer and the lifecycle OTel collector stay off; drop provenance comments. 中文： - 仅保留 HF_HUB_CACHE：gb300-nv 通过挂载的共享缓存从 Hub 提供该模型， Dynamo 与 SGLang 均读取 HF_HUB_CACHE。 - 删除与镜像默认值相同的环境变量（SGLANG_DSPARK_FOLDED_SAMPLING 已弃用； folded proposal、BF16 markov W2 与 static ragged verify 均为默认值）， 以及仅由 SGLang 自身 HTTP 服务读取的环境变量（Dynamo 以 token 输入方式 运行 worker，不会启动该服务）。 - 删除与 agentx 预设相同的客户端参数（warmup grace 1800 秒、失败请求阈值 0.10、TCP user timeout 900000 毫秒）。 - 分离式角色使用 Mooncake，删除 NIXL/UCX 环境变量。 - 将分离式 prefill 的 HiCache 声明为 DRAM KV offload，使结果记录 kv-offloading 为 dram、后端为 hicache。 - 说明 Tachometer 与生命周期 OTel collector 保持关闭的原因，并删除来源说明注释。 Co-Authored-By: Claude Opus 5.5 --- .../sglang/gb300-fp4/agentx/agg-variants.yaml | 22 +++--------- .../gb300-fp4/agentx/disagg-variants.yaml | 34 +++---------------- inferencex-e2e/configs/nvidia-master.yaml | 4 +++ inferencex-e2e/perf-changelog.yaml | 4 +-- 4 files changed, 14 insertions(+), 50 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml index 680fcbdfcd..3c658c854d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -7,14 +7,14 @@ base: frameworks: dynamo: "1.6.0.dev20261014" sglang: "0.0.0.dev1+ga6a8c5601" - # The image ships ai-dynamo 1.6.0.dev20261014 (includes ai-dynamo/dynamo#16005). dynamo: install: false request_plane: tcp + # gb300-nv serves this model from the Hub; read it from the shared mounted cache. environment: - HF_HOME: /hf_hub_cache HF_HUB_CACHE: /hf_hub_cache - HF_DATASETS_CACHE: /hf_hub_cache/datasets + # Tachometer's per-node exporters slow decode steps and add head-node host memory; + # InferenceX does not read its output. observability: tachometer: enabled: false @@ -36,6 +36,7 @@ base: type: nats placement: node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. - name: otel-collector type: otel-collector enabled: false @@ -65,13 +66,6 @@ base: env: PYTHONNOUSERSITE: "1" PYTHONUNBUFFERED: "1" - SGLANG_TIMEOUT_KEEP_ALIVE: "900" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV41_REASONING_EFFORT: high - SGLANG_DSPARK_OPT_MARKOV_W2_BF16: "True" - # AUTO leaves DSpark draft sampling eager for V4.1 TP shards, which leaves c1 - # decode launch-bound; fold it into the draft CUDA graph. - SGLANG_DSPARK_FOLDED_SAMPLING: "2" args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true @@ -97,10 +91,8 @@ base: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" # The client splits this value on whitespace, so it has none. AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}' - AGENTIC_WARMUP_GRACE_PERIOD: "1800" override_tp2_c16: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp2ep2-c16 @@ -130,13 +122,9 @@ override_tp4_c1: tensor-parallel-size: 4 expert-parallel-size: 1 prefill-decode-interval: 0 - # A subagent prefill waits instead of stalling the running decode. max-running-requests: 1 enable-decoder-swa-bounded-replay: true cuda-graph-backend-prefill: disabled - env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "0" - SGLANG_RAGGED_VERIFY_MODE: static override_tp4_c4: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c4 @@ -153,7 +141,6 @@ override_tp4_c4: env: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - SGLANG_RAGGED_VERIFY_MODE: static override_tp4_c16: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c16 @@ -171,4 +158,3 @@ override_tp4_c16: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank SGLANG_NUMA_BIND_V2: "0" - SGLANG_RAGGED_VERIFY_MODE: static diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml index bfaeb4f493..0a8178acc8 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -7,14 +7,14 @@ base: frameworks: dynamo: "1.6.0.dev20261014" sglang: "0.0.0.dev1+ga6a8c5601" - # The image ships ai-dynamo 1.6.0.dev20261014 (includes ai-dynamo/dynamo#16005). dynamo: install: false request_plane: tcp + # gb300-nv serves this model from the Hub; read it from the shared mounted cache. environment: - HF_HOME: /hf_hub_cache HF_HUB_CACHE: /hf_hub_cache - HF_DATASETS_CACHE: /hf_hub_cache/datasets + # Tachometer's per-node exporters slow decode steps and add head-node host memory; + # InferenceX does not read its output. observability: tachometer: enabled: false @@ -35,6 +35,7 @@ base: type: nats placement: node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. - name: otel-collector type: otel-collector enabled: false @@ -65,10 +66,6 @@ base: gpus: 2 env: PYTHONUNBUFFERED: "1" - HF_HUB_CACHE: /hf_hub_cache - SGLANG_TIMEOUT_KEEP_ALIVE: "900" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV41_REASONING_EFFORT: high SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" @@ -77,17 +74,9 @@ base: SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "32768" SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: "1" - SGLANG_RAGGED_VERIFY_MODE: static NCCL_MNNVL_ENABLE: "1" NCCL_CUMEM_ENABLE: "1" MC_FORCE_MNNVL: "1" - NIXL_UCX_SGL_ENABLE: "1" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_MODULE_DIR: "/opt/sglang/lib/python3.12/site-packages/nixl_cu13.libs/ucx" args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true @@ -125,10 +114,6 @@ base: gpus: 16 env: PYTHONUNBUFFERED: "1" - HF_HUB_CACHE: /hf_hub_cache - SGLANG_TIMEOUT_KEEP_ALIVE: "900" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV41_REASONING_EFFORT: high SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank # One request gateway process saturates a CPU core serving 16 DP ranks. @@ -139,17 +124,9 @@ base: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "16384" - SGLANG_RAGGED_VERIFY_MODE: static NCCL_MNNVL_ENABLE: "1" NCCL_CUMEM_ENABLE: "1" MC_FORCE_MNNVL: "1" - NIXL_UCX_SGL_ENABLE: "1" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_MODULE_DIR: "/opt/sglang/lib/python3.12/site-packages/nixl_cu13.libs/ucx" args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true @@ -186,12 +163,9 @@ base: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" # The client splits this value on whitespace, so it has none. AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}' - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.10" AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" - AGENTIC_WARMUP_GRACE_PERIOD: "1800" override_4p1d_dep2_dep16_c1228: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-4p1d-dep2-dep16-c1228 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 39c1c5a0d3..54cfbcbed9 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9032,6 +9032,8 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: - spec-decoding: mtp conc-list: [1228] srt-recipe: disagg-variants.yaml:override_4p1d_dep2_dep16_c1228 + kv-offloading: dram + kv-offload-backend: { name: hicache } prefill: num-worker: 4 tp: 2 @@ -9045,6 +9047,8 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: - spec-decoding: mtp conc-list: [2884] srt-recipe: disagg-variants.yaml:override_6p1d_dep2_dep16_c2884 + kv-offloading: dram + kv-offload-backend: { name: hicache } prefill: num-worker: 6 tp: 2 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 574fecfc64..51aa665b1b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10003,8 +10003,8 @@ - agentic-coding description: - "Run the GB300 DeepSeek-V4.1-Flash Dynamo + SGLang points on lmsysorg/sglang:nightly-dev-cu13-20261015-a6a8c560 (digest sha256:0a1ba33aa4e032b8c5bcebdc69c45664074c032074add4f7db01629ec1fcc381) with the ai-dynamo 1.6.0.dev20261014 it ships (dynamo.install: false, TCP request plane). Tachometer exporters and the OTel collector are off." - - "Aggregated: replace TP4 c1 and TP2/EP2 c1 with TP4/EP1 c1, c4 and c16 and TP2/EP2 c16 on one node each. Engine args and env follow the matching standalone SGLang points of dsv41flash-fp4-gb300-sglang-agentic-dspark (#4033); the Dynamo frontend tokenizes with fastokens, so SGLang's tokenizer-backend and tokenizer-worker-num are dropped." - - "Disaggregated: add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg with 4x DEP2 prefill (2 nodes) and one DEP16 decode (4 nodes) at c1228, and 6x DEP2 prefill (3 nodes) and one DEP16 decode at c2884. MegaMoE all-to-all, FlashMLA attention, DSpark block size 5, Mooncake KV transfer, prefill HiCache write-back (ratio 1.02 at c1228, 1.1 at c2884), 8 decode gateway workers and NCCL DP-batch sync. One CPU node hosts etcd, NATS, the frontend and the AgentX client." + - "Aggregated: replace TP4 c1 and TP2/EP2 c1 with TP4/EP1 c1, c4 and c16 and TP2/EP2 c16 on one node each. Engine args follow the matching standalone SGLang points of dsv41flash-fp4-gb300-sglang-agentic-dspark (#4033). The Dynamo frontend tokenizes with fastokens, so SGLang's tokenizer-backend and tokenizer-worker-num are dropped, as are env settings that restate this image's defaults or only configure SGLang's own HTTP server." + - "Disaggregated: add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg with 4x DEP2 prefill (2 nodes) and one DEP16 decode (4 nodes) at c1228, and 6x DEP2 prefill (3 nodes) and one DEP16 decode at c2884. MegaMoE all-to-all, FlashMLA attention, DSpark block size 5, Mooncake KV transfer, prefill HiCache write-back DRAM KV offload (ratio 1.02 at c1228, 1.1 at c2884), 8 decode gateway workers and NCCL DP-batch sync. One CPU node hosts etcd, NATS, the frontend and the AgentX client." - "Remove dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg (six TP2/EP2 1PxD points, c16 to c256, on the 20260928 image)." - "The Dynamo frontend renders the chat template, so the AgentX client sends thinking=true, reasoning_effort=high and temperature 1 with every request." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4180