diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml index 4a43da219d..3c658c854d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/agg-variants.yaml @@ -1,14 +1,23 @@ - schema: 2 base: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg identity: model: {} + frameworks: + dynamo: "1.6.0.dev20261014" + sglang: "0.0.0.dev1+ga6a8c5601" dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" + install: false + request_plane: tcp + # gb300-nv serves this model from the Hub; read it from the shared mounted cache. + environment: + HF_HUB_CACHE: /hf_hub_cache + # Tachometer's per-node exporters slow decode steps and add head-node host memory; + # InferenceX does not read its output. + observability: + tachometer: + enabled: false slurm: time_limit: "4:00:00" # Cold weight loads from the shared HF cache plus graph capture take 15-20 min. @@ -18,7 +27,6 @@ base: resources: gpu_type: gb300 gpus_per_node: 4 - # Dynamo 1.5 discovers workers through etcd and also needs NATS. services: - name: etcd type: etcd @@ -28,15 +36,25 @@ base: type: nats placement: node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. + - name: otel-collector + type: otel-collector + enabled: false frontend: type: dynamo enable_multiple_frontends: false env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" args: router-mode: kv - router-session-affinity-ttl-secs: "3600" + router-session-affinity-ttl-secs: "14400" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" @@ -45,81 +63,98 @@ base: agg: nodes: 1 workers: 1 - gpus: 2 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" PYTHONNOUSERSITE: "1" PYTHONUNBUFFERED: "1" - HF_HUB_CACHE: /hf_hub_cache - # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. - SGLANG_TIMEOUT_KEEP_ALIVE: "900" - # AgentX measures thinking on, the regime of the golden acceptance curve. - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV41_REASONING_EFFORT: high - SGLANG_DSPARK_OPT_MARKOV_W2_BF16: "True" - # TP2 keeps the row-sharded Engram tables in host DRAM. - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" - SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true - tensor-parallel-size: 2 - expert-parallel-size: 2 mem-fraction-static: 0.8 - # Keep active decode requests progressing while long prefixes are queued. - prefill-decode-interval: 16 + chunked-prefill-size: 16384 # Admission never exceeds the captured decode graph batch. cuda-graph-max-bs-decode: 64 - # DSpark is the checkpoint's bundled draft; the block size is its only knob. speculative-algorithm: DSPARK speculative-dspark-block-size: 5 reasoning-parser: auto tool-call-parser: auto - # Draft passes under long-context load outlast the 1800 s default watchdog. watchdog-timeout: 3600 enable-metrics: true - weight-loader-prefetch-checkpoints: true - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' sbatch_directives: mem: "0" exclusive: "" srun_options: container-remap-root: "" + # Engram host tables need the node's full memory in every worker step. + mem: "0" benchmark: env: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}' + +override_tp2_c16: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp2ep2-c16 + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + expert-parallel-size: 2 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + prefill-decode-interval: 96 + swa-prefix-tails: 1024 + max-running-requests: 32 + env: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_NUMA_BIND_V2: "0" override_tp4_c1: - name: agg-gb300-tp4ep1-c1 + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c1 roles: agg: gpus: 4 args: tensor-parallel-size: 4 expert-parallel-size: 1 - chunked-prefill-size: 4096 - max-running-requests: 2 + prefill-decode-interval: 0 + max-running-requests: 1 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled + +override_tp4_c4: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c4 + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + prefill-decode-interval: 0 + swa-prefix-tails: 4096 + max-running-requests: 128 + cuda-graph-max-bs-decode: 128 env: - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "0" - SGLANG_RAGGED_VERIFY_MODE: static - benchmark: - env: - AGENTIC_WARMUP_GRACE_PERIOD: "1800" + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank -override_tp2_c1: - name: agg-gb300-tp2ep2-c1 +override_tp4_c16: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-agg-tp4ep1-c16 roles: agg: + gpus: 4 args: - chunked-prefill-size: 4096 - max-running-requests: 2 + tensor-parallel-size: 4 + expert-parallel-size: 1 + prefill-decode-interval: 192 + swa-prefix-tails: 4096 + max-running-requests: 128 + cuda-graph-max-bs-decode: 128 env: - SGLANG_RAGGED_VERIFY_MODE: static - benchmark: - env: - AGENTIC_WARMUP_GRACE_PERIOD: "1800" + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank + SGLANG_NUMA_BIND_V2: "0" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml index c457c1f7c6..0a8178acc8 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -1,16 +1,24 @@ - schema: 2 base: name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg identity: model: {} + frameworks: + dynamo: "1.6.0.dev20261014" + sglang: "0.0.0.dev1+ga6a8c5601" dynamo: - install: true - source: - wheel: "1.5.0.dev20260914" + install: false + request_plane: tcp + # gb300-nv serves this model from the Hub; read it from the shared mounted cache. + environment: + HF_HUB_CACHE: /hf_hub_cache + # Tachometer's per-node exporters slow decode steps and add head-node host memory; + # InferenceX does not read its output. + observability: + tachometer: + enabled: false slurm: - # c96 AgentX runs take ~2.5 h including the 60 min profiling phase and the gate. time_limit: "6:00:00" health_check: max_attempts: 1440 @@ -18,8 +26,6 @@ base: resources: gpu_type: gb300 gpus_per_node: 4 - # Dynamo 1.5 discovers workers through etcd and also needs NATS; both run with the - # frontend on the prefill node (two nodes total for 1P1D). services: - name: etcd type: etcd @@ -29,40 +35,45 @@ base: type: nats placement: node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. + - name: otel-collector + type: otel-collector + enabled: false frontend: type: dynamo enable_multiple_frontends: false + # One CPU node hosts the frontend, etcd, NATS and the AgentX client. + placement: + node: dedicated env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" args: router-mode: kv - # Session-affinity TTL as measured for the single-decode 1P1D cells; the - # multi-decode overrides below set it to 1 s (per-turn load-aware decode pick). - router-session-affinity-ttl-secs: "3600" + router-session-affinity-ttl-secs: "14400" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" engine: sglang roles: prefill: - nodes: 1 - workers: 1 gpus: 2 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - PYTHONNOUSERSITE: "1" PYTHONUNBUFFERED: "1" - HF_HUB_CACHE: /hf_hub_cache - SGLANG_TIMEOUT_KEEP_ALIVE: "900" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV41_REASONING_EFFORT: high - SGLANG_DSPARK_OPT_MARKOV_W2_BF16: "True" SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "32768" + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: "1" NCCL_MNNVL_ENABLE: "1" NCCL_CUMEM_ENABLE: "1" MC_FORCE_MNNVL: "1" @@ -70,13 +81,21 @@ base: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true tensor-parallel-size: 2 + data-parallel-size: 2 expert-parallel-size: 2 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: megamoe + dsv4-attn-backend: flashmla disaggregation-transfer-backend: mooncake - # Long-context first turns (P90 ISL ~250k tokens): one 16k-token chunk per step. - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - max-running-requests: 64 - swa-prefix-tails: 4096 + chunked-prefill-size: 65536 + max-running-requests: 128 + swa-prefix-tails: 2048 + enable-decoder-swa-bounded-replay: true + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: direct + hicache-mem-layout: page_first_direct mem-fraction-static: 0.85 speculative-algorithm: DSPARK speculative-dspark-block-size: 5 @@ -87,36 +106,38 @@ base: weight-loader-prefetch-checkpoints: true weight-loader-drop-cache-after-load: false model-loader-extra-config: '{"enable_multithread_load":true}' - # KV events for the Dynamo KV router; srtctl allocates the ZMQ ports per worker. + # Only prefill publishes KV events to the Dynamo KV router. kv_events: true decode: - nodes: 1 + nodes: 4 workers: 1 - gpus: 2 + gpus: 16 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - PYTHONNOUSERSITE: "1" PYTHONUNBUFFERED: "1" - HF_HUB_CACHE: /hf_hub_cache - SGLANG_TIMEOUT_KEEP_ALIVE: "900" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV41_REASONING_EFFORT: high - SGLANG_DSPARK_OPT_MARKOV_W2_BF16: "True" SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT: per_rank - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + # One request gateway process saturates a CPU core serving 16 DP ranks. + DYN_SGL_GATEWAY_WORKERS: "8" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + # Sync the per-step DP batch over NCCL instead of gloo. + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "16384" NCCL_MNNVL_ENABLE: "1" NCCL_CUMEM_ENABLE: "1" MC_FORCE_MNNVL: "1" args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true - tensor-parallel-size: 2 - expert-parallel-size: 2 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: megamoe + dsv4-attn-backend: flashmla disaggregation-transfer-backend: mooncake - max-running-requests: 128 - cuda-graph-max-bs-decode: 128 mem-fraction-static: 0.85 speculative-algorithm: DSPARK speculative-dspark-block-size: 5 @@ -127,90 +148,49 @@ base: weight-loader-prefetch-checkpoints: true weight-loader-drop-cache-after-load: false model-loader-extra-config: '{"enable_multithread_load":true}' - kv_events: true sbatch_directives: mem: "0" exclusive: "" srun_options: container-remap-root: "" + # Engram host tables need the node's full memory in every worker step. + mem: "0" benchmark: + placement: + node: dedicated + colocate_with_frontend: true env: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - -override_1p1d_c96: - name: disagg-gb300-1p1d-tp2ep2-c96 + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}' + AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" -override_1p2d_c48: - name: disagg-gb300-1p2d-tp2ep2-c48 - frontend: - args: - router-session-affinity-ttl-secs: "1" - roles: - decode: - workers: 2 - args: - max-running-requests: 64 - cuda-graph-max-bs-decode: 64 - -override_1p2d_c64: - name: disagg-gb300-1p2d-tp2ep2-c64 - frontend: - args: - router-session-affinity-ttl-secs: "1" - roles: - decode: - workers: 2 - args: - max-running-requests: 64 - cuda-graph-max-bs-decode: 64 - -override_1p2d_spread_c16: - name: disagg-gb300-1p2d-spread-tp2ep2-c16 - frontend: - args: - router-session-affinity-ttl-secs: "1" - roles: - decode: - nodes: 2 - workers: 2 - args: - max-running-requests: 64 - cuda-graph-max-bs-decode: 64 - -override_1p1d_hicache_c160: - name: disagg-gb300-1p1d-hicache-tp2ep2-c160 +override_4p1d_dep2_dep16_c1228: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-4p1d-dep2-dep16-c1228 roles: prefill: + nodes: 2 + workers: 4 args: - enable-hierarchical-cache: true - hicache-ratio: 1 - hicache-write-policy: write_back - hicache-io-backend: direct + hicache-ratio: 1.02 decode: args: - max-running-requests: 192 - cuda-graph-max-bs-decode: 192 + max-running-requests: 384 + cuda-graph-max-bs-decode: 24 + disaggregation-decode-extra-slots: 36 -override_2p1d_hicache_c256: - name: disagg-gb300-2p1d-hicache-tp2ep2-c256 - frontend: - args: - router-session-affinity-ttl-secs: "1" +override_6p1d_dep2_dep16_c2884: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-6p1d-dep2-dep16-c2884 roles: prefill: - nodes: 2 - workers: 2 + nodes: 3 + workers: 6 args: - max-running-requests: 128 - enable-hierarchical-cache: true - hicache-ratio: 1 - hicache-write-policy: write_back - hicache-io-backend: direct + hicache-ratio: 1.1 decode: args: - max-running-requests: 256 - cuda-graph-max-bs-decode: 256 + max-running-requests: 640 + cuda-graph-max-bs-decode: 40 + disaggregation-decode-extra-slots: 80 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index a34a7f4518..e3e7c920b0 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9026,13 +9026,13 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128, 160], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 16], srt-recipe: agentic.yaml } dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-20260928-81f27fb3@sha256:d9e4917808cfaa4b3be033a0c85a3a73d71eb93c17accbfa2ac3e96f551f3a33 + image: lmsysorg/sglang:nightly-dev-cu13-20261015-a6a8c560@sha256:0a1ba33aa4e032b8c5bcebdc69c45664074c032074add4f7db01629ec1fcc381 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.5.0.dev20260914" } + router: { name: dynamo-router, version: "1.6.0.dev20261014" } multinode: true disagg: false srt-recipe-dir: dsv41flash/sglang/gb300-fp4/agentx @@ -9050,22 +9050,41 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false - spec-decoding: mtp - conc-list: [1] + conc-list: [4] num-nodes: 1 - srt-recipe: agg-variants.yaml:override_tp2_c1 + srt-recipe: agg-variants.yaml:override_tp4_c4 + worker: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [16] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp4_c16 + worker: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [16] + num-nodes: 1 + srt-recipe: agg-variants.yaml:override_tp2_c16 worker: num-worker: 1 tp: 2 ep: 2 dp-attn: false -dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-20260928-81f27fb3@sha256:d9e4917808cfaa4b3be033a0c85a3a73d71eb93c17accbfa2ac3e96f551f3a33 + +dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20261015-a6a8c560@sha256:0a1ba33aa4e032b8c5bcebdc69c45664074c032074add4f7db01629ec1fcc381 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.5.0.dev20260914" } + router: { name: dynamo-router, version: "1.6.0.dev20261014" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -9075,83 +9094,35 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: - dram-utilization: 0.80 search-space: - spec-decoding: mtp - conc-list: [16] - srt-recipe: disagg-variants.yaml:override_1p2d_spread_c16 - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false - decode: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: false - - spec-decoding: mtp - conc-list: [48] - srt-recipe: disagg-variants.yaml:override_1p2d_c48 - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false - decode: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: false - - spec-decoding: mtp - conc-list: [64] - srt-recipe: disagg-variants.yaml:override_1p2d_c64 - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false - decode: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: false - - spec-decoding: mtp - conc-list: [96] - srt-recipe: disagg-variants.yaml:override_1p1d_c96 - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false - decode: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false - - spec-decoding: mtp - conc-list: [160] - srt-recipe: disagg-variants.yaml:override_1p1d_hicache_c160 + conc-list: [1228] + srt-recipe: disagg-variants.yaml:override_4p1d_dep2_dep16_c1228 + kv-offloading: dram + kv-offload-backend: { name: hicache } prefill: - num-worker: 1 + num-worker: 4 tp: 2 ep: 2 - dp-attn: false + dp-attn: true decode: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false + tp: 16 + ep: 16 + dp-attn: true - spec-decoding: mtp - conc-list: [256] - srt-recipe: disagg-variants.yaml:override_2p1d_hicache_c256 + conc-list: [2884] + srt-recipe: disagg-variants.yaml:override_6p1d_dep2_dep16_c2884 + kv-offloading: dram + kv-offload-backend: { name: hicache } prefill: - num-worker: 2 + num-worker: 6 tp: 2 ep: 2 - dp-attn: false + dp-attn: true decode: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: false + tp: 16 + ep: 16 + dp-attn: true dsv41flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg: image: vllm/vllm-openai:nightly-aarch64-dev-20261012-59e75f4 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b88d216b21..c92380448e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10014,6 +10014,19 @@ - "Keep the batch-8 cap on running requests and decode graphs from the GPU-resident sweep." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4181 +- config-keys: + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg + scenario-type: + - agentic-coding + description: + - "Run the GB300 DeepSeek-V4.1-Flash Dynamo + SGLang points on lmsysorg/sglang:nightly-dev-cu13-20261015-a6a8c560 (digest sha256:0a1ba33aa4e032b8c5bcebdc69c45664074c032074add4f7db01629ec1fcc381) with the ai-dynamo 1.6.0.dev20261014 it ships (dynamo.install: false, TCP request plane). Tachometer exporters and the OTel collector are off." + - "Aggregated: replace TP4 c1 and TP2/EP2 c1 with TP4/EP1 c1, c4 and c16 and TP2/EP2 c16 on one node each. Engine args follow the matching standalone SGLang points of dsv41flash-fp4-gb300-sglang-agentic-dspark (#4033). The Dynamo frontend tokenizes with fastokens, so SGLang's tokenizer-backend and tokenizer-worker-num are dropped, as are env settings that restate this image's defaults or only configure SGLang's own HTTP server." + - "Disaggregated: add dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg with 4x DEP2 prefill (2 nodes) and one DEP16 decode (4 nodes) at c1228, and 6x DEP2 prefill (3 nodes) and one DEP16 decode at c2884. MegaMoE all-to-all, FlashMLA attention, DSpark block size 5, Mooncake KV transfer, prefill HiCache write-back DRAM KV offload (ratio 1.02 at c1228, 1.1 at c2884), 8 decode gateway workers and NCCL DP-batch sync. One CPU node hosts etcd, NATS, the frontend and the AgentX client." + - "Remove dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg (six TP2/EP2 1PxD points, c16 to c256, on the 20260928 image)." + - "The Dynamo frontend renders the chat template, so the AgentX client sends thinking=true, reasoning_effort=high and temperature 1 with every request." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4180 + - config-keys: - musespark-fp4-h100-sglang-agentic-dflash scenario-type: