diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml index 0a8178acc8..d05ff9634b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -5,11 +5,12 @@ base: identity: model: {} frameworks: - dynamo: "1.6.0.dev20261014" - sglang: "0.0.0.dev1+ga6a8c5601" + dynamo: "1.6.0.dev20261019" + sglang: "0.0.0.dev1+gc1c287a42" dynamo: install: false request_plane: tcp + event_plane: zmq # gb300-nv serves this model from the Hub; read it from the shared mounted cache. environment: HF_HUB_CACHE: /hf_hub_cache @@ -31,10 +32,12 @@ base: type: etcd placement: node: infra - - name: nats - type: nats - placement: - node: infra + - name: mooncake-master + type: mooncake-master + args: + - --eviction_high_watermark_ratio=0.90 + - --client_ttl=60 + - --rpc_thread_num=64 # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. - name: otel-collector type: otel-collector @@ -42,7 +45,8 @@ base: frontend: type: dynamo enable_multiple_frontends: false - # One CPU node hosts the frontend, etcd, NATS and the AgentX client. + numa_bind: true + # One CPU node hosts the frontend, etcd, the Mooncake master and the AgentX client. placement: node: dedicated env: @@ -77,28 +81,45 @@ base: NCCL_MNNVL_ENABLE: "1" NCCL_CUMEM_ENABLE: "1" MC_FORCE_MNNVL: "1" + MOONCAKE_PROTOCOL: rdma + # The store registers host buffers only over RDMA; without a device list it + # picks NVLink for same-domain peers. + MOONCAKE_DEVICE: "@fabric.rdma-devices" + MOONCAKE_GLOBAL_SEGMENT_SIZE: 80gb + MOONCAKE_STANDALONE_STORAGE: "0" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # Some GB300 nodes list a BMC interface with the same address on every node first. + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true tensor-parallel-size: 2 + tokenizer-backend: fastokens + router-mode: kv + router-session-affinity-ttl-secs: "14400" data-parallel-size: 2 expert-parallel-size: 2 enable-dp-attention: true enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true moe-a2a-backend: megamoe - dsv4-attn-backend: flashmla + dsv4-attn-backend: flashmla_mega disaggregation-transfer-backend: mooncake chunked-prefill-size: 65536 max-running-requests: 128 swa-prefix-tails: 2048 enable-decoder-swa-bounded-replay: true - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: direct + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake + hicache-storage-backend-extra-config: '{"enable_group_semantics":true}' + hicache-ratio: 1.02 hicache-mem-layout: page_first_direct mem-fraction-static: 0.85 speculative-algorithm: DSPARK speculative-dspark-block-size: 5 + speculative-num-steps: 5 reasoning-parser: auto tool-call-parser: auto watchdog-timeout: 3600 @@ -106,7 +127,6 @@ base: weight-loader-prefetch-checkpoints: true weight-loader-drop-cache-after-load: false model-loader-extra-config: '{"enable_multithread_load":true}' - # Only prefill publishes KV events to the Dynamo KV router. kv_events: true decode: nodes: 4 @@ -120,27 +140,44 @@ base: DYN_SGL_GATEWAY_WORKERS: "8" SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - # Sync the per-step DP batch over NCCL instead of gloo. - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "16384" + SGLANG_OPT_DSV41_BF16_INDEX_LOGITS: "1" NCCL_MNNVL_ENABLE: "1" NCCL_CUMEM_ENABLE: "1" MC_FORCE_MNNVL: "1" + MOONCAKE_PROTOCOL: rdma + MOONCAKE_DEVICE: "@fabric.rdma-devices" + MOONCAKE_STANDALONE_STORAGE: "0" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" args: served-model-name: deepseek-ai/DeepSeek-V4.1-Flash trust-remote-code: true tensor-parallel-size: 16 + tokenizer-backend: fastokens + router-mode: round-robin + load-balance-method: total_tokens data-parallel-size: 16 expert-parallel-size: 16 enable-dp-attention: true enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true moe-a2a-backend: megamoe - dsv4-attn-backend: flashmla + dsv4-attn-backend: flashmla_mega disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + stream-interval: 32 + swa-prefix-tails: 2048 + enable-decoder-swa-bounded-replay: true + enable-unified-cache-external-linker: true + unified-cache-external-linker-backend: mooncake mem-fraction-static: 0.85 speculative-algorithm: DSPARK speculative-dspark-block-size: 5 + speculative-num-steps: 5 reasoning-parser: auto tool-call-parser: auto watchdog-timeout: 3600 @@ -148,6 +185,7 @@ base: weight-loader-prefetch-checkpoints: true weight-loader-drop-cache-after-load: false model-loader-extra-config: '{"enable_multithread_load":true}' + kv_events: false sbatch_directives: mem: "0" exclusive: "" @@ -167,30 +205,50 @@ base: AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}' AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" -override_4p1d_dep2_dep16_c1228: - name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-4p1d-dep2-dep16-c1228 +override_3p1d_dep2_dep16_c1228: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-3p1d-dep2-dep16-c1228 roles: prefill: + # Three DEP2 workers leave two GPUs of the second node idle. nodes: 2 - workers: 4 - args: - hicache-ratio: 1.02 + workers: 3 decode: + env: + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb args: max-running-requests: 384 cuda-graph-max-bs-decode: 24 disaggregation-decode-extra-slots: 36 -override_6p1d_dep2_dep16_c2884: - name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-6p1d-dep2-dep16-c2884 +override_5p1d_dep2_dep16_c2884: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-5p1d-dep2-dep16-c2884 roles: prefill: + # Five DEP2 workers leave two GPUs of the third node idle. nodes: 3 - workers: 6 + workers: 5 args: - hicache-ratio: 1.1 + max-prefill-tokens: 32768 decode: + env: + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb args: max-running-requests: 640 cuda-graph-max-bs-decode: 40 disaggregation-decode-extra-slots: 80 + +override_8p1d_dep2_dep16_c4096: + name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-8p1d-dep2-dep16-c4096 + roles: + prefill: + nodes: 4 + workers: 8 + args: + max-prefill-tokens: 32768 + decode: + env: + MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb + args: + max-running-requests: 1024 + cuda-graph-max-bs-decode: 64 + disaggregation-decode-extra-slots: 128 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 464fa85db8..9e87e0fb01 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9322,13 +9322,13 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: dp-attn: false dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20261015-a6a8c560@sha256:0a1ba33aa4e032b8c5bcebdc69c45664074c032074add4f7db01629ec1fcc381 + image: lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.6.0.dev20261014" } + router: { name: dynamo-router, version: "1.6.0.dev20261019" } kv-p2p-transfer: mooncake multinode: true disagg: true @@ -9339,11 +9339,11 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: search-space: - spec-decoding: mtp conc-list: [1228] - srt-recipe: disagg-variants.yaml:override_4p1d_dep2_dep16_c1228 + srt-recipe: disagg-variants.yaml:override_3p1d_dep2_dep16_c1228 kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } prefill: - num-worker: 4 + num-worker: 3 tp: 2 ep: 2 dp-attn: true @@ -9354,11 +9354,26 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg: dp-attn: true - spec-decoding: mtp conc-list: [2884] - srt-recipe: disagg-variants.yaml:override_6p1d_dep2_dep16_c2884 + srt-recipe: disagg-variants.yaml:override_5p1d_dep2_dep16_c2884 kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } prefill: - num-worker: 6 + num-worker: 5 + tp: 2 + ep: 2 + dp-attn: true + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [4096] + srt-recipe: disagg-variants.yaml:override_8p1d_dep2_dep16_c4096 + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } + prefill: + num-worker: 8 tp: 2 ep: 2 dp-attn: true diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 8c82ba91cd..38564eb03d 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10806,3 +10806,20 @@ description: - "Route llm-d DEP8 disagg and agg workers with data-parallel-external-lb (one vLLM API per DP rank for the EPP) and VLLM_USE_RUST_FRONTEND=1, matching the Sep 28 standalone llm-d wide-EP launch; agg TP8 keeps a single TP8 worker without external LB. Drop 2P×1D disagg agentic concurrency 576 from the master search space after repeated warmup stalls." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741 + +- config-keys: + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg + scenario-type: + - agentic-coding + description: + - "Include the unchanged aggregated TP4 c1/c4/c16 and TP2 c16 points in the sweep and evaluation coverage, retaining their pinned October 15 image." + - "Move the disaggregated points to lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4 (digest sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb) with the ai-dynamo 1.6.0.dev20261019 it ships and the ZMQ event plane. One CPU node hosts etcd, the Mooncake master, the frontend and the AgentX client." + - "Replace prefill HiCache DRAM offload with the Mooncake external cache linker on both roles (MOONCAKE_GLOBAL_SEGMENT_SIZE 80gb on prefill, 140gb on decode). The higher prefix hit rate lets c1228 run 3x DEP2 prefill instead of 4x and c2884 5x instead of 6x." + - "Add c4096 with 8x DEP2 prefill (4 nodes) and one DEP16 decode: 64 requests per decode rank, 128 extra decode slots." + - "Other engine changes: flashmla_mega attention and local DP control broadcast on both roles; on decode, 2048 SWA prefix tails, decoder SWA bounded replay, stream interval 32 and polling interval 8, with the NCCL DP-batch sync env removed; c2884 and c4096 cap prefill batches at 32768 tokens." + - "Use the image's upstream DeepSeek-V4.1 external linker pool support (sgl-project/sglang#44623) without SGLang source patches." + - "Keep KV routing and session affinity on prefill; use round-robin routing on decode so SGLang places requests with total_tokens load balancing." + - "Use fastokens in both worker roles and BF16 decode index logits." + - "Disable decode KV-event publishing for round-robin routing; retain prefill KV events and metrics on both roles." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4303