[NVIDIA][AgentX] Update GB300 DSV4.1-Flash disagg topology, caching and routing / [NVIDIA][AgentX] 更新 GB300 DSV4.1-Flash 分离式拓扑、缓存与路由

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml

@@ -5,11 +5,12 @@ base:
   identity:
     model: {}
     frameworks:
-      dynamo: "1.6.0.dev20261014"
-      sglang: "0.0.0.dev1+ga6a8c5601"
+      dynamo: "1.6.0.dev20261019"
+      sglang: "0.0.0.dev1+gc1c287a42"
   dynamo:
     install: false
     request_plane: tcp
+    event_plane: zmq
   # gb300-nv serves this model from the Hub; read it from the shared mounted cache.
   environment:
     HF_HUB_CACHE: /hf_hub_cache
@@ -31,18 +32,21 @@ base:
       type: etcd
       placement:
         node: infra
-    - name: nats
-      type: nats
-      placement:
-        node: infra
+    - name: mooncake-master
+      type: mooncake-master
+      args:
+        - --eviction_high_watermark_ratio=0.90
+        - --client_ttl=60
+        - --rpc_thread_num=64
     # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it.
     - name: otel-collector
       type: otel-collector
       enabled: false
   frontend:
     type: dynamo
     enable_multiple_frontends: false
-    # One CPU node hosts the frontend, etcd, NATS and the AgentX client.
+    numa_bind: true
+    # One CPU node hosts the frontend, etcd, the Mooncake master and the AgentX client.
     placement:
       node: dedicated
     env:
@@ -77,36 +81,52 @@ base:
         NCCL_MNNVL_ENABLE: "1"
         NCCL_CUMEM_ENABLE: "1"
         MC_FORCE_MNNVL: "1"
+        MOONCAKE_PROTOCOL: rdma
+        # The store registers host buffers only over RDMA; without a device list it
+        # picks NVLink for same-domain peers.
+        MOONCAKE_DEVICE: "@fabric.rdma-devices"
+        MOONCAKE_GLOBAL_SEGMENT_SIZE: 80gb
+        MOONCAKE_STANDALONE_STORAGE: "0"
+        MC_STORE_CLIENT_METRIC_INTERVAL: "5"
+        MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
+        # Some GB300 nodes list a BMC interface with the same address on every node first.
+        MC_TCP_BIND_ADDRESS: "{node}"
+        WITH_NVIDIA_PEERMEM: "0"
       args:
         served-model-name: deepseek-ai/DeepSeek-V4.1-Flash
         trust-remote-code: true
         tensor-parallel-size: 2
+        tokenizer-backend: fastokens
+        router-mode: kv
+        router-session-affinity-ttl-secs: "14400"
         data-parallel-size: 2
         expert-parallel-size: 2
         enable-dp-attention: true
         enable-dp-lm-head: true
+        enable-dp-attention-local-control-broadcast: true
         moe-a2a-backend: megamoe
-        dsv4-attn-backend: flashmla
+        dsv4-attn-backend: flashmla_mega
         disaggregation-transfer-backend: mooncake
         chunked-prefill-size: 65536
         max-running-requests: 128
         swa-prefix-tails: 2048
         enable-decoder-swa-bounded-replay: true
-        enable-hierarchical-cache: true
-        hicache-write-policy: write_back
-        hicache-io-backend: direct
+        enable-unified-cache-external-linker: true
+        unified-cache-external-linker-backend: mooncake
+        hicache-storage-backend-extra-config: '{"enable_group_semantics":true}'
+        hicache-ratio: 1.02
         hicache-mem-layout: page_first_direct
         mem-fraction-static: 0.85
         speculative-algorithm: DSPARK
         speculative-dspark-block-size: 5
+        speculative-num-steps: 5
         reasoning-parser: auto
         tool-call-parser: auto
         watchdog-timeout: 3600
         enable-metrics: true
         weight-loader-prefetch-checkpoints: true
         weight-loader-drop-cache-after-load: false
         model-loader-extra-config: '{"enable_multithread_load":true}'
-      # Only prefill publishes KV events to the Dynamo KV router.
       kv_events: true
     decode:
       nodes: 4
@@ -120,34 +140,52 @@ base:
         DYN_SGL_GATEWAY_WORKERS: "8"
         SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600"
         SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600"
-        # Sync the per-step DP batch over NCCL instead of gloo.
-        SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1"
         SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
         SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "16384"
+        SGLANG_OPT_DSV41_BF16_INDEX_LOGITS: "1"
         NCCL_MNNVL_ENABLE: "1"
         NCCL_CUMEM_ENABLE: "1"
         MC_FORCE_MNNVL: "1"
+        MOONCAKE_PROTOCOL: rdma
+        MOONCAKE_DEVICE: "@fabric.rdma-devices"
+        MOONCAKE_STANDALONE_STORAGE: "0"
+        MC_STORE_CLIENT_METRIC_INTERVAL: "5"
+        MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
+        MC_TCP_BIND_ADDRESS: "{node}"
+        WITH_NVIDIA_PEERMEM: "0"
       args:
         served-model-name: deepseek-ai/DeepSeek-V4.1-Flash
         trust-remote-code: true
         tensor-parallel-size: 16
+        tokenizer-backend: fastokens
+        router-mode: round-robin
+        load-balance-method: total_tokens
         data-parallel-size: 16
         expert-parallel-size: 16
         enable-dp-attention: true
         enable-dp-lm-head: true
+        enable-dp-attention-local-control-broadcast: true
         moe-a2a-backend: megamoe
-        dsv4-attn-backend: flashmla
+        dsv4-attn-backend: flashmla_mega
         disaggregation-transfer-backend: mooncake
+        disaggregation-decode-polling-interval: 8
+        stream-interval: 32
+        swa-prefix-tails: 2048
+        enable-decoder-swa-bounded-replay: true
+        enable-unified-cache-external-linker: true
+        unified-cache-external-linker-backend: mooncake
         mem-fraction-static: 0.85
         speculative-algorithm: DSPARK
         speculative-dspark-block-size: 5
+        speculative-num-steps: 5
         reasoning-parser: auto
         tool-call-parser: auto
         watchdog-timeout: 3600
         enable-metrics: true
         weight-loader-prefetch-checkpoints: true
         weight-loader-drop-cache-after-load: false
         model-loader-extra-config: '{"enable_multithread_load":true}'
+      kv_events: false
   sbatch_directives:
     mem: "0"
     exclusive: ""
@@ -167,30 +205,50 @@ base:
       AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"reasoning_effort":"high","thinking":true},"temperature":1}'
       AIPERF_DATASET_MMAP_CACHE_ENABLED: "true"
 
-override_4p1d_dep2_dep16_c1228:
-  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-4p1d-dep2-dep16-c1228
+override_3p1d_dep2_dep16_c1228:
+  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-3p1d-dep2-dep16-c1228
   roles:
     prefill:
+      # Three DEP2 workers leave two GPUs of the second node idle.
       nodes: 2
-      workers: 4
-      args:
-        hicache-ratio: 1.02
+      workers: 3
     decode:
+      env:
+        MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
       args:
         max-running-requests: 384
         cuda-graph-max-bs-decode: 24
         disaggregation-decode-extra-slots: 36
 
-override_6p1d_dep2_dep16_c2884:
-  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-6p1d-dep2-dep16-c2884
+override_5p1d_dep2_dep16_c2884:
+  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-5p1d-dep2-dep16-c2884
   roles:
     prefill:
+      # Five DEP2 workers leave two GPUs of the third node idle.
       nodes: 3
-      workers: 6
+      workers: 5
       args:
-        hicache-ratio: 1.1
+        max-prefill-tokens: 32768
     decode:
+      env:
+        MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
       args:
         max-running-requests: 640
         cuda-graph-max-bs-decode: 40
         disaggregation-decode-extra-slots: 80
+
+override_8p1d_dep2_dep16_c4096:
+  name: dsv41flash-fp4-gb300-dynamo-sglang-agentx-disagg-8p1d-dep2-dep16-c4096
+  roles:
+    prefill:
+      nodes: 4
+      workers: 8
+      args:
+        max-prefill-tokens: 32768
+    decode:
+      env:
+        MOONCAKE_GLOBAL_SEGMENT_SIZE: 140gb
+      args:
+        max-running-requests: 1024
+        cuda-graph-max-bs-decode: 64
+        disaggregation-decode-extra-slots: 128

inferencex-e2e/configs/nvidia-master.yaml

@@ -9322,13 +9322,13 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg:
           dp-attn: false
 
 dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg:
-  image: lmsysorg/sglang:nightly-dev-cu13-20261015-a6a8c560@sha256:0a1ba33aa4e032b8c5bcebdc69c45664074c032074add4f7db01629ec1fcc381
+  image: lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb
   model: deepseek-ai/DeepSeek-V4.1-Flash
   model-prefix: dsv41flash
   runner: cluster:gb300-nv
   precision: fp4
   framework: dynamo-sglang
-  router: { name: dynamo-router, version: "1.6.0.dev20261014" }
+  router: { name: dynamo-router, version: "1.6.0.dev20261019" }
   kv-p2p-transfer: mooncake
   multinode: true
   disagg: true
@@ -9339,11 +9339,11 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg:
       search-space:
       - spec-decoding: mtp
         conc-list: [1228]
-        srt-recipe: disagg-variants.yaml:override_4p1d_dep2_dep16_c1228
+        srt-recipe: disagg-variants.yaml:override_3p1d_dep2_dep16_c1228
         kv-offloading: dram
-        kv-offload-backend: { name: hicache }
+        kv-offload-backend: { name: mooncake, version: "0.3.13.post1" }
         prefill:
-          num-worker: 4
+          num-worker: 3
           tp: 2
           ep: 2
           dp-attn: true
@@ -9354,11 +9354,26 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg:
           dp-attn: true
       - spec-decoding: mtp
         conc-list: [2884]
-        srt-recipe: disagg-variants.yaml:override_6p1d_dep2_dep16_c2884
+        srt-recipe: disagg-variants.yaml:override_5p1d_dep2_dep16_c2884
         kv-offloading: dram
-        kv-offload-backend: { name: hicache }
+        kv-offload-backend: { name: mooncake, version: "0.3.13.post1" }
         prefill:
-          num-worker: 6
+          num-worker: 5
+          tp: 2
+          ep: 2
+          dp-attn: true
+        decode:
+          num-worker: 1
+          tp: 16
+          ep: 16
+          dp-attn: true
+      - spec-decoding: mtp
+        conc-list: [4096]
+        srt-recipe: disagg-variants.yaml:override_8p1d_dep2_dep16_c4096
+        kv-offloading: dram
+        kv-offload-backend: { name: mooncake, version: "0.3.13.post1" }
+        prefill:
+          num-worker: 8
           tp: 2
           ep: 2
           dp-attn: true

inferencex-e2e/perf-changelog.yaml

@@ -10806,3 +10806,20 @@
   description:
     - "Route llm-d DEP8 disagg and agg workers with data-parallel-external-lb (one vLLM API per DP rank for the EPP) and VLLM_USE_RUST_FRONTEND=1, matching the Sep 28 standalone llm-d wide-EP launch; agg TP8 keeps a single TP8 worker without external LB. Drop 2P×1D disagg agentic concurrency 576 from the master search space after repeated warmup stalls."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741
+
+- config-keys:
+    - dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg
+    - dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Include the unchanged aggregated TP4 c1/c4/c16 and TP2 c16 points in the sweep and evaluation coverage, retaining their pinned October 15 image."
+    - "Move the disaggregated points to lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4 (digest sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb) with the ai-dynamo 1.6.0.dev20261019 it ships and the ZMQ event plane. One CPU node hosts etcd, the Mooncake master, the frontend and the AgentX client."
+    - "Replace prefill HiCache DRAM offload with the Mooncake external cache linker on both roles (MOONCAKE_GLOBAL_SEGMENT_SIZE 80gb on prefill, 140gb on decode). The higher prefix hit rate lets c1228 run 3x DEP2 prefill instead of 4x and c2884 5x instead of 6x."
+    - "Add c4096 with 8x DEP2 prefill (4 nodes) and one DEP16 decode: 64 requests per decode rank, 128 extra decode slots."
+    - "Other engine changes: flashmla_mega attention and local DP control broadcast on both roles; on decode, 2048 SWA prefix tails, decoder SWA bounded replay, stream interval 32 and polling interval 8, with the NCCL DP-batch sync env removed; c2884 and c4096 cap prefill batches at 32768 tokens."
+    - "Use the image's upstream DeepSeek-V4.1 external linker pool support (sgl-project/sglang#44623) without SGLang source patches."
+    - "Keep KV routing and session affinity on prefill; use round-robin routing on decode so SGLang places requests with total_tokens load balancing."
+    - "Use fastokens in both worker roles and BF16 decode index logits."
+    - "Disable decode KV-event publishing for round-robin routing; retain prefill KV events and metrics on both roles."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4303