[PowerX] collect native TileRT 8k1k power / 采集 TileRT 8k1k 原生功耗

.github/workflows/test-process-result.yml

@@ -44,6 +44,9 @@ on:
       - 'utils/test_inject_srt_power_concurrencies.py'
       - 'utils/test_process_result.py'
       - 'utils/test_benchmark_serving.py'
+      - 'benchmarks/multi_node/tilert_utils/**'
+      - 'benchmarks/multi_node/glm5.1_fp8_b200_tilert-disagg.sh'
+      - 'runners/test_tilert_power_lifecycle.py'
 
 permissions:
   contents: read
@@ -65,7 +68,7 @@ jobs:
         run: |
           cd utils
           uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \
-            python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py ../runners/test_kimik3_bh_power.py -v
+            python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py ../runners/test_kimik3_bh_power.py ../runners/test_tilert_power_lifecycle.py -v
 
       - name: Test serving client result persistence
         run: |

benchmarks/multi_node/tilert_utils/run_node.sh

@@ -42,6 +42,36 @@ AGENTIC_LOGS_DIR=${AGENTIC_LOGS_DIR:-$RESULT_DIR/LOGS/agentic}
 mkdir -p "$BENCHMARK_LOGS_DIR"
 
 DONE_SENTINEL="$BENCHMARK_LOGS_DIR/.tilert_done.${SLURM_JOB_ID:-local}"
+source "$(dirname "$0")/../../native_power_lifecycle.sh"
+finish_tilert_node() {
+    local rc=$? pid
+    trap - EXIT
+    if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 && -n "${POWERX_COLLECTOR_PID:-}" ]]; then
+        if [[ "$TILERT_ROLE" == prefill || "$rc" != 0 ]]; then
+            powerx_stop_collectors || rc=$?
+        else
+            powerx_reap_collector || rc=$?
+        fi
+    fi
+    if [[ "$TILERT_ROLE" == prefill ]]; then
+        printf '%s\n' "$rc" > "$DONE_SENTINEL.tmp" || rc=1
+        if [[ -n "${POWERX_HOST_UID:-}" ]]; then
+            chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$DONE_SENTINEL.tmp" || rc=1
+        fi
+        mv -f "$DONE_SENTINEL.tmp" "$DONE_SENTINEL" || rc=1
+    fi
+    for pid in "${ROUTER_PID:-}" "${PREFILL_PID:-}" "${DECODE_PID:-}"; do
+        [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true
+    done
+    exit "$rc"
+}
+trap finish_tilert_node EXIT
+trap 'exit 143' TERM HUP
+trap 'exit 130' INT
+if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then
+    powerx_start_collector "/powerx_native/node-$POWERX_RANK" "/powerx_control" \
+        nvidia "$POWERX_RANK" "$TILERT_ROLE" "$POWERX_GPU_COUNT" 2
+fi
 echo "[tilert-run_node] ROLE=$TILERT_ROLE host=$(hostname) DECODE_HOST=$DECODE_HOST PREFILL_HOST=$PREFILL_HOST"
 
 log_and_run_bg() {
@@ -211,6 +241,9 @@ run_bench_and_eval() {
     wait_for_server_ready --port "$ROUTER_PORT" \
         --server-log "$BENCHMARK_LOGS_DIR/tilert_router.log" --server-pid "$ROUTER_PID"
     local rc=0 conc np
+    if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then
+        powerx_wait_collectors ready || return $?
+    fi
     if [[ "${EVAL_ONLY:-false}" != "true" ]]; then
     for conc in $CONC_LIST; do
         np=$(( conc * 10 ))
@@ -228,7 +261,7 @@ run_bench_and_eval() {
             || { rc=$?; echo "[bench] WARNING: conc=$conc failed/timed out (rc=$rc)"; }
     done
     fi
-    run_lm_eval
+    run_lm_eval || rc=$?
     return $rc
 }
 
@@ -278,13 +311,13 @@ case "$TILERT_ROLE" in
             sleep 5
         done
         if [[ -f "$DONE_SENTINEL" ]]; then
-            echo "[decode] done sentinel received, shutting down"; kill "$DECODE_PID" 2>/dev/null || true; exit 0
+            echo "[decode] done sentinel received, shutting down"
+            exit "$(cat "$DONE_SENTINEL")"
         fi
         echo "[decode] decode_server exited early (see $BENCHMARK_LOGS_DIR/tilert_decode.log)"; exit 1
         ;;
     prefill)
         rdma_preflight || exit 1
-        rm -f "$DONE_SENTINEL"
         if [[ "$TILERT_IS_AGENTIC" == "1" ]]; then
             resolve_trace_source
             install_agentic_deps
@@ -300,8 +333,7 @@ case "$TILERT_ROLE" in
         else
             run_bench_and_eval; BENCH_RC=$?
         fi
-        touch "$DONE_SENTINEL"
-        kill "$ROUTER_PID" "$PREFILL_PID" 2>/dev/null || true
+        # EXIT drains both collectors before the decode role sees completion.
         exit $BENCH_RC
         ;;
     *)

benchmarks/multi_node/tilert_utils/submit.sh

@@ -1,6 +1,11 @@
 #!/usr/bin/env bash
+set -euo pipefail
 set -x
 NODES=$(( ${PREFILL_NODES:-1} + ${DECODE_NODES:-1} ))
+if [[ "${PREFILL_NODES:-1}" != 1 || "${DECODE_NODES:-1}" != 1 ]]; then
+    echo "TileRT native launcher requires one physical node per role" >&2
+    exit 1
+fi
 GPUS_PER_NODE="${GPUS_PER_NODE:-${GPU_COUNT:-$(( ${PREFILL_TP:-8} > ${DECODE_TP:-8} ? ${PREFILL_TP:-8} : ${DECODE_TP:-8} ))}}"
 
 SQUASH_DIR="${B200_SQUASH_DIR:-/home/sa-shared/containers}"
@@ -14,15 +19,97 @@ DECODE_IMAGE="${DECODE_IMAGE:-$IMAGE}"
 squash_path() { echo "$SQUASH_DIR/$(echo "$1" | sed 's/[\/:@#]/_/g').sqsh"; }
 DECODE_SQUASH="$(squash_path "$DECODE_IMAGE")"
 PREFILL_SQUASH="$(squash_path "$PREFILL_IMAGE")"
+MODEL_MOUNTS="$MODEL_PATH:$MODEL_PATH"
+if [[ -n "${HF_HUB_CACHE_HOST_PATH:-}" ]]; then
+    # HF snapshots link to sibling blobs outside the snapshot directory.
+    MODEL_MOUNTS="$HF_HUB_CACHE_HOST_PATH:$HF_HUB_CACHE_HOST_PATH,$MODEL_MOUNTS"
+fi
 
-salloc --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \
-    --nodes="$NODES" --gres=gpu:"$GPUS_PER_NODE" --exclusive --mem=0 \
-    --time="${SALLOC_TIME_LIMIT:-480}" --no-shell --job-name="$RUNNER_NAME"
-JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1)
-
-mapfile -t HOSTS < <(scontrol show hostnames "$(squeue -j "$JOB_ID" -h -o %N)")
-[[ "${#HOSTS[@]}" -ge 2 ]] || { echo "expected >=2 nodes, got: ${HOSTS[*]}"; exit 1; }
+if [[ "${TILERT_IN_ALLOCATION:-0}" != 1 ]]; then
+    # Run inside the allocation returned by this request. Looking up a runner
+    # name can attach to an older job; salloc supplies the authoritative ID.
+    export TILERT_IN_ALLOCATION=1
+    exec salloc --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \
+        --nodes="$NODES" --gres=gpu:"$GPUS_PER_NODE" --exclusive --mem=0 \
+        --time="${SALLOC_TIME_LIMIT:-480}" --job-name="$RUNNER_NAME" \
+        "$BASH" "$0" "$@"
+fi
+JOB_ID="${SLURM_JOB_ID:?salloc did not provide its allocation ID}"
+mapfile -t HOSTS < <(scontrol show hostnames "${SLURM_JOB_NODELIST:?salloc did not provide its nodes}")
+[[ "${#HOSTS[@]}" -eq 2 ]] || { echo "expected 2 nodes, got: ${HOSTS[*]}"; exit 1; }
 export DECODE_HOST="${HOSTS[0]}" PREFILL_HOST="${HOSTS[1]}"
+export POWERX_NATIVE_ENABLED=0
+if [[ "${REQUIRE_POWER:-0}" =~ ^(1|true|TRUE|yes|YES)$ && "$ISL" == 8192 && "$OSL" == 1024 && "${IS_AGENTIC:-0}" != 1 && "${SCENARIO_TYPE:-}" != agentic-coding && "${EVAL_ONLY:-false}" != true ]]; then
+    export POWERX_NATIVE_ENABLED=1
+fi
+export SLURM_JOB_ID="$JOB_ID"
+POWERX_MOUNTS=""
+POWERX_ENV="POWERX_NATIVE_ENABLED"
+if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then
+    export POWERX_HOST_UID="$(id -u)" POWERX_HOST_GID="$(id -g)"
+    export POWERX_COLLECTOR_REVISION="$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)"
+    export POWERX_RAW_ROOT="${POWERX_RAW_ROOT:-/tmp/inferencex-native-$JOB_ID}"
+    export POWERX_CONTROL_ROOT="$GITHUB_WORKSPACE/LOGS/power_control-$JOB_ID"
+    mkdir -p "$POWERX_CONTROL_ROOT" "$GITHUB_WORKSPACE/LOGS/native_power"
+    chmod 777 "$POWERX_CONTROL_ROOT"
+    srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 mkdir -p "$POWERX_RAW_ROOT"
+    srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 chmod 777 "$POWERX_RAW_ROOT"
+    POWERX_MOUNTS=",$POWERX_RAW_ROOT:/powerx_native,$POWERX_CONTROL_ROOT:/powerx_control"
+    POWERX_ENV+=",CUDA_VISIBLE_DEVICES,POWERX_HOST_UID,POWERX_HOST_GID,POWERX_COLLECTOR_REVISION,POWERX_NODE_NAME,POWERX_CLOCK_SYNCHRONIZED,POWERX_RANK,POWERX_GPU_COUNT"
+fi
+rm -f "${BENCHMARK_LOGS_DIR:-$GITHUB_WORKSPACE}/.tilert_done.$JOB_ID"
+# Keep node-local receipts inside the allocation until both serving steps drain.
+# A caught cancellation exits through the same staging path as normal completion.
+wait_owned_step() {
+    local pid="$1" deadline=$((SECONDS + ${TILERT_DECODE_DRAIN:-60})) rc=0
+    while kill -0 "$pid" 2>/dev/null; do
+        if (( SECONDS >= deadline )); then
+            echo "[submit] step $pid did not drain before timeout" >&2
+            kill -TERM "$pid" 2>/dev/null || true
+            sleep 2
+            kill -KILL "$pid" 2>/dev/null || true
+            rc=1
+            break
+        fi
+        sleep 1
+    done
+    wait "$pid" || rc=$?
+    return "$rc"
+}
+
+finish_tilert_submit() {
+    local rc=$? step_rc pid role_rank
+    trap - EXIT
+    trap '' TERM HUP INT
+    if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then
+        printf 'stop\n' > "$POWERX_CONTROL_ROOT/stop" || { [[ "$rc" != 0 ]] || rc=1; }
+    fi
+    if [[ "$rc" != 0 ]]; then
+        for pid in "${PREFILL_SRUN_PID:-}" "${DECODE_SRUN_PID:-}"; do
+            [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true
+        done
+    fi
+    for pid in "${PREFILL_SRUN_PID:-}" "${DECODE_SRUN_PID:-}"; do
+        [[ -n "$pid" ]] || continue
+        step_rc=0
+        wait_owned_step "$pid" || step_rc=$?
+        [[ "$rc" != 0 ]] || rc=$step_rc
+    done
+    if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then
+        for role_rank in 0 1; do
+            # A failed or missing rank must not suppress the other rank's audit.
+            srun --jobid="$JOB_ID" --nodelist="${HOSTS[$role_rank]}" --ntasks=1 \
+                bash -c 'cp -R "$POWERX_RAW_ROOT/node-$1" "$GITHUB_WORKSPACE/LOGS/native_power/"' bash "$role_rank" &
+            step_rc=0
+            wait_owned_step "$!" || step_rc=$?
+            [[ "$rc" != 0 ]] || rc=$step_rc
+        done
+    fi
+    exit "$rc"
+}
+trap finish_tilert_submit EXIT
+trap 'exit 143' TERM HUP
+trap 'exit 130' INT
 
 import_image() {
     local image_ref="$1" squash_file="$2" host="$3"
@@ -43,32 +130,38 @@ mkdir -p "$TILERT_WEIGHTS_DIR"
 
 run_role() {
     local role="$1" host="$2" squash_file="$3"
+    local rank=0 gpu_count="${DECODE_TP:-8}"
+    if [[ "$role" == prefill ]]; then rank=1; gpu_count="${PREFILL_TP:-8}"; fi
+    if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then
+        export POWERX_RANK="$rank" POWERX_GPU_COUNT="$gpu_count" POWERX_NODE_NAME="$host"
+        export CUDA_VISIBLE_DEVICES="$(seq -s, 0 "$((gpu_count - 1))")"
+        export POWERX_CLOCK_SYNCHRONIZED
+        POWERX_CLOCK_SYNCHRONIZED=$(srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \
+            bash -c 'timedatectl show -p NTPSynchronized --value 2>/dev/null || echo false')
+    fi
     # The tilert image bakes no NVIDIA_VISIBLE_DEVICES (unlike vllm-openai), and
     # enroot's nvidia hook only injects the driver when it is set — without it the
     # decode container has no libcuda and torch dies with "Found no NVIDIA driver".
     # docker --gpus sets this implicitly, which is why the image works elsewhere.
     # Exported here (not in --export) because the capabilities value contains a
     # comma, which srun's --export parsing would split on.
     export NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=compute,utility
-    srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \
+    exec srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \
         --container-image="$squash_file" \
-        --container-mounts="$GITHUB_WORKSPACE:/workspace,$MODEL_PATH:$MODEL_PATH,$TILERT_WEIGHTS_DIR:$TILERT_WEIGHTS_DIR" \
+        --container-mounts="$GITHUB_WORKSPACE:/workspace,$MODEL_MOUNTS,$TILERT_WEIGHTS_DIR:$TILERT_WEIGHTS_DIR$POWERX_MOUNTS" \
         --container-workdir=/workspace --no-container-entrypoint \
+        --container-env="$POWERX_ENV" \
         --export=ALL,TILERT_ROLE="$role",DECODE_HOST="$DECODE_HOST",PREFILL_HOST="$PREFILL_HOST",PORT="${PORT:-8888}" \
         bash "/workspace/benchmarks/multi_node/tilert_utils/run_node.sh"
 }
 
 run_role decode "$DECODE_HOST" "$DECODE_SQUASH" &
 DECODE_SRUN_PID=$!
 
-run_role prefill "$PREFILL_HOST" "$PREFILL_SQUASH"
-PREFILL_RC=$?
-
-for _ in $(seq 1 "${TILERT_DECODE_DRAIN:-60}"); do
-    kill -0 "$DECODE_SRUN_PID" 2>/dev/null || break
-    sleep 1
-done
-kill -0 "$DECODE_SRUN_PID" 2>/dev/null && { echo "[submit] decode srun did not exit on its own, killing it"; kill "$DECODE_SRUN_PID" 2>/dev/null; }
-wait "$DECODE_SRUN_PID" 2>/dev/null || true
-
+# Both roles run as owned children so a signal interrupts the shell's wait and
+# reaches EXIT cleanup immediately; run_role execs srun to preserve that PID.
+run_role prefill "$PREFILL_HOST" "$PREFILL_SQUASH" &
+PREFILL_SRUN_PID=$!
+PREFILL_RC=0
+wait "$PREFILL_SRUN_PID" || PREFILL_RC=$?
 exit "$PREFILL_RC"

configs/nvidia-master.yaml

@@ -10273,6 +10273,7 @@ glm5.1-fp8-b200-tilert:
           - "DECODE_NODES=1"
     - isl: 8192
       osl: 1024
+      require-power: true
       search-space:
       - spec-decoding: "mtp"
         conc-list: [1]

docs/configuration-procedures.md

@@ -114,6 +114,12 @@ The runner-name prefix is load-bearing: workflow routing uses `launch_${RUNNER_N
 6. Verify every runner is **Idle** in [repository runner settings](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners) before adding it to sweep traffic.
 7. Verify launcher mounts for `_work`, HF cache, staged weights, and squash images from a compute node. Root containers must not leave root-owned files in the shared workspace.
 
+## Native TileRT power
+
+For GLM-5.1 on B200 Nscale, `MODEL_PATH` can select an existing shared checkpoint instead of the default `/scratch/models/GLM-5.1-FP8`. When it selects an HF snapshot, also set `HF_HUB_CACHE_HOST_PATH` to the existing cache root; TileRT mounts that root at the same absolute path so snapshot links to sibling blobs remain readable. Keep `TILERT_WEIGHTS_DIR` pointed at the separately converted decode weights.
+
+Only fixed 8192/1024 `glm5.1-fp8-b200-tilert` requires native power. TileRT runs inside its returned `salloc` allocation, retains both role exit codes and drains collectors before staging audits. Exactly one physical node per role is supported. Other sequence lengths, AgentX and eval-only do not enable this collector. Hardware qualification and publication remain pending.
+
 ## Register an srt-slurm recipe
 
 Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md). Checked-in recipes: [`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/).

docs/configuration-procedures_zh.md

@@ -114,6 +114,12 @@ runner 名称前缀是关键契约：workflow 通过 `launch_${RUNNER_NAME%%_*}.
 6. 将 runner 加入 sweep 流量前，在[仓库 runner 设置页](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners)确认每个 runner 都是 **Idle**。
 7. 从计算节点验证 launcher 对 `_work`、HF cache、预置权重和 squash 镜像的挂载。root 容器不得在共享 workspace 留下 root 所有的文件。
 
+## TileRT 原生功耗
+
+B200 Nscale 的 GLM-5.1 可用 `MODEL_PATH` 指定已有共享权重，覆盖默认的 `/scratch/models/GLM-5.1-FP8`。若指定 HF snapshot，还需把 `HF_HUB_CACHE_HOST_PATH` 设为现有缓存根目录；TileRT 按相同绝对路径挂载整个缓存，使 snapshot 指向同级 blobs 的软链接可读。`TILERT_WEIGHTS_DIR` 仍指向单独转换的 decode 权重。
+
+仅固定 8192/1024 的 `glm5.1-fp8-b200-tilert` 要求原生功耗。TileRT 在 `salloc` 返回的分配内运行，保留两个角色的退出码，并在保存审计数据前等待采集器排空。每个角色仅支持一个物理节点。其他序列长度、AgentX 和 eval-only 不启用此采集器。硬件资格验证与发布仍待完成。
+
 ## 注册 srt-slurm 配方
 
 映射来源：[`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md)。检入的配方：[`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/)。

perf-changelog.yaml

@@ -7441,3 +7441,20 @@
     - "Replace the removed decode-only alias --cuda-graph-max-bs with its canonical name --cuda-graph-max-bs-decode, retaining the same min(2*CONC, 64) cap. The 20260910 image includes sgl-project/sglang#38375, so argparse interprets the removed spelling as an ambiguous prefix of the decode and prefill options and exits before serving. Leave --cuda-graph-max-bs-prefill unset: it controls a separate prefill token-capture ceiling, not request concurrency."
     - "Switch DSA prefill and decode from tilelang to triton (sgl-project/sglang#30575 Fast Triton Sparse MLA). The 20260910 image already contains that backend; kv-cache-dtype stays fp8_e4m3, which the Triton DSA path requires."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900
+
+- config-keys:
+  - glm5.1-fp8-b200-tilert
+  scenario-type:
+  - fixed-seq-len
+  description:
+  - Require native power only for TileRT 8k1k; keep native launches inside the owned
+    allocation and retain both role exit statuses.
+  - 仅对 TileRT 8k1k 要求原生功耗；原生启动始终使用本任务分配，并保留两个角色退出状态。
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3057
+
+- config-keys:
+    - glm5.1-fp8-b200-tilert
+  description:
+    - "Preserve an explicit prepared GLM-5.1 model path and mount shared HF cache blobs in native TileRT containers."
+    - "保留显式指定的 GLM-5.1 预备模型路径，并在原生 TileRT 容器中挂载共享 HF 缓存 blobs。"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3057

runners/launch_b200-nscale-compat.sh

@@ -56,7 +56,7 @@ elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then
     export MODEL_PATH="/scratch/models/GLM-5-FP8"
     export SRT_SLURM_MODEL_PREFIX="glm5-fp8"
 elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp8" ]]; then
-    export MODEL_PATH="/scratch/models/GLM-5.1-FP8"
+    export MODEL_PATH="${MODEL_PATH:-/scratch/models/GLM-5.1-FP8}"
     export SRT_SLURM_MODEL_PREFIX="glm5.1-fp8"
 elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then
     export MODEL_PATH="/scratch/models/GLM-5-NVFP4"

runners/launch_b200-nscale-slurm.sh

@@ -41,6 +41,10 @@ if [[ "$IS_MULTINODE" != "true" ]]; then
     run_compat_launcher
 fi
 
+if [[ "$FRAMEWORK" == "tilert" && "${IS_AGENTIC:-0}" != "1" ]]; then
+    run_compat_launcher
+fi
+
 if [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then
     export MODEL_PATH="${MODEL_PATH:-$NSCALE_MODEL_ROOT/DeepSeek-V4-Pro}"
     export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro"