[PowerX] verify Slurm completion and normal llm-d exits / 验证 Slurm 完成与 llm-d 正常退出

.github/workflows/benchmark-multinode-tmpl.yml

@@ -538,6 +538,7 @@ jobs:
             ${{ env.RESULT_FILENAME }}_*.json
             agg_${{ env.RESULT_FILENAME }}_*.json
             power_validation_${{ env.RESULT_FILENAME }}_*.json
+            slurm_job_*_outcome.txt
             LOGS/power/**
             LOGS/native_power/**
             result_processing_${{ env.RESULT_FILENAME }}.json

.github/workflows/benchmark-tmpl.yml

@@ -474,6 +474,7 @@ jobs:
             gpu_metrics_identity.json
             gpu_metrics_identity.csv
             power_validation_${{ env.RESULT_FILENAME }}.json
+            slurm_job_*_outcome.txt
             results/gpu_metrics*.csv
             results/gpu_metrics*_context.json
             results/gpu_metrics_identity.json

.github/workflows/test-changelog-gate.yml

@@ -31,6 +31,8 @@ on:
       - "runners/launch_*.sh"
       - "runners/slurm_utils.sh"
       - "runners/test_slurm_utils.py"
+      - "runners/test_llmd_lifecycle.py"
+      - "benchmarks/multi_node/llm-d/**"
       - "utils/ci_priority.py"
       - "utils/test_ci_priority.py"
       - ".github/workflows/reuse-sweep-comment.yml"
@@ -123,4 +125,5 @@ jobs:
             utils/evals/test_batched_eval.py \
             utils/evals/test_run_eval_dispatch.py \
             runners/test_slurm_utils.py \
+            runners/test_llmd_lifecycle.py \
             -v -n 4

benchmarks/multi_node/llm-d/job.slurm

@@ -63,25 +63,11 @@ export DOCKER_CONT_NAME
 : "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}"
 DOCKER_MOUNT_PATH="/workspace"
 
-cleanup() {
-    echo "[${SLURM_JOB_ID}] cleanup on $(hostname)"
-    [[ -n "${WATCHER_PID:-}" ]] && kill "$WATCHER_PID" 2>/dev/null || true
-}
-trap cleanup INT TERM HUP EXIT
-
-# Coordinator-done watcher. server.sh on the decode coordinator writes
-# this marker after the bench finishes; we then scancel the allocation
-# from outside the container (the image has no SLURM client tools).
-# Without this, workers `wait` on local vLLM forever and the job runs
-# to TIME_LIMIT.
+# Workers exit through server.sh after reading the coordinator's final status.
+# Normal completion must not cancel an otherwise successful allocation.
 BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID"
 rm -f "$BENCH_DONE_MARKER"
-(
-    while [[ ! -f "$BENCH_DONE_MARKER" ]]; do sleep 5; done
-    echo "[${SLURM_JOB_ID}] coordinator finished; scancel'ing job"
-    scancel "$SLURM_JOB_ID" 2>/dev/null || true
-) &
-WATCHER_PID=$!
+main_rc=0
 
 # Container engine: 'docker' (default) for clusters where the SLURM
 # user can talk to /var/run/docker.sock (e.g. h200-dgxc-slurm); 'pyxis'
@@ -173,7 +159,7 @@ exec docker run --rm \
         ${DOCKER_MOUNT_PATH}/benchmarks/multi_node/llm-d/server.sh \
             2>&1 | tee /benchmark_logs/slurm_job-'\"\$SLURM_JOB_ID\"'_rank_'\"\$SLURM_PROCID\"'.log
     '
-"
+" || main_rc=$?
 
     srun bash -c "docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f" || true
 
@@ -257,9 +243,11 @@ export MODEL_DIR=/models
 export BENCHMARK_LOGS_DIR=/benchmark_logs
 '"$DOCKER_MOUNT_PATH"'/benchmarks/multi_node/llm-d/server.sh \
     2>&1 | tee /benchmark_logs/slurm_job-${SLURM_JOB_ID}_rank_${SLURM_PROCID}.log
-'
+' || main_rc=$?
 
 else
     echo "Unsupported LLMD_CONTAINER_ENGINE: $LLMD_CONTAINER_ENGINE (expected docker|pyxis)" >&2
     exit 1
 fi
+
+exit "$main_rc"

benchmarks/multi_node/llm-d/server.sh

@@ -114,6 +114,27 @@ else
     DP_ADDR="$DECODE_DP_ADDR"
 fi
 
+# One coordinator publishes the benchmark status; workers leave normally when
+# it finishes so Slurm can distinguish success from cancellation.
+BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID"
+BENCH_RC=0
+
+finish_llmd_node() {
+    local rc=$? pid
+    trap - EXIT
+    if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then
+        printf '%s\n' "$rc" > "$BENCH_DONE_MARKER.tmp" &&
+            mv -f "$BENCH_DONE_MARKER.tmp" "$BENCH_DONE_MARKER" || rc=1
+    fi
+    for pid in "${ENVOY_PID:-}" "${EPP_PID:-}" "${SIDECAR_PID:-}" "${VLLM_PID:-}"; do
+        [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true
+    done
+    exit "$rc"
+}
+trap finish_llmd_node EXIT
+trap 'exit 143' TERM HUP
+trap 'exit 130' INT
+
 DP_SIZE_LOCAL="$GPUS_PER_NODE"
 START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL))
 
@@ -327,11 +348,7 @@ fi
 # ================================================================
 # Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval
 # ================================================================
-if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then
-
-    # Release the allocation whenever the coordinator exits.
-    BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID"
-    trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT
+if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then
 
     # ---- Write endpoints.yaml (file-discovery) ----
     # namespace must match EPP's --pool-namespace (file-discovery filters by it;
@@ -582,10 +599,8 @@ PY
             )
         fi
 
-        # Non-fatal: a failed or timed-out conc point must not abort the sweep
-        # or (under set -e) skip the allocation release below. The EXIT trap
-        # releases the allocation regardless, but continuing here lets a
-        # multi-conc sweep record every point it can.
+        # Continue collecting available points after a failure, retaining the
+        # nonzero verdict for the coordinator's final status and worker shutdown.
         run_benchmark_serving \
             --bench-serving-dir /workspace \
             --tokenizer /models \
@@ -600,7 +615,7 @@ PY
             --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \
             --result-dir "$BENCHMARK_LOGS_DIR/" \
             "${bench_extra_args[@]}" \
-            || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)"
+            || { BENCH_RC=$?; echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$BENCH_RC)"; }
     done
     fi
 
@@ -631,10 +646,16 @@ PY
         )
     fi
 
-    # Signal job.slurm (outside the container, where scancel exists) to release
-    # the allocation; without it workers wait until TIME_LIMIT.
-    touch "$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID"
+    exit "$BENCH_RC"
 else
-    # Workers (prefill leader, prefill/decode workers): keep vLLM alive.
-    wait
+    while [[ ! -f "$BENCH_DONE_MARKER" ]]; do
+        if ! kill -0 "$VLLM_PID" 2>/dev/null; then
+            # The coordinator may publish completion and stop the engine
+            # between the marker check and this process check.
+            [[ -f "$BENCH_DONE_MARKER" ]] && break
+            exit 1
+        fi
+        sleep 2
+    done
+    exit "$(cat "$BENCH_DONE_MARKER")"
 fi

docs/results-and-ingestion.md

@@ -114,6 +114,10 @@ Processing and diagnostic power-audit uploads run after launcher or validation f
 
 The native collector sets UTC and records context beside its CSV for portable replay; existing benchmark monitors keep their current behavior. Its launcher integration requires separate hardware qualification. The offline adapter accepts this context without changing producers. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts.
 
+### Slurm completion receipts
+
+Shared Slurm waiting verifies the terminal allocation state and exit code, consulting `scontrol` when `sacct` is missing or non-terminal and retaining `slurm_job_*_outcome.txt`. Launchers stage available evidence before returning failure. llm-d workers exit using the coordinator’s atomically published status-bearing completion marker; normal completion no longer cancels the allocation.
+
 ## Eval artifacts
 
 ### Per-config identity and collection

docs/results-and-ingestion_zh.md

@@ -114,6 +114,10 @@ PR changelog 选择具有代表性的 NVIDIA 和 AMD 覆盖，并非所有受影
 
 原生采集器单独设置 UTC，并在 CSV 旁记录上下文以支持跨环境回放；现有基准监控行为保持不变。启动器接入需要另行完成硬件验证。离线适配器接受该上下文，不改变现有生产端。正式窗口外的无效样本不能构成覆盖；`boundary_degenerate_rows` 保留其逐 GPU 计数。
 
+### Slurm 完成状态文件
+
+共享 Slurm 等待逻辑检查分配的最终状态和退出码；当 `sacct` 记录缺失或尚未进入最终状态时查询 `scontrol`，并保留 `slurm_job_*_outcome.txt`。启动器先保存已有证据再返回失败。llm-d 工作进程根据协调进程原子发布的完成状态退出；正常结束不再取消 Slurm 分配。
+
 ## 评测工件
 
 ### 单配置身份和收集

perf-changelog.yaml

@@ -7467,3 +7467,15 @@
     - "Collect shared measured GPU power for the existing two-node GB200 GLM-5.2 aggregate AgentX recipe with producer 80d7203e424f903c9017de4608ee2044afce9574, exact selected-concurrency windows, and post-job adapter validation. Preserve native status and invalid diagnostics before failure. Ordinary Slurm limits, serving settings, other AgentX recipes, and fixed-sequence producer pins are unchanged."
     - "为现有双节点 GB200 GLM-5.2 聚合 AgentX 配方接入共享 GPU 实测功耗，按所选并发绑定测量窗口，并在任务结束后通过适配器校验。返回失败前保留原生状态及无效诊断；普通 Slurm 时限、服务配置、其他 AgentX 配方和固定序列 producer 版本保持不变。"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2928
+
+- config-keys:
+    - qwen3.5-fp8-b200-sglang
+    - qwen3.5-fp8-h100-sglang
+    - qwen3.5-fp8-gb200-dynamo-sglang
+    - dsv4-fp4-gb200-llmd-vllm
+  scenario-type:
+    - fixed-seq-len
+  description:
+    - "Verify terminal Slurm receipts and exit llm-d workers normally after preserving available evidence."
+    - "验证 Slurm 最终状态，并在保留已有证据后正常退出 llm-d 工作进程。"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3052

runners/launch_b200-nscale-compat.sh

@@ -449,10 +449,12 @@ EOF
     tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null
 
     wait $POLL_PID
+    SRT_JOB_RC=0
+    verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$?
 
     set -x
 
-    echo "Job $JOB_ID completed!"
+    echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence"
     echo "Collecting results..."
 
     if [ ! -d "$LOGS_DIR" ]; then
@@ -504,6 +506,8 @@ EOF
     done
     find . -name '.nfs*' -delete 2>/dev/null || true
 
+    if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi
+
 else
 
     SQUASH_FILE="/data/home/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"

runners/launch_b200-nscale-slurm.sh

@@ -397,13 +397,10 @@ LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log"
 # streams until the job leaves the queue.
 SRT_JOB_RC=0
 stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$?
-if [[ "$SRT_JOB_RC" != "0" && "$USES_AGENTX_POWER" != "1" ]]; then
-    exit "$SRT_JOB_RC"
-fi
 
 set -x
 
-echo "Job $JOB_ID completed!"
+echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence"
 echo "Collecting results..."
 
 if [ ! -d "$LOGS_DIR" ]; then
@@ -428,7 +425,7 @@ fi
 cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS"
 bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"
 
-if [[ "$AGENTX_POWER_RC" != "0" ]]; then
+if [[ "$AGENTX_POWER_RC" != "0" && "$SRT_JOB_RC" == "0" ]]; then
     echo "ERROR: AgentX power validation failed; available audit and server artifacts were staged" >&2
     exit "$AGENTX_POWER_RC"
 fi
@@ -487,3 +484,5 @@ for i in 1 2 3 4 5; do
     sleep 10
 done
 find . -name '.nfs*' -delete 2>/dev/null || true
+
+if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi

runners/launch_b300-dsxe.sh

@@ -338,10 +338,12 @@ echo "Tailing LOG_FILE: $LOG_FILE"
 tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null
 
 wait $POLL_PID
+SRT_JOB_RC=0
+verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$?
 
 set -x
 
-echo "Job $JOB_ID completed!"
+echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence"
 echo "Collecting results..."
 
 if [ ! -d "$LOGS_DIR" ]; then
@@ -393,6 +395,8 @@ for i in 1 2 3 4 5; do
 done
 find . -name '.nfs*' -delete 2>/dev/null || true
 
+if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi
+
 else
     # HF_HUB_CACHE is set to help with dataset download inside the container
     # for eval jobs.

runners/launch_gb200-nv.sh

@@ -159,7 +159,8 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then
     trap 'bundle_server_logs "$BENCHMARK_LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true' EXIT INT TERM HUP
 
     LOG_FILE="${BENCHMARK_LOGS_DIR}/slurm_job-${JOB_ID}.out"
-    stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1
+    SRT_JOB_RC=0
+    stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$?
 
     while IFS= read -r -d '' result_file; do
         copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1
@@ -174,7 +175,7 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then
     fi
 
     scancel "$JOB_ID" 2>/dev/null || true
-    exit 0
+    exit "$SRT_JOB_RC"
 fi
 
 # MODEL_PATH: Override with pre-downloaded paths on GB200 runner
@@ -852,14 +853,12 @@ LOGS_DIR="outputs/$JOB_ID/logs"
 LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log"
 
 AGENTX_POWER_RC=0
-stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || AGENTX_POWER_RC=$?
-if [[ "$AGENTX_POWER_RC" != "0" && "$USES_AGENTX_POWER" != "1" ]]; then
-    exit 1
-fi
+SRT_JOB_RC=0
+stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$?
 
 set -x
 
-echo "Job $JOB_ID finished!"
+echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence"
 echo "Collecting results..."
 
 if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then
@@ -916,7 +915,7 @@ else
     echo "Warning: Logs directory not found at $LOGS_DIR"
 fi
 
-if [[ "$AGENTX_POWER_RC" != "0" ]]; then
+if [[ "$AGENTX_POWER_RC" != "0" && "$SRT_JOB_RC" == "0" ]]; then
     echo "ERROR: AgentX job or power validation failed; available audit and server artifacts were staged" >&2
     exit "$AGENTX_POWER_RC"
 fi
@@ -987,3 +986,5 @@ fi
 if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then
     copy_eval_artifacts "$LOGS_DIR/eval_results" "$GITHUB_WORKSPACE" || exit 1
 fi
+
+exit "$SRT_JOB_RC"

runners/launch_gb300-nv.sh

@@ -669,10 +669,12 @@ echo "Tailing LOG_FILE: $LOG_FILE"
 tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null
 
 wait $POLL_PID
+SRT_JOB_RC=0
+verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$?
 
 set -x
 
-echo "Job $JOB_ID completed!"
+echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence"
 echo "Collecting results..."
 
 if [ -d "$LOGS_DIR" ]; then
@@ -745,3 +747,5 @@ for i in 1 2 3 4 5; do
     sleep 10
 done
 find . -name '.nfs*' -delete 2>/dev/null || true
+
+if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi

runners/launch_h100-dgxc-slurm.sh

@@ -195,10 +195,12 @@ EOF
     tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null
 
     wait $POLL_PID
+    SRT_JOB_RC=0
+    verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$?
 
     set -x
 
-    echo "Job $JOB_ID completed!"
+    echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence"
     echo "Collecting results..."
 
     if [ ! -d "$LOGS_DIR" ]; then
@@ -244,6 +246,8 @@ EOF
     done
     find . -name '.nfs*' -delete 2>/dev/null || true
 
+    if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi
+
 else
 
     HF_HUB_CACHE_MOUNT="/mnt/nfs/sa-shared/gharunners/hf-hub-cache/"

runners/launch_h200-dgxc-slurm.sh

@@ -348,11 +348,12 @@ EOF
     LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log"
     trap 'rc=$?; bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT INT TERM HUP
 
-    stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1
+    SRT_JOB_RC=0
+    stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$?
 
     set -x
 
-    echo "Job $JOB_ID completed!"
+    echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence"
     echo "Collecting results..."
 
     if [ ! -d "$LOGS_DIR" ]; then
@@ -422,6 +423,8 @@ EOF
     done
     find . -name '.nfs*' -delete 2>/dev/null || true
 
+    if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi
+
 else
     SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"