From 85986f4cefd085906873a6e63a9e6fee8d04e646 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:24:14 -0700 Subject: [PATCH 1/5] fix: verify Slurm completion and exit llm-d workers normally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验证 Slurm 完成状态并让 llm-d 工作进程正常退出，先保存诊断证据。 --- .../workflows/benchmark-multinode-tmpl.yml | 1 + .github/workflows/benchmark-tmpl.yml | 1 + benchmarks/multi_node/llm-d/job.slurm | 26 ++---- benchmarks/multi_node/llm-d/server.sh | 46 ++++++---- docs/results-and-ingestion.md | 4 + docs/results-and-ingestion_zh.md | 4 + perf-changelog.yaml | 12 +++ runners/launch_b200-nscale-compat.sh | 6 +- runners/launch_b200-nscale-slurm.sh | 7 +- runners/launch_b300-dsxe.sh | 6 +- runners/launch_gb200-nv.sh | 12 ++- runners/launch_gb300-nv.sh | 6 +- runners/launch_h100-dgxc-slurm.sh | 6 +- runners/launch_h200-dgxc-slurm.sh | 7 +- runners/launch_mi355x-amds.sh | 5 ++ runners/slurm_utils.sh | 29 +++++- runners/test_llmd_lifecycle.py | 90 +++++++++++++++++++ runners/test_slurm_utils.py | 57 ++++++++++++ 18 files changed, 278 insertions(+), 47 deletions(-) create mode 100644 runners/test_llmd_lifecycle.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index bae33211bc..cfbaefce87 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -517,6 +517,7 @@ jobs: ${{ env.RESULT_FILENAME }}_*.json agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json + slurm_job_*_outcome.txt LOGS/power/** LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index b08ef0775f..d788a80233 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -467,6 +467,7 @@ jobs: gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json + slurm_job_*_outcome.txt results/gpu_metrics*.csv results/gpu_metrics_identity.json results/agentic_power_window.json diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm index 536507bee5..e3338c31dd 100644 --- a/benchmarks/multi_node/llm-d/job.slurm +++ b/benchmarks/multi_node/llm-d/job.slurm @@ -63,25 +63,11 @@ export DOCKER_CONT_NAME : "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}" DOCKER_MOUNT_PATH="/workspace" -cleanup() { - echo "[${SLURM_JOB_ID}] cleanup on $(hostname)" - [[ -n "${WATCHER_PID:-}" ]] && kill "$WATCHER_PID" 2>/dev/null || true -} -trap cleanup INT TERM HUP EXIT - -# Coordinator-done watcher. server.sh on the decode coordinator writes -# this marker after the bench finishes; we then scancel the allocation -# from outside the container (the image has no SLURM client tools). -# Without this, workers `wait` on local vLLM forever and the job runs -# to TIME_LIMIT. +# Workers exit through server.sh after reading the coordinator's final status. +# Normal completion must not cancel an otherwise successful allocation. BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" rm -f "$BENCH_DONE_MARKER" -( - while [[ ! -f "$BENCH_DONE_MARKER" ]]; do sleep 5; done - echo "[${SLURM_JOB_ID}] coordinator finished; scancel'ing job" - scancel "$SLURM_JOB_ID" 2>/dev/null || true -) & -WATCHER_PID=$! +main_rc=0 # Container engine: 'docker' (default) for clusters where the SLURM # user can talk to /var/run/docker.sock (e.g. h200-dgxc-slurm); 'pyxis' @@ -173,7 +159,7 @@ exec docker run --rm \ ${DOCKER_MOUNT_PATH}/benchmarks/multi_node/llm-d/server.sh \ 2>&1 | tee /benchmark_logs/slurm_job-'\"\$SLURM_JOB_ID\"'_rank_'\"\$SLURM_PROCID\"'.log ' -" +" || main_rc=$? srun bash -c "docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f" || true @@ -257,9 +243,11 @@ export MODEL_DIR=/models export BENCHMARK_LOGS_DIR=/benchmark_logs '"$DOCKER_MOUNT_PATH"'/benchmarks/multi_node/llm-d/server.sh \ 2>&1 | tee /benchmark_logs/slurm_job-${SLURM_JOB_ID}_rank_${SLURM_PROCID}.log -' +' || main_rc=$? else echo "Unsupported LLMD_CONTAINER_ENGINE: $LLMD_CONTAINER_ENGINE (expected docker|pyxis)" >&2 exit 1 fi + +exit "$main_rc" diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 89894b210c..b38940a859 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -114,6 +114,27 @@ else DP_ADDR="$DECODE_DP_ADDR" fi +# One coordinator publishes the benchmark status; workers leave normally when +# it finishes so Slurm can distinguish success from cancellation. +BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" +BENCH_RC=0 + +finish_llmd_node() { + local rc=$? pid + trap - EXIT + if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then + printf '%s\n' "$rc" > "$BENCH_DONE_MARKER.tmp" && + mv -f "$BENCH_DONE_MARKER.tmp" "$BENCH_DONE_MARKER" || rc=1 + fi + for pid in "${ENVOY_PID:-}" "${EPP_PID:-}" "${SIDECAR_PID:-}" "${VLLM_PID:-}"; do + [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true + done + exit "$rc" +} +trap finish_llmd_node EXIT +trap 'exit 143' TERM HUP +trap 'exit 130' INT + DP_SIZE_LOCAL="$GPUS_PER_NODE" START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) @@ -327,11 +348,7 @@ fi # ================================================================ # Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval # ================================================================ -if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then - - # Release the allocation whenever the coordinator exits. - BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" - trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT +if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then # ---- Write endpoints.yaml (file-discovery) ---- # namespace must match EPP's --pool-namespace (file-discovery filters by it; @@ -582,10 +599,8 @@ PY ) fi - # Non-fatal: a failed or timed-out conc point must not abort the sweep - # or (under set -e) skip the allocation release below. The EXIT trap - # releases the allocation regardless, but continuing here lets a - # multi-conc sweep record every point it can. + # Continue collecting available points after a failure, retaining the + # nonzero verdict for the coordinator's final status and worker shutdown. run_benchmark_serving \ --bench-serving-dir /workspace \ --tokenizer /models \ @@ -600,7 +615,7 @@ PY --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \ --result-dir "$BENCHMARK_LOGS_DIR/" \ "${bench_extra_args[@]}" \ - || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" + || { BENCH_RC=$?; echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$BENCH_RC)"; } done fi @@ -631,10 +646,11 @@ PY ) fi - # Signal job.slurm (outside the container, where scancel exists) to release - # the allocation; without it workers wait until TIME_LIMIT. - touch "$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" + exit "$BENCH_RC" else - # Workers (prefill leader, prefill/decode workers): keep vLLM alive. - wait + while [[ ! -f "$BENCH_DONE_MARKER" ]]; do + kill -0 "$VLLM_PID" 2>/dev/null || exit 1 + sleep 2 + done + exit "$(cat "$BENCH_DONE_MARKER")" fi diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index daba2715b7..2fcc7c0396 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -96,6 +96,10 @@ Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physica InferenceX-app treats routing fields as columns or config dimensions and stores numeric measurements in `benchmark_results.metrics` JSONB. The mapper supports v1 shared topology, v2 split prefill/decode topology, and nested v3 AgentX metrics. Unknown numeric metrics are retained and warned about, which permits schema growth without silently losing numeric data. +### Slurm completion receipts + +Shared Slurm waiting verifies the terminal allocation state and exit code, consulting `scontrol` when `sacct` is missing or non-terminal and retaining `slurm_job_*_outcome.txt`. Launchers stage available evidence before returning failure. llm-d workers exit using the coordinator’s atomically published status-bearing completion marker; normal completion no longer cancels the allocation. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index cc5c9e3026..edb2bb3a32 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -96,6 +96,10 @@ shape: array of benchmark row objects InferenceX-app 将路由字段作为列或配置维度，并把数值测量存入 `benchmark_results.metrics` JSONB。映射器支持共享拓扑的 v1、拆分 prefill/decode 拓扑的 v2，以及嵌套 AgentX 指标的 v3。未知数值指标会被保留并产生警告，因此架构可以扩展，同时不会无提示地丢失数值数据。 +### Slurm 完成状态文件 + +共享 Slurm 等待逻辑检查分配的最终状态和退出码；当 `sacct` 记录缺失或尚未进入最终状态时查询 `scontrol`，并保留 `slurm_job_*_outcome.txt`。启动器先保存已有证据再返回失败。llm-d 工作进程根据协调进程原子发布的完成状态退出；正常结束不再取消 Slurm 分配。 + ## 评测工件 ### 单配置身份和收集 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 88c163461a..db1cd76273 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7399,3 +7399,15 @@ - "Disable adaptive verification in the EVAL_ONLY DSpark config as well. It trims verification requests on device, which the ROCm DeepseekV4IndexerBackend does not support, so the eval-only engine refused to start (run 34651830283, c32). Evals keep real block rejection; throughput settings are unchanged." - "EVAL_ONLY 的 DSpark 配置同样关闭自适应验证：它会在设备端裁剪验证请求，而 ROCm 的 DeepseekV4IndexerBackend 不支持该操作，导致仅评测引擎拒绝启动（运行 34651830283，c32）。评测仍保留真实块拒绝采样；吞吐设置不变。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + - dsv4-fp4-gb200-llmd-vllm + scenario-type: + - fixed-seq-len + description: + - "Verify terminal Slurm receipts and exit llm-d workers normally after preserving available evidence." + - "验证 Slurm 最终状态，并在保留已有证据后正常退出 llm-d 工作进程。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3052 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 960e33e9dc..dd634bb534 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -449,10 +449,12 @@ EOF tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID + SRT_JOB_RC=0 + verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -504,6 +506,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else SQUASH_FILE="/data/home/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index b5e0ef627c..62cd88d276 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -373,11 +373,12 @@ LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" # Waits for the log file to appear, fails fast if the job dies first, then # streams until the job leaves the queue. -stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -448,3 +449,5 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index c0c959d252..e68951f730 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -338,10 +338,12 @@ echo "Tailing LOG_FILE: $LOG_FILE" tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID +SRT_JOB_RC=0 +verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -393,6 +395,8 @@ for i in 1 2 3 4 5; do done find . -name '.nfs*' -delete 2>/dev/null || true +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else # HF_HUB_CACHE is set to help with dataset download inside the container # for eval jobs. diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index b6090491bd..4a695aa9fb 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -158,7 +158,8 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then trap 'bundle_server_logs "$BENCHMARK_LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true' EXIT INT TERM HUP LOG_FILE="${BENCHMARK_LOGS_DIR}/slurm_job-${JOB_ID}.out" - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + SRT_JOB_RC=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? while IFS= read -r -d '' result_file; do copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 @@ -173,7 +174,7 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then fi scancel "$JOB_ID" 2>/dev/null || true - exit 0 + exit "$SRT_JOB_RC" fi # MODEL_PATH: Override with pre-downloaded paths on GB200 runner @@ -822,11 +823,12 @@ trap 'exit 143' TERM HUP LOGS_DIR="outputs/$JOB_ID/logs" LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" -stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ -d "$LOGS_DIR" ]; then @@ -908,3 +910,5 @@ fi if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then copy_eval_artifacts "$LOGS_DIR/eval_results" "$GITHUB_WORKSPACE" || exit 1 fi + +exit "$SRT_JOB_RC" diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 9249e91f27..fede5b1f60 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -669,10 +669,12 @@ echo "Tailing LOG_FILE: $LOG_FILE" tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID +SRT_JOB_RC=0 +verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ -d "$LOGS_DIR" ]; then @@ -745,3 +747,5 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index af43c740c0..de50907f6a 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -195,10 +195,12 @@ EOF tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID + SRT_JOB_RC=0 + verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -244,6 +246,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else HF_HUB_CACHE_MOUNT="/mnt/nfs/sa-shared/gharunners/hf-hub-cache/" diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 95f519fd3f..dbc2d5f58d 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -348,11 +348,12 @@ EOF LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" trap 'rc=$?; bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT INT TERM HUP - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + SRT_JOB_RC=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -422,6 +423,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index accd850219..6e07dad917 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -123,6 +123,10 @@ if [[ "$IS_MULTINODE" == "true" ]]; then wait $POLL_PID + source "$GITHUB_WORKSPACE/runners/slurm_utils.sh" + slurm_outcome_rc=0 + verify_slurm_job_completion "$JOB_ID" || slurm_outcome_rc=$? + set -x # FIXME: The below is bad and is a result of the indirection of the ways in which @@ -249,6 +253,7 @@ PY sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true # Log preservation and cleanup handled by EXIT trap (cleanup_and_save_logs) + exit "$slurm_outcome_rc" else diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 8a6462f121..a8fe5b0d7f 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -45,6 +45,32 @@ slurm_job_is_active() { squeue -j "$job_id" --noheader 2>/dev/null | grep -q "$job_id" } +verify_slurm_job_completion() { + local job_id="$1" records record state="" exit_code="" + # Disappearance from squeue only means the job is no longer active. Read + # the allocation's terminal record, not a successful batch/extern step. + records=$(sacct -j "$job_id" --noheader --parsable2 --format=JobIDRaw,State,ExitCode 2>/dev/null) || records="" + record=$(printf '%s\n' "$records" | awk -F'|' -v job="$job_id" '$1 == job {print; exit}') + if [[ -n "$record" ]]; then + IFS='|' read -r _ state exit_code <<< "$record" + fi + case "$state" in + COMPLETED|FAILED|CANCELLED*|TIMEOUT|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED|DEADLINE|BOOT_FAIL) ;; + *) + # Accounting may be missing or still report RUNNING after squeue + # empties. Ask the controller before judging the final outcome. + record=$(scontrol show job "$job_id" --oneliner 2>/dev/null) || record="" + state=$(printf '%s\n' "$record" | sed -n 's/.*JobState=\([^ ]*\).*/\1/p') + exit_code=$(printf '%s\n' "$record" | sed -n 's/.*ExitCode=\([^ ]*\).*/\1/p') + ;; + esac + printf '%s\n' "$record" > "${GITHUB_WORKSPACE:-.}/slurm_job_${job_id}_outcome.txt" + if [[ "$state" != "COMPLETED" || "$exit_code" != "0:0" ]]; then + echo "ERROR: Slurm job $job_id ended with state=${state:-unknown} exit=${exit_code:-unknown}" >&2 + return 1 + fi +} + stream_slurm_job_log() { local job_id="$1" local log_file="$2" @@ -52,7 +78,7 @@ stream_slurm_job_log() { while [[ ! -f "$log_file" ]]; do if ! slurm_job_is_active "$job_id"; then echo "ERROR: job $job_id failed before creating $log_file" >&2 - scontrol show job "$job_id" || true + verify_slurm_job_completion "$job_id" || true return 1 fi sleep 5 @@ -68,6 +94,7 @@ stream_slurm_job_log() { echo "Tailing $log_file" tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null wait "$poll_pid" + verify_slurm_job_completion "$job_id" } copy_to_workspace() { diff --git a/runners/test_llmd_lifecycle.py b/runners/test_llmd_lifecycle.py new file mode 100644 index 0000000000..0b21452490 --- /dev/null +++ b/runners/test_llmd_lifecycle.py @@ -0,0 +1,90 @@ +"""Exercise llm-d shutdown with local stand-ins for the Slurm/container boundary.""" +import os +from pathlib import Path +import re +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' +SERVER = ROOT / 'benchmarks/multi_node/llm-d/server.sh' + + +@pytest.mark.parametrize('main_rc', [0, 7]) +def test_llmd_completed_coordinator_does_not_cancel_allocation(tmp_path, main_rc): + cwd = tmp_path / 'repo/benchmarks/multi_node/llm-d' + cwd.mkdir(parents=True) + model, logs, bindir = tmp_path / 'model', tmp_path / 'logs', tmp_path / 'bin' + for path in (model, logs, bindir): + path.mkdir() + squash = tmp_path / 'image.sqsh' + squash.write_text('fixture image') + commands = { + 'scontrol': '#!/bin/sh\nprintf "node-a\\nnode-b\\n"\n', + 'scancel': '#!/bin/sh\necho unexpected-cancel >> "$CANCEL_RECEIPT"\n', + 'sleep': '#!/bin/sh\n/bin/sleep 0.01\n', + 'srun': '#!' + sys.executable + '\n' + r''' +import os, pathlib, subprocess, sys, time +args = sys.argv[1:] +if any(a.startswith('--container-image=') for a in args): + marker = pathlib.Path(os.environ['BENCHMARK_LOGS_DIR']) / ('.bench_done.' + os.environ['SLURM_JOB_ID']) + marker.write_text(os.environ['MAIN_RC'] + '\n') + time.sleep(0.2) + sys.exit(int(os.environ['MAIN_RC'])) +if 'ip route' in ' '.join(args): + print('127.0.0.1') + sys.exit(0) +args = [arg for arg in args if not arg.startswith('--')] +sys.exit(subprocess.run(args).returncode) +''', + } + for name, script in commands.items(): + path = bindir / name + path.write_text(script) + path.chmod(0o755) + defaults = dict.fromkeys('PREFILL_WORKERS DECODE_WORKERS PREFILL_DP_SIZE DECODE_DP_SIZE ' + 'BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_MAX_CONCURRENCY ' + 'BENCH_REQUEST_RATE BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER ' + 'RUN_EVAL EVAL_ONLY EVAL_CONC EVAL_FRAMEWORK EVAL_LIMIT EVAL_SUITE ' + 'SWEBENCH_GEN_MODE SWEBENCH_USE_MODAL MODAL_TOKEN_ID MODAL_TOKEN_SECRET ' + 'IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX ' + 'RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE'.split(), '1') + env = {**os.environ, **defaults, 'PATH': str(bindir) + os.pathsep + os.environ['PATH'], + 'SLURM_JOB_ID': 'local-fixture', 'SLURM_JOB_NODELIST': 'node-[a-b]', + 'NUM_NODES': '2', 'PREFILL_NODES': '1', 'DECODE_NODES': '1', 'GPUS_PER_NODE': '2', + 'MODEL_DIR': str(model), 'MODEL_NAME': 'fixture', 'BENCHMARK_LOGS_DIR': str(logs), + 'LLMD_CONTAINER_ENGINE': 'pyxis', 'LLMD_SQUASH_FILE': str(squash), + 'MAIN_RC': str(main_rc), 'CANCEL_RECEIPT': str(tmp_path / 'cancelled')} + result = subprocess.run(['bash', str(JOB)], cwd=cwd, env=env, + capture_output=True, text=True, timeout=10) + assert result.returncode == main_rc, result.stderr + result.stdout + assert not (tmp_path / 'cancelled').exists() + + +@pytest.mark.parametrize('node_rc', [0, 7]) +def test_llmd_node_records_status_and_stops_owned_server(tmp_path, node_rc): + function = re.search(r'^finish_llmd_node\(\) \{\n.*?^\}', + SERVER.read_text(), flags=re.MULTILINE | re.DOTALL).group() + command = function + r''' +NODE_RANK=1 PREFILL_NODES=1 +BENCH_DONE_MARKER="$1/done" +# Observe the publication boundary after shell redirection opens its target, +# but before printf can write the status seen by workers. +printf() { + if [[ -e "$BENCH_DONE_MARKER" ]]; then + echo 'worker can observe an incomplete status' >&2 + return 1 + fi + builtin printf "$@" +} +sleep 60 & +VLLM_PID=$! +trap finish_llmd_node EXIT +exit "$2" +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(tmp_path), str(node_rc)], + capture_output=True, text=True, timeout=5) + assert result.returncode == node_rc, result.stderr + assert (tmp_path / 'done').read_text().strip() == str(node_rc) diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 31fa6f2183..64a89b8cb5 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -631,3 +631,60 @@ def test_mi355x_agentic_model_mount_and_routing( script = f"benchmarks/single_node/agentic/{prefix}_fp4_mi355x_vllm_mtp.sh" assert args[-2] == script assert (REPO_ROOT / script).is_file() + + +@pytest.mark.parametrize('state,exit_code,expected', [ + ('COMPLETED', '0:0', 0), ('FAILED', '1:0', 1), ('TIMEOUT', '0:15', 1), + ('COMPLETED', '1:0', 1), ('CANCELLED', '0:15', 1), +]) +def test_slurm_terminal_allocation_status_is_required(tmp_path, state, exit_code, expected): + result = run_bash( + f'source "$1"; export GITHUB_WORKSPACE="$2"; ' + f'sacct() {{ printf "42|{state}|{exit_code}\\n42.batch|COMPLETED|0:0\\n"; }}; ' + 'scontrol() { echo "JobId=42 JobState=COMPLETED ExitCode=0:0"; }; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == f'42|{state}|{exit_code}' + + +@pytest.mark.parametrize('accounting_state,controller_state,exit_code,expected', [ + ('', 'COMPLETED', '0:0', 0), + ('RUNNING', 'COMPLETED', '0:0', 0), + ('COMPLETING', 'COMPLETED', '0:0', 0), + ('RUNNING', 'FAILED', '1:0', 1), + ('RUNNING', 'RUNNING', '0:0', 1), +]) +def test_slurm_recent_completion_falls_back_to_controller( + tmp_path, accounting_state, controller_state, exit_code, expected, +): + record = f'42|{accounting_state}|0:0' if accounting_state else '' + controller = f'JobId=42 JobState={controller_state} ExitCode={exit_code}' + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; ' + f'sacct() {{ echo "{record}"; }}; ' + f'scontrol() {{ echo "{controller}"; }}; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == controller + + +def test_slurm_unknown_terminal_state_is_not_success(tmp_path): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; sacct() { return 1; }; ' + 'scontrol() { return 1; }; verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == 1 + assert 'state=unknown' in result.stderr + + +def test_slurm_exit_before_log_retains_terminal_receipt(tmp_path): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; ' + 'slurm_job_is_active() { return 1; }; ' + 'sacct() { printf "42|FAILED|1:0\\n"; }; ' + 'stream_slurm_job_log 42 "$2/missing.log"', SLURM_UTILS, tmp_path, + ) + assert result.returncode == 1 + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == '42|FAILED|1:0' From 86b86214e902a10293e51aa0cefe9b682aaac5f2 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:39:34 -0700 Subject: [PATCH 2/5] fix: parse Slurm allocation exit codes without derived status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按完整字段解析 Slurm 分配退出码，避免误用 DerivedExitCode。 --- runners/slurm_utils.sh | 2 +- runners/test_slurm_utils.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index a8fe5b0d7f..85a2603f75 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -61,7 +61,7 @@ verify_slurm_job_completion() { # empties. Ask the controller before judging the final outcome. record=$(scontrol show job "$job_id" --oneliner 2>/dev/null) || record="" state=$(printf '%s\n' "$record" | sed -n 's/.*JobState=\([^ ]*\).*/\1/p') - exit_code=$(printf '%s\n' "$record" | sed -n 's/.*ExitCode=\([^ ]*\).*/\1/p') + exit_code=$(printf '%s\n' "$record" | tr ' ' '\n' | sed -n 's/^ExitCode=//p') ;; esac printf '%s\n' "$record" > "${GITHUB_WORKSPACE:-.}/slurm_job_${job_id}_outcome.txt" diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 64a89b8cb5..7d52e1f661 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -688,3 +688,13 @@ def test_slurm_exit_before_log_retains_terminal_receipt(tmp_path): ) assert result.returncode == 1 assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == '42|FAILED|1:0' + + +@pytest.mark.parametrize('exit_code,derived,expected', [('0:0', '7:0', 0), ('1:0', '0:0', 1)]) +def test_slurm_controller_checks_allocation_not_derived_exit(tmp_path, exit_code, derived, expected): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; sacct() { return 1; }; ' + f'scontrol() {{ echo "JobId=42 JobState=COMPLETED ExitCode={exit_code} DerivedExitCode={derived}"; }}; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr From 20e5e71dfe1ca3b17a6ce95adbabe0f6fb93e7c1 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:43:52 -0700 Subject: [PATCH 3/5] ci: run llm-d lifecycle regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在现有 CPU 工作流中执行 llm-d 生命周期回归，并跟踪相关脚本变更。 --- .github/workflows/test-changelog-gate.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-changelog-gate.yml b/.github/workflows/test-changelog-gate.yml index 7d49c2ff11..d88effabec 100644 --- a/.github/workflows/test-changelog-gate.yml +++ b/.github/workflows/test-changelog-gate.yml @@ -31,6 +31,8 @@ on: - "runners/launch_*.sh" - "runners/slurm_utils.sh" - "runners/test_slurm_utils.py" + - "runners/test_llmd_lifecycle.py" + - "benchmarks/multi_node/llm-d/**" - "utils/ci_priority.py" - "utils/test_ci_priority.py" - ".github/workflows/reuse-sweep-comment.yml" @@ -123,4 +125,5 @@ jobs: utils/evals/test_batched_eval.py \ utils/evals/test_run_eval_dispatch.py \ runners/test_slurm_utils.py \ + runners/test_llmd_lifecycle.py \ -v -n 4 From fe3a0d49fb07d44015a21be39039078dd01e6bfd Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:48:30 -0700 Subject: [PATCH 4/5] test: synchronize shared replay signal readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将已验证的信号就绪屏障移至共享原生采集前置，避免重放 exec 前的竞态。 --- utils/agentic/aggregation/test_power_lifecycle.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/utils/agentic/aggregation/test_power_lifecycle.py b/utils/agentic/aggregation/test_power_lifecycle.py index 20710e68eb..6d955af12f 100644 --- a/utils/agentic/aggregation/test_power_lifecycle.py +++ b/utils/agentic/aggregation/test_power_lifecycle.py @@ -280,8 +280,13 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( }} stop_gpu_monitor() {{ printf 'monitor-stop\n' >> {str(event_log)!r}; }} fake_replay() {{ - printf 'replay-ready\n' >> {str(event_log)!r} - exec sleep 30 + exec {sys.executable!r} -c ' +import signal, sys, time +signal.signal(signal.SIGINT, signal.SIG_DFL) +signal.signal(signal.SIGTERM, signal.SIG_DFL) +print("replay-ready", file=open(sys.argv[1], "a"), flush=True) +time.sleep(30) +' {str(event_log)!r} }} trap 'printf "parent-exit\\n" >> {str(event_log)!r}' EXIT trap 'printf "parent-int\\n" >> {str(event_log)!r}; exit 130' INT @@ -301,7 +306,8 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( ) try: # The monitor starts before the production signal traps are installed. - # Wait for replay so the signal actually exercises those traps. + # Publish readiness from the execed process after restoring signal handling; + # a shell marker before exec races with the group SIGINT. deadline = time.monotonic() + 5 while time.monotonic() < deadline: if event_log.exists() and "replay-ready" in event_log.read_text().splitlines(): From 8f893424639d57ea1b60318940f1cde98ce64b81 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 13:34:00 -0700 Subject: [PATCH 5/5] fix: recheck llm-d completion after engine shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：工作节点检测到引擎退出时再次读取完成标记，避免协调节点正常结束被误判为失败；增加成功、非零状态及缺失标记的回归测试。 --- benchmarks/multi_node/llm-d/server.sh | 7 ++++++- runners/test_llmd_lifecycle.py | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index b38940a859..e77ac6f4c5 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -649,7 +649,12 @@ PY exit "$BENCH_RC" else while [[ ! -f "$BENCH_DONE_MARKER" ]]; do - kill -0 "$VLLM_PID" 2>/dev/null || exit 1 + if ! kill -0 "$VLLM_PID" 2>/dev/null; then + # The coordinator may publish completion and stop the engine + # between the marker check and this process check. + [[ -f "$BENCH_DONE_MARKER" ]] && break + exit 1 + fi sleep 2 done exit "$(cat "$BENCH_DONE_MARKER")" diff --git a/runners/test_llmd_lifecycle.py b/runners/test_llmd_lifecycle.py index 0b21452490..a6e2efd2b5 100644 --- a/runners/test_llmd_lifecycle.py +++ b/runners/test_llmd_lifecycle.py @@ -88,3 +88,27 @@ def test_llmd_node_records_status_and_stops_owned_server(tmp_path, node_rc): capture_output=True, text=True, timeout=5) assert result.returncode == node_rc, result.stderr assert (tmp_path / 'done').read_text().strip() == str(node_rc) + + +@pytest.mark.parametrize('coordinator_rc', [0, 7, None]) +def test_llmd_worker_rechecks_completion_when_engine_stops(tmp_path, coordinator_rc): + worker = SERVER.read_text().rsplit('\nelse\n', 1)[1].rsplit('\nfi', 1)[0] + command = r''' +BENCH_DONE_MARKER="$1/done" +VLLM_PID=123 +kill() { + # The loop already observed no marker. Publish the coordinator's result + # before reporting the distributed engine shutdown at its next PID check. + [[ ! -f "$BENCH_DONE_MARKER" ]] || return 99 + if [[ "$2" != 123 ]]; then return 99; fi + if [[ "$COORDINATOR_RC" != missing ]]; then + printf '%s\n' "$COORDINATOR_RC" > "$BENCH_DONE_MARKER" + fi + return 1 +} +''' + worker + env = {**os.environ, 'COORDINATOR_RC': str(coordinator_rc) if coordinator_rc is not None else 'missing'} + result = subprocess.run(['bash', '-c', command, 'bash', str(tmp_path)], env=env, + capture_output=True, text=True, timeout=5) + expected = coordinator_rc if coordinator_rc is not None else 1 + assert result.returncode == expected, result.stderr