diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 81f4cdb806..1dea154aa2 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -538,6 +538,7 @@ jobs: ${{ env.RESULT_FILENAME }}_*.json agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json + slurm_job_*_outcome.txt LOGS/power/** LOGS/native_power/** result_processing_${{ env.RESULT_FILENAME }}.json diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 529b04b665..7c20674e5f 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -474,6 +474,7 @@ jobs: gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json + slurm_job_*_outcome.txt results/gpu_metrics*.csv results/gpu_metrics*_context.json results/gpu_metrics_identity.json diff --git a/.github/workflows/test-changelog-gate.yml b/.github/workflows/test-changelog-gate.yml index 7d49c2ff11..d88effabec 100644 --- a/.github/workflows/test-changelog-gate.yml +++ b/.github/workflows/test-changelog-gate.yml @@ -31,6 +31,8 @@ on: - "runners/launch_*.sh" - "runners/slurm_utils.sh" - "runners/test_slurm_utils.py" + - "runners/test_llmd_lifecycle.py" + - "benchmarks/multi_node/llm-d/**" - "utils/ci_priority.py" - "utils/test_ci_priority.py" - ".github/workflows/reuse-sweep-comment.yml" @@ -123,4 +125,5 @@ jobs: utils/evals/test_batched_eval.py \ utils/evals/test_run_eval_dispatch.py \ runners/test_slurm_utils.py \ + runners/test_llmd_lifecycle.py \ -v -n 4 diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm index 536507bee5..e3338c31dd 100644 --- a/benchmarks/multi_node/llm-d/job.slurm +++ b/benchmarks/multi_node/llm-d/job.slurm @@ -63,25 +63,11 @@ export DOCKER_CONT_NAME : "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}" DOCKER_MOUNT_PATH="/workspace" -cleanup() { - echo "[${SLURM_JOB_ID}] cleanup on $(hostname)" - [[ -n "${WATCHER_PID:-}" ]] && kill "$WATCHER_PID" 2>/dev/null || true -} -trap cleanup INT TERM HUP EXIT - -# Coordinator-done watcher. server.sh on the decode coordinator writes -# this marker after the bench finishes; we then scancel the allocation -# from outside the container (the image has no SLURM client tools). -# Without this, workers `wait` on local vLLM forever and the job runs -# to TIME_LIMIT. +# Workers exit through server.sh after reading the coordinator's final status. +# Normal completion must not cancel an otherwise successful allocation. BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" rm -f "$BENCH_DONE_MARKER" -( - while [[ ! -f "$BENCH_DONE_MARKER" ]]; do sleep 5; done - echo "[${SLURM_JOB_ID}] coordinator finished; scancel'ing job" - scancel "$SLURM_JOB_ID" 2>/dev/null || true -) & -WATCHER_PID=$! +main_rc=0 # Container engine: 'docker' (default) for clusters where the SLURM # user can talk to /var/run/docker.sock (e.g. h200-dgxc-slurm); 'pyxis' @@ -173,7 +159,7 @@ exec docker run --rm \ ${DOCKER_MOUNT_PATH}/benchmarks/multi_node/llm-d/server.sh \ 2>&1 | tee /benchmark_logs/slurm_job-'\"\$SLURM_JOB_ID\"'_rank_'\"\$SLURM_PROCID\"'.log ' -" +" || main_rc=$? srun bash -c "docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f" || true @@ -257,9 +243,11 @@ export MODEL_DIR=/models export BENCHMARK_LOGS_DIR=/benchmark_logs '"$DOCKER_MOUNT_PATH"'/benchmarks/multi_node/llm-d/server.sh \ 2>&1 | tee /benchmark_logs/slurm_job-${SLURM_JOB_ID}_rank_${SLURM_PROCID}.log -' +' || main_rc=$? else echo "Unsupported LLMD_CONTAINER_ENGINE: $LLMD_CONTAINER_ENGINE (expected docker|pyxis)" >&2 exit 1 fi + +exit "$main_rc" diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 89894b210c..e77ac6f4c5 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -114,6 +114,27 @@ else DP_ADDR="$DECODE_DP_ADDR" fi +# One coordinator publishes the benchmark status; workers leave normally when +# it finishes so Slurm can distinguish success from cancellation. +BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" +BENCH_RC=0 + +finish_llmd_node() { + local rc=$? pid + trap - EXIT + if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then + printf '%s\n' "$rc" > "$BENCH_DONE_MARKER.tmp" && + mv -f "$BENCH_DONE_MARKER.tmp" "$BENCH_DONE_MARKER" || rc=1 + fi + for pid in "${ENVOY_PID:-}" "${EPP_PID:-}" "${SIDECAR_PID:-}" "${VLLM_PID:-}"; do + [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true + done + exit "$rc" +} +trap finish_llmd_node EXIT +trap 'exit 143' TERM HUP +trap 'exit 130' INT + DP_SIZE_LOCAL="$GPUS_PER_NODE" START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) @@ -327,11 +348,7 @@ fi # ================================================================ # Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval # ================================================================ -if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then - - # Release the allocation whenever the coordinator exits. - BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" - trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT +if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then # ---- Write endpoints.yaml (file-discovery) ---- # namespace must match EPP's --pool-namespace (file-discovery filters by it; @@ -582,10 +599,8 @@ PY ) fi - # Non-fatal: a failed or timed-out conc point must not abort the sweep - # or (under set -e) skip the allocation release below. The EXIT trap - # releases the allocation regardless, but continuing here lets a - # multi-conc sweep record every point it can. + # Continue collecting available points after a failure, retaining the + # nonzero verdict for the coordinator's final status and worker shutdown. run_benchmark_serving \ --bench-serving-dir /workspace \ --tokenizer /models \ @@ -600,7 +615,7 @@ PY --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \ --result-dir "$BENCHMARK_LOGS_DIR/" \ "${bench_extra_args[@]}" \ - || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" + || { BENCH_RC=$?; echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$BENCH_RC)"; } done fi @@ -631,10 +646,16 @@ PY ) fi - # Signal job.slurm (outside the container, where scancel exists) to release - # the allocation; without it workers wait until TIME_LIMIT. - touch "$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" + exit "$BENCH_RC" else - # Workers (prefill leader, prefill/decode workers): keep vLLM alive. - wait + while [[ ! -f "$BENCH_DONE_MARKER" ]]; do + if ! kill -0 "$VLLM_PID" 2>/dev/null; then + # The coordinator may publish completion and stop the engine + # between the marker check and this process check. + [[ -f "$BENCH_DONE_MARKER" ]] && break + exit 1 + fi + sleep 2 + done + exit "$(cat "$BENCH_DONE_MARKER")" fi diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index b24c0f625a..2984b18d6f 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -114,6 +114,10 @@ Processing and diagnostic power-audit uploads run after launcher or validation f The native collector sets UTC and records context beside its CSV for portable replay; existing benchmark monitors keep their current behavior. Its launcher integration requires separate hardware qualification. The offline adapter accepts this context without changing producers. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts. +### Slurm completion receipts + +Shared Slurm waiting verifies the terminal allocation state and exit code, consulting `scontrol` when `sacct` is missing or non-terminal and retaining `slurm_job_*_outcome.txt`. Launchers stage available evidence before returning failure. llm-d workers exit using the coordinator’s atomically published status-bearing completion marker; normal completion no longer cancels the allocation. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index a10b82e22b..eece846db2 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -114,6 +114,10 @@ PR changelog 选择具有代表性的 NVIDIA 和 AMD 覆盖,并非所有受影 原生采集器单独设置 UTC,并在 CSV 旁记录上下文以支持跨环境回放;现有基准监控行为保持不变。启动器接入需要另行完成硬件验证。离线适配器接受该上下文,不改变现有生产端。正式窗口外的无效样本不能构成覆盖;`boundary_degenerate_rows` 保留其逐 GPU 计数。 +### Slurm 完成状态文件 + +共享 Slurm 等待逻辑检查分配的最终状态和退出码;当 `sacct` 记录缺失或尚未进入最终状态时查询 `scontrol`,并保留 `slurm_job_*_outcome.txt`。启动器先保存已有证据再返回失败。llm-d 工作进程根据协调进程原子发布的完成状态退出;正常结束不再取消 Slurm 分配。 + ## 评测工件 ### 单配置身份和收集 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 187e02fee7..6b2f08fd40 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7467,3 +7467,15 @@ - "Collect shared measured GPU power for the existing two-node GB200 GLM-5.2 aggregate AgentX recipe with producer 80d7203e424f903c9017de4608ee2044afce9574, exact selected-concurrency windows, and post-job adapter validation. Preserve native status and invalid diagnostics before failure. Ordinary Slurm limits, serving settings, other AgentX recipes, and fixed-sequence producer pins are unchanged." - "为现有双节点 GB200 GLM-5.2 聚合 AgentX 配方接入共享 GPU 实测功耗,按所选并发绑定测量窗口,并在任务结束后通过适配器校验。返回失败前保留原生状态及无效诊断;普通 Slurm 时限、服务配置、其他 AgentX 配方和固定序列 producer 版本保持不变。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2928 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + - dsv4-fp4-gb200-llmd-vllm + scenario-type: + - fixed-seq-len + description: + - "Verify terminal Slurm receipts and exit llm-d workers normally after preserving available evidence." + - "验证 Slurm 最终状态,并在保留已有证据后正常退出 llm-d 工作进程。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3052 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 5f822ec530..f70cbab3f8 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -449,10 +449,12 @@ EOF tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID + SRT_JOB_RC=0 + verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -504,6 +506,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else SQUASH_FILE="/data/home/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index fb95bd7ae4..b3e74e9e2e 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -397,13 +397,10 @@ LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" # streams until the job leaves the queue. SRT_JOB_RC=0 stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? -if [[ "$SRT_JOB_RC" != "0" && "$USES_AGENTX_POWER" != "1" ]]; then - exit "$SRT_JOB_RC" -fi set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -428,7 +425,7 @@ fi cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -if [[ "$AGENTX_POWER_RC" != "0" ]]; then +if [[ "$AGENTX_POWER_RC" != "0" && "$SRT_JOB_RC" == "0" ]]; then echo "ERROR: AgentX power validation failed; available audit and server artifacts were staged" >&2 exit "$AGENTX_POWER_RC" fi @@ -487,3 +484,5 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index c0c959d252..e68951f730 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -338,10 +338,12 @@ echo "Tailing LOG_FILE: $LOG_FILE" tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID +SRT_JOB_RC=0 +verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -393,6 +395,8 @@ for i in 1 2 3 4 5; do done find . -name '.nfs*' -delete 2>/dev/null || true +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else # HF_HUB_CACHE is set to help with dataset download inside the container # for eval jobs. diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index d122051bec..da1cf38f57 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -159,7 +159,8 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then trap 'bundle_server_logs "$BENCHMARK_LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true' EXIT INT TERM HUP LOG_FILE="${BENCHMARK_LOGS_DIR}/slurm_job-${JOB_ID}.out" - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + SRT_JOB_RC=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? while IFS= read -r -d '' result_file; do copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 @@ -174,7 +175,7 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then fi scancel "$JOB_ID" 2>/dev/null || true - exit 0 + exit "$SRT_JOB_RC" fi # MODEL_PATH: Override with pre-downloaded paths on GB200 runner @@ -852,14 +853,12 @@ LOGS_DIR="outputs/$JOB_ID/logs" LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" AGENTX_POWER_RC=0 -stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || AGENTX_POWER_RC=$? -if [[ "$AGENTX_POWER_RC" != "0" && "$USES_AGENTX_POWER" != "1" ]]; then - exit 1 -fi +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID finished!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then @@ -916,7 +915,7 @@ else echo "Warning: Logs directory not found at $LOGS_DIR" fi -if [[ "$AGENTX_POWER_RC" != "0" ]]; then +if [[ "$AGENTX_POWER_RC" != "0" && "$SRT_JOB_RC" == "0" ]]; then echo "ERROR: AgentX job or power validation failed; available audit and server artifacts were staged" >&2 exit "$AGENTX_POWER_RC" fi @@ -987,3 +986,5 @@ fi if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then copy_eval_artifacts "$LOGS_DIR/eval_results" "$GITHUB_WORKSPACE" || exit 1 fi + +exit "$SRT_JOB_RC" diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 9249e91f27..fede5b1f60 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -669,10 +669,12 @@ echo "Tailing LOG_FILE: $LOG_FILE" tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID +SRT_JOB_RC=0 +verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ -d "$LOGS_DIR" ]; then @@ -745,3 +747,5 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index af43c740c0..de50907f6a 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -195,10 +195,12 @@ EOF tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID + SRT_JOB_RC=0 + verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -244,6 +246,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else HF_HUB_CACHE_MOUNT="/mnt/nfs/sa-shared/gharunners/hf-hub-cache/" diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 95f519fd3f..dbc2d5f58d 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -348,11 +348,12 @@ EOF LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" trap 'rc=$?; bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT INT TERM HUP - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + SRT_JOB_RC=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -422,6 +423,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index accd850219..6e07dad917 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -123,6 +123,10 @@ if [[ "$IS_MULTINODE" == "true" ]]; then wait $POLL_PID + source "$GITHUB_WORKSPACE/runners/slurm_utils.sh" + slurm_outcome_rc=0 + verify_slurm_job_completion "$JOB_ID" || slurm_outcome_rc=$? + set -x # FIXME: The below is bad and is a result of the indirection of the ways in which @@ -249,6 +253,7 @@ PY sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true # Log preservation and cleanup handled by EXIT trap (cleanup_and_save_logs) + exit "$slurm_outcome_rc" else diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 49f33f6b22..8360fe0f83 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -45,6 +45,32 @@ slurm_job_is_active() { squeue -j "$job_id" --noheader 2>/dev/null | grep -q "$job_id" } +verify_slurm_job_completion() { + local job_id="$1" records record state="" exit_code="" + # Disappearance from squeue only means the job is no longer active. Read + # the allocation's terminal record, not a successful batch/extern step. + records=$(sacct -j "$job_id" --noheader --parsable2 --format=JobIDRaw,State,ExitCode 2>/dev/null) || records="" + record=$(printf '%s\n' "$records" | awk -F'|' -v job="$job_id" '$1 == job {print; exit}') + if [[ -n "$record" ]]; then + IFS='|' read -r _ state exit_code <<< "$record" + fi + case "$state" in + COMPLETED|FAILED|CANCELLED*|TIMEOUT|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED|DEADLINE|BOOT_FAIL) ;; + *) + # Accounting may be missing or still report RUNNING after squeue + # empties. Ask the controller before judging the final outcome. + record=$(scontrol show job "$job_id" --oneliner 2>/dev/null) || record="" + state=$(printf '%s\n' "$record" | sed -n 's/.*JobState=\([^ ]*\).*/\1/p') + exit_code=$(printf '%s\n' "$record" | tr ' ' '\n' | sed -n 's/^ExitCode=//p') + ;; + esac + printf '%s\n' "$record" > "${GITHUB_WORKSPACE:-.}/slurm_job_${job_id}_outcome.txt" + if [[ "$state" != "COMPLETED" || "$exit_code" != "0:0" ]]; then + echo "ERROR: Slurm job $job_id ended with state=${state:-unknown} exit=${exit_code:-unknown}" >&2 + return 1 + fi +} + stream_slurm_job_log() { local job_id="$1" local log_file="$2" @@ -52,7 +78,7 @@ stream_slurm_job_log() { while [[ ! -f "$log_file" ]]; do if ! slurm_job_is_active "$job_id"; then echo "ERROR: job $job_id failed before creating $log_file" >&2 - scontrol show job "$job_id" || true + verify_slurm_job_completion "$job_id" || true return 1 fi sleep 5 @@ -68,6 +94,7 @@ stream_slurm_job_log() { echo "Tailing $log_file" tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null wait "$poll_pid" + verify_slurm_job_completion "$job_id" } copy_to_workspace() { diff --git a/runners/test_llmd_lifecycle.py b/runners/test_llmd_lifecycle.py new file mode 100644 index 0000000000..a6e2efd2b5 --- /dev/null +++ b/runners/test_llmd_lifecycle.py @@ -0,0 +1,114 @@ +"""Exercise llm-d shutdown with local stand-ins for the Slurm/container boundary.""" +import os +from pathlib import Path +import re +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' +SERVER = ROOT / 'benchmarks/multi_node/llm-d/server.sh' + + +@pytest.mark.parametrize('main_rc', [0, 7]) +def test_llmd_completed_coordinator_does_not_cancel_allocation(tmp_path, main_rc): + cwd = tmp_path / 'repo/benchmarks/multi_node/llm-d' + cwd.mkdir(parents=True) + model, logs, bindir = tmp_path / 'model', tmp_path / 'logs', tmp_path / 'bin' + for path in (model, logs, bindir): + path.mkdir() + squash = tmp_path / 'image.sqsh' + squash.write_text('fixture image') + commands = { + 'scontrol': '#!/bin/sh\nprintf "node-a\\nnode-b\\n"\n', + 'scancel': '#!/bin/sh\necho unexpected-cancel >> "$CANCEL_RECEIPT"\n', + 'sleep': '#!/bin/sh\n/bin/sleep 0.01\n', + 'srun': '#!' + sys.executable + '\n' + r''' +import os, pathlib, subprocess, sys, time +args = sys.argv[1:] +if any(a.startswith('--container-image=') for a in args): + marker = pathlib.Path(os.environ['BENCHMARK_LOGS_DIR']) / ('.bench_done.' + os.environ['SLURM_JOB_ID']) + marker.write_text(os.environ['MAIN_RC'] + '\n') + time.sleep(0.2) + sys.exit(int(os.environ['MAIN_RC'])) +if 'ip route' in ' '.join(args): + print('127.0.0.1') + sys.exit(0) +args = [arg for arg in args if not arg.startswith('--')] +sys.exit(subprocess.run(args).returncode) +''', + } + for name, script in commands.items(): + path = bindir / name + path.write_text(script) + path.chmod(0o755) + defaults = dict.fromkeys('PREFILL_WORKERS DECODE_WORKERS PREFILL_DP_SIZE DECODE_DP_SIZE ' + 'BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_MAX_CONCURRENCY ' + 'BENCH_REQUEST_RATE BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER ' + 'RUN_EVAL EVAL_ONLY EVAL_CONC EVAL_FRAMEWORK EVAL_LIMIT EVAL_SUITE ' + 'SWEBENCH_GEN_MODE SWEBENCH_USE_MODAL MODAL_TOKEN_ID MODAL_TOKEN_SECRET ' + 'IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX ' + 'RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE'.split(), '1') + env = {**os.environ, **defaults, 'PATH': str(bindir) + os.pathsep + os.environ['PATH'], + 'SLURM_JOB_ID': 'local-fixture', 'SLURM_JOB_NODELIST': 'node-[a-b]', + 'NUM_NODES': '2', 'PREFILL_NODES': '1', 'DECODE_NODES': '1', 'GPUS_PER_NODE': '2', + 'MODEL_DIR': str(model), 'MODEL_NAME': 'fixture', 'BENCHMARK_LOGS_DIR': str(logs), + 'LLMD_CONTAINER_ENGINE': 'pyxis', 'LLMD_SQUASH_FILE': str(squash), + 'MAIN_RC': str(main_rc), 'CANCEL_RECEIPT': str(tmp_path / 'cancelled')} + result = subprocess.run(['bash', str(JOB)], cwd=cwd, env=env, + capture_output=True, text=True, timeout=10) + assert result.returncode == main_rc, result.stderr + result.stdout + assert not (tmp_path / 'cancelled').exists() + + +@pytest.mark.parametrize('node_rc', [0, 7]) +def test_llmd_node_records_status_and_stops_owned_server(tmp_path, node_rc): + function = re.search(r'^finish_llmd_node\(\) \{\n.*?^\}', + SERVER.read_text(), flags=re.MULTILINE | re.DOTALL).group() + command = function + r''' +NODE_RANK=1 PREFILL_NODES=1 +BENCH_DONE_MARKER="$1/done" +# Observe the publication boundary after shell redirection opens its target, +# but before printf can write the status seen by workers. +printf() { + if [[ -e "$BENCH_DONE_MARKER" ]]; then + echo 'worker can observe an incomplete status' >&2 + return 1 + fi + builtin printf "$@" +} +sleep 60 & +VLLM_PID=$! +trap finish_llmd_node EXIT +exit "$2" +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(tmp_path), str(node_rc)], + capture_output=True, text=True, timeout=5) + assert result.returncode == node_rc, result.stderr + assert (tmp_path / 'done').read_text().strip() == str(node_rc) + + +@pytest.mark.parametrize('coordinator_rc', [0, 7, None]) +def test_llmd_worker_rechecks_completion_when_engine_stops(tmp_path, coordinator_rc): + worker = SERVER.read_text().rsplit('\nelse\n', 1)[1].rsplit('\nfi', 1)[0] + command = r''' +BENCH_DONE_MARKER="$1/done" +VLLM_PID=123 +kill() { + # The loop already observed no marker. Publish the coordinator's result + # before reporting the distributed engine shutdown at its next PID check. + [[ ! -f "$BENCH_DONE_MARKER" ]] || return 99 + if [[ "$2" != 123 ]]; then return 99; fi + if [[ "$COORDINATOR_RC" != missing ]]; then + printf '%s\n' "$COORDINATOR_RC" > "$BENCH_DONE_MARKER" + fi + return 1 +} +''' + worker + env = {**os.environ, 'COORDINATOR_RC': str(coordinator_rc) if coordinator_rc is not None else 'missing'} + result = subprocess.run(['bash', '-c', command, 'bash', str(tmp_path)], env=env, + capture_output=True, text=True, timeout=5) + expected = coordinator_rc if coordinator_rc is not None else 1 + assert result.returncode == expected, result.stderr diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 31fa6f2183..7d52e1f661 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -631,3 +631,70 @@ def test_mi355x_agentic_model_mount_and_routing( script = f"benchmarks/single_node/agentic/{prefix}_fp4_mi355x_vllm_mtp.sh" assert args[-2] == script assert (REPO_ROOT / script).is_file() + + +@pytest.mark.parametrize('state,exit_code,expected', [ + ('COMPLETED', '0:0', 0), ('FAILED', '1:0', 1), ('TIMEOUT', '0:15', 1), + ('COMPLETED', '1:0', 1), ('CANCELLED', '0:15', 1), +]) +def test_slurm_terminal_allocation_status_is_required(tmp_path, state, exit_code, expected): + result = run_bash( + f'source "$1"; export GITHUB_WORKSPACE="$2"; ' + f'sacct() {{ printf "42|{state}|{exit_code}\\n42.batch|COMPLETED|0:0\\n"; }}; ' + 'scontrol() { echo "JobId=42 JobState=COMPLETED ExitCode=0:0"; }; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == f'42|{state}|{exit_code}' + + +@pytest.mark.parametrize('accounting_state,controller_state,exit_code,expected', [ + ('', 'COMPLETED', '0:0', 0), + ('RUNNING', 'COMPLETED', '0:0', 0), + ('COMPLETING', 'COMPLETED', '0:0', 0), + ('RUNNING', 'FAILED', '1:0', 1), + ('RUNNING', 'RUNNING', '0:0', 1), +]) +def test_slurm_recent_completion_falls_back_to_controller( + tmp_path, accounting_state, controller_state, exit_code, expected, +): + record = f'42|{accounting_state}|0:0' if accounting_state else '' + controller = f'JobId=42 JobState={controller_state} ExitCode={exit_code}' + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; ' + f'sacct() {{ echo "{record}"; }}; ' + f'scontrol() {{ echo "{controller}"; }}; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == controller + + +def test_slurm_unknown_terminal_state_is_not_success(tmp_path): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; sacct() { return 1; }; ' + 'scontrol() { return 1; }; verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == 1 + assert 'state=unknown' in result.stderr + + +def test_slurm_exit_before_log_retains_terminal_receipt(tmp_path): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; ' + 'slurm_job_is_active() { return 1; }; ' + 'sacct() { printf "42|FAILED|1:0\\n"; }; ' + 'stream_slurm_job_log 42 "$2/missing.log"', SLURM_UTILS, tmp_path, + ) + assert result.returncode == 1 + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == '42|FAILED|1:0' + + +@pytest.mark.parametrize('exit_code,derived,expected', [('0:0', '7:0', 0), ('1:0', '0:0', 1)]) +def test_slurm_controller_checks_allocation_not_derived_exit(tmp_path, exit_code, derived, expected): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; sacct() { return 1; }; ' + f'scontrol() {{ echo "JobId=42 JobState=COMPLETED ExitCode={exit_code} DerivedExitCode={derived}"; }}; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr diff --git a/utils/test_gb300_power_official_contract.py b/utils/test_gb300_power_official_contract.py index 422b885427..ded89814c4 100644 --- a/utils/test_gb300_power_official_contract.py +++ b/utils/test_gb300_power_official_contract.py @@ -311,12 +311,15 @@ def test_gb200_native_status_waits_for_terminal_and_fails_closed( assert (tmp_path / "logs/power/native-job-status-attempts.txt").read_text() == f"{attempts}\n" -def test_gb200_agentx_window_injection_and_failure_artifacts(tmp_path: Path) -> None: +@pytest.mark.parametrize("stream_rc,expected_rc", [(0, 1), (1, 1), (7, 7)]) +def test_gb200_agentx_window_injection_and_failure_artifacts( + tmp_path: Path, stream_rc: int, expected_rc: int +) -> None: launcher = (REPO_ROOT / "runners/launch_gb200-nv.sh").read_text() injection_end = launcher.index("# Don't leak the login-node venv") injection_start = launcher.rfind('if [[ "$USES_AGENTX_POWER" == "1" ]]; then', 0, injection_end) collection_start = launcher.index("AGENTX_POWER_RC=0") - collection_end = launcher.index('\nif [[ "${EVAL_ONLY:-false}" != "true" ]]; then', collection_start) + collection_end = len(launcher) workspace, compute, producer = (tmp_path / name for name in ("workspace", "compute", "producer")) for path in (workspace, compute, producer): path.mkdir() @@ -335,19 +338,19 @@ def test_gb200_agentx_window_injection_and_failure_artifacts(tmp_path: Path) -> env = os.environ.copy() env.update(GITHUB_WORKSPACE=str(workspace), INFMAX_WORKSPACE=str(compute), PYTHONPATH=str(REPO_ROOT), USES_AGENTX_POWER="1", USES_DCGM_POWER="1", - CONFIG_PATH=str(recipe), CONC_LIST="1", RESULT_FILENAME="result", EVAL_ONLY="false", + CONFIG_PATH=str(recipe), CONC_LIST="1", RESULT_FILENAME="result", EVAL_ONLY="false", IS_AGENTIC="1", JOB_ID="42", LOGS_DIR="outputs/42/logs", LOG_FILE="outputs/42/logs/sweep_42.log", AGENTX_POWER_SRT_SLURM_PIN=AGENTX_PRODUCER_PIN) result = subprocess.run(["/bin/bash"], input=( "set -euo pipefail\n" f"source {shlex.quote(str(REPO_ROOT / 'runners/slurm_utils.sh'))}\n" f"python3() {{ {shlex.quote(sys.executable)} \"$@\"; }}\n" - "stream_slurm_job_log() { return 1; }\n" + f"stream_slurm_job_log() {{ return {stream_rc}; }}\n" "sacct() { printf '42|FAILED|1:0\\n'; }\n" + launcher[injection_start:injection_end] + launcher[collection_start:collection_end] ), text=True, capture_output=True, cwd=producer, env=env) - assert result.returncode == 1, result.stderr + assert result.returncode == expected_rc, result.stderr assert yaml.safe_load(recipe.read_text())["benchmark"]["concurrencies"] == [1] aggregate = json.loads((workspace / "result_conc1.json").read_text()) assert aggregate["power_valid"] == 0