diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index b761d71c4..d3c32fd5c 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -10,6 +10,9 @@ on: - '.github/workflows/e2e-tests.yml' - '.github/workflows/test-process-result.yml' - 'benchmarks/benchmark_lib.sh' + - 'benchmarks/multi_node/amd_utils/**' + - 'runners/test_amd_monitor_wiring.py' + - 'runners/test_amd_power_lifecycle.py' - 'benchmarks/native_power_collect.sh' - 'benchmarks/native_power_lifecycle.sh' - 'runners/test_native_collector_barriers.py' @@ -67,7 +70,7 @@ jobs: run: | cd utils uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \ - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py ../runners/test_tilert_power_lifecycle.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py ../runners/test_amd_monitor_wiring.py ../runners/test_amd_power_lifecycle.py ../runners/test_tilert_power_lifecycle.py -v - name: Test serving client result persistence run: | diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 305a118ef..5c387f0d5 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -153,8 +153,12 @@ unset _benchmark_caller # -------------------------------- GPU_MONITOR_PID="" +GPU_MONITOR_SOURCE_PID="" +GPU_MONITOR_PIPE="" GPU_MONITOR_VENDOR="" GPU_MONITOR_INTERVAL=1 +# Bounded wait for AMD telemetry to cover a stop request; 0 skips the wait. +AMD_MONITOR_STOP_TIMEOUT_S="${AMD_MONITOR_STOP_TIMEOUT_S:-30}" GPU_METRICS_CSV="${GPU_METRICS_CSV:-gpu_metrics.csv}" NVIDIA_GPU_MONITOR_QUERY="timestamp,index,power.draw,temperature.gpu,clocks.current.sm,clocks.current.memory,utilization.gpu,utilization.memory" export GPU_METRICS_CSV @@ -196,8 +200,24 @@ start_gpu_monitor() { # Python; measured on MI355X: trailing ticks were lost at kill without it). # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated # headers, and flush every row so killing the pipe cannot discard buffered samples. - PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ - | awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" & + # Track both processes: killing only awk can leave amd-smi alive until + # its next write. Keep the FIFO beside this run's raw CSV, never shared. + GPU_MONITOR_PIPE="${output}.pipe.$$" + if ! mkfifo "$GPU_MONITOR_PIPE"; then + echo '[GPU Monitor] Warning: AMD telemetry FIFO is unavailable' >&2 + # A colliding path may belong to another stream; teardown must not remove it. + GPU_MONITOR_PIPE="" + GPU_MONITOR_VENDOR="" + case "${REQUIRE_POWER:-0}" in + 1|true|TRUE|yes|YES) return 1 ;; + esac + return 0 + fi + PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv \ + > "$GPU_MONITOR_PIPE" 2>/dev/null & + GPU_MONITOR_SOURCE_PID=$! + awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' \ + < "$GPU_MONITOR_PIPE" > "$output" & GPU_MONITOR_PID=$! # Hardware energy-accumulator + identity snapshots; the end-side twin in # stop_gpu_monitor lets auditors cross-check the integrated energy @@ -215,19 +235,21 @@ start_gpu_monitor() { # Stop the background GPU monitor and report file size. stop_gpu_monitor() { if [[ -n "$GPU_MONITOR_PID" ]] && kill -0 "$GPU_MONITOR_PID" 2>/dev/null; then - # benchmark_end_time_unix is recorded shortly before the benchmark - # process exits, so the stream must cover one more sample past it for - # deterministic boundary interpolation. NVIDIA appends a one-shot - # post-exit sample below; amd-smi one-shot CSV has no timestamp column, - # so the AMD path instead lets the watch stream emit final ticks before - # the kill. Two extra intervals: amd-smi stamps integer seconds, so a - # tick in the same second as the window end still fails bracketing — - # the stream needs a tick at the NEXT whole second (measured on MI355X: - # end=...153.325 vs last sample ...153.0). + # The aggregator requires, for every GPU, a usable sample stamped at or + # after the (fractional) benchmark window end, which is always <= the + # wall clock when this stop runs. NVIDIA appends a one-shot post-exit + # sample below; amd-smi one-shot CSV has no timestamp column, so the + # AMD path polls the output file until every GPU's watch stream shows + # a usable tick at the next whole second — amd-smi stamps integer + # seconds, so that tick strictly covers any fractional window end + # (measured on MI355X: end=...609.157 vs last sample ...605). Observing + # the file rather than sleeping also defeats pipe-buffer loss when the + # awk consumer is killed: covered rows are already on disk. if [[ "$GPU_MONITOR_VENDOR" == "amd" ]]; then - sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 )) + _wait_for_amd_stop_coverage fi - kill "$GPU_MONITOR_PID" 2>/dev/null + # The monitor may exit during the coverage wait; still finish cleanup. + kill "$GPU_MONITOR_PID" 2>/dev/null || true wait "$GPU_MONITOR_PID" 2>/dev/null || true case "$GPU_MONITOR_VENDOR" in nvidia) @@ -249,6 +271,13 @@ stop_gpu_monitor() { echo "[GPU Monitor] Collected $lines rows -> $GPU_METRICS_CSV" fi fi + if [[ -n "$GPU_MONITOR_SOURCE_PID" ]]; then + kill "$GPU_MONITOR_SOURCE_PID" 2>/dev/null || true + wait "$GPU_MONITOR_SOURCE_PID" 2>/dev/null || true + fi + [[ -z "$GPU_MONITOR_PIPE" ]] || rm -f "$GPU_MONITOR_PIPE" + GPU_MONITOR_SOURCE_PID="" + GPU_MONITOR_PIPE="" GPU_MONITOR_PID="" GPU_MONITOR_VENDOR="" } @@ -271,6 +300,105 @@ _repair_truncated_gpu_metrics_tail() { return 0 } +# Print the newest telemetry tick (whole epoch seconds) that EVERY observed +# GPU has covered with a usable sample (numeric epoch timestamp, numeric +# power > 0), or nothing when the stream holds no usable epoch-stamped row +# (e.g. an amd-smi build emitting ISO timestamps). Column detection mirrors +# _POWER_COL_RE/_POWER_EXCLUDE_RE/_GPU_INDEX_COL_RE in utils/aggregate_power.py. +# POSIX awk only: the ROCm container images ship mawk/busybox awk. +_amd_monitor_min_covered_tick() { + [[ -f "$GPU_METRICS_CSV" ]] || return 0 + awk -F, ' + NR == 1 { + for (i = 1; i <= NF; i++) { + name = tolower($i) + gsub(/^ +| +$/, "", name) + sub(/\r$/, "", name) + if (!power_col && name ~ /power/ && name !~ /limit|cap|max|min/) + power_col = i + if (!gpu_col && name ~ /^(index|gpu|gpu_id|gpu_index|card|device)$/) + gpu_col = i + } + next + } + !power_col || !gpu_col { next } + { + # amd-smi quotes list-valued cells that embed commas; neutralize + # them so the power cell keeps its header-relative position. + line = $0 + sub(/\r$/, "", line) + if (line ~ /"/) { + n = split(line, seg, /"/) + line = "" + for (i = 1; i <= n; i++) { + if (i % 2 == 0) gsub(/,/, ";", seg[i]) + line = line seg[i] + } + } + count = split(line, cell, /,/) + if (count < power_col || count < gpu_col) next + if (cell[1] !~ /^[0-9]+(\.[0-9]+)?$/) next + if (cell[power_col] !~ /^[0-9]+(\.[0-9]+)?$/) next + if (cell[power_col] + 0 <= 0) next + if (cell[gpu_col] == "") next + ts = cell[1] + 0 + # Mirror _parse_timestamp in utils/aggregate_power.py: normalize + # millisecond epochs so a ms-stamping amd-smi build cannot + # trivially satisfy any second-scale stop target. + if (ts > 1e12) ts /= 1000 + gpu = cell[gpu_col] + if (!(gpu in newest) || ts > newest[gpu]) + newest[gpu] = ts + } + END { + have = 0 + for (gpu in newest) + if (!have || newest[gpu] < min) { min = newest[gpu]; have = 1 } + if (have) printf "%d\n", min + } + ' "$GPU_METRICS_CSV" 2>/dev/null + return 0 +} + +# Block until every observed GPU has a usable tick at/after the first whole +# second past stop entry, so any window end preceding the stop request is +# bracketed on file. Bounded by AMD_MONITOR_STOP_TIMEOUT_S; always returns 0 — +# on timeout or early monitor death it warns and lets aggregation attribute +# the missing coverage (fail-safe, never fail-silent). +_wait_for_amd_stop_coverage() { + local target deadline covered timeout_s + # A non-integer timeout (e.g. "30s") would abort the whole stop_gpu_monitor + # call under `set -e` at the arithmetic below, leaking the monitor process + # and skipping tail repair + the energy sidecar; fall back to the default. + timeout_s="${AMD_MONITOR_STOP_TIMEOUT_S:-30}" + if [[ ! "$timeout_s" =~ ^-?[0-9]+$ ]]; then + echo "[GPU Monitor] Warning: ignoring non-integer AMD_MONITOR_STOP_TIMEOUT_S='$timeout_s', using 30" >&2 + timeout_s=30 + fi + if [[ "$timeout_s" -le 0 ]]; then + return 0 + fi + target=$(( $(date +%s) + 1 )) + deadline=$(( target + timeout_s )) + while :; do + covered=$(_amd_monitor_min_covered_tick) + # The first usable row may arrive after stop begins. Keep the same + # deadline for empty or unsupported streams instead of stopping early. + if [[ -n "$covered" && "$covered" -ge "$target" ]]; then + return 0 + fi + if ! _background_process_is_running "$GPU_MONITOR_PID"; then + echo "[GPU Monitor] Warning: AMD monitor exited before covering the stop request (covered=$covered target=$target)" >&2 + return 0 + fi + if [[ "$(date +%s)" -ge "$deadline" ]]; then + echo "[GPU Monitor] Warning: AMD telemetry never covered the stop request within ${timeout_s}s (covered=$covered target=$target)" >&2 + return 0 + fi + sleep 1 + done +} + # Write one best-effort amd-smi snapshot; remove the file rather than keep a # partial one when the invocation fails. _write_amd_smi_sidecar() { @@ -3284,9 +3412,15 @@ run_agentic_replay_and_write_outputs() ( esac _stop_agentx_power_monitor() { + local mode="${1:-}" if [ "$agentx_monitor_stopped" = "0" ]; then - agentx_monitor_stopped=1 + if [ "$mode" = "abort" ]; then + # A cancelled run's power validity is moot; skip the AMD + # coverage wait so signal teardown stays fast. + AMD_MONITOR_STOP_TIMEOUT_S=0 + fi stop_gpu_monitor + agentx_monitor_stopped=1 fi } @@ -3330,10 +3464,11 @@ run_agentic_replay_and_write_outputs() ( agentx_monitor_stopped=0 # This function runs in a subshell, so these handlers cannot replace # launcher-owned traps. The stopped flag keeps explicit and signal/EXIT - # cleanup idempotent. - trap '_stop_agentx_power_monitor' EXIT - trap '_stop_agentx_power_monitor; exit 130' INT - trap '_stop_agentx_power_monitor; exit 143' TERM + # cleanup idempotent after stopping completes. If a signal interrupts + # the normal coverage wait, abort cleanup must still kill the monitor. + trap '_stop_agentx_power_monitor abort' EXIT + trap '_stop_agentx_power_monitor abort; exit 130' INT + trap '_stop_agentx_power_monitor abort; exit 143' TERM fi echo "$REPLAY_CMD" > "$result_dir/benchmark_command.txt" diff --git a/benchmarks/multi_node/amd_utils/bench.sh b/benchmarks/multi_node/amd_utils/bench.sh index 3dde0a68a..357dfdbe8 100755 --- a/benchmarks/multi_node/amd_utils/bench.sh +++ b/benchmarks/multi_node/amd_utils/bench.sh @@ -52,6 +52,16 @@ profile_folder="${log_path}/${ENGINE}_isl_${chosen_isl}_osl_${chosen_osl}" mkdir -p "$profile_folder" source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/power.sh" +power_required=0 +case "${REQUIRE_POWER:-0}" in + 1|true|TRUE|yes|YES) power_required=1 ;; +esac +if ! wait_amd_multinode_power ready; then + [[ "$power_required" == 0 ]] || exit 1 + echo 'PowerX: continuing without ready optional telemetry' >&2 +fi +benchmark_exit_code=0 REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" @@ -101,6 +111,7 @@ for max_concurrency in "${chosen_concurrencies[@]}"; do fi fi + point_exit_code=0 run_benchmark_serving \ --bench-serving-dir "$REPO_ROOT" \ --model "$BENCH_MODEL" \ @@ -113,7 +124,8 @@ for max_concurrency in "${chosen_concurrencies[@]}"; do --max-concurrency "$max_concurrency" \ --result-filename "$export_file" \ --result-dir /workspace/ \ - $extra_flags + $extra_flags || point_exit_code=$? + if [[ "$point_exit_code" != 0 ]]; then benchmark_exit_code=$point_exit_code; break; fi echo "-----------------------------------------" @@ -123,3 +135,10 @@ for max_concurrency in "${chosen_concurrencies[@]}"; do sleep 10 fi done + +# Stop every node while all prefill/decode servers are still alive. +if ! wait_amd_multinode_power done; then + echo 'PowerX: collector completion failed' >&2 + if [[ "$power_required" == 1 && "$benchmark_exit_code" == 0 ]]; then benchmark_exit_code=1; fi +fi +exit "$benchmark_exit_code" diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index cb6f4ad4d..2df345612 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -237,8 +237,9 @@ fi # Node Selection # ============================================================================= -NUM_NODES=$((xP + yD)) -echo "NUM_NODES: $NUM_NODES (xP=$xP + yD=$yD)" +# Workers can span multiple physical nodes. Preserve the submit-time count. +NUM_NODES="${NUM_NODES:-$(( ((PREFILL_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE) * xP + ((DECODE_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE) * yD ))}" +echo "NUM_NODES: $NUM_NODES (prefill workers=$xP, decode workers=$yD)" FULL_NODELIST=$(scontrol show hostnames "$SLURM_JOB_NODELIST") SELECTED_NODES=$(echo "$FULL_NODELIST" | head -n $NUM_NODES) @@ -318,6 +319,10 @@ export DRY_RUN="${DRY_RUN:-0}" export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" export KEEP_CONTAINERS="${KEEP_CONTAINERS:-0}" export ENGINE=$ENGINE +export POWERX_HOST_UID=$(id -u) +export POWERX_HOST_GID=$(id -g) +export POWERX_COLLECTOR_REVISION=$(git -C "$DI_REPO_DIR" rev-parse HEAD) +mkdir -p "$BENCHMARK_LOGS_DIR/power-control-${SLURM_JOB_ID}" # Eval-related env vars (threaded from submit.sh) export RUN_EVAL="${RUN_EVAL:-false}" @@ -377,6 +382,16 @@ else echo "[WARN] $RDMA_CHECK_SCRIPT not found; skipping RDMA QoS/DCQCN pre-flight check" fi +stage_native_power() { + srun --overlap --nodelist="$SELECTED_NODELIST_SRUN" --ntasks="$NUM_NODES" bash -c ' + src="/tmp/slurm_job-${SLURM_JOB_ID}/native_power/node-${SLURM_PROCID}" + if [[ -d "$src" ]]; then + mkdir -p "$BENCHMARK_LOGS_DIR/native_power" + cp -r "$src" "$BENCHMARK_LOGS_DIR/native_power/" + fi + ' +} + cleanup() { echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up container + stale logs..." # Backstop: on scancel/timeout/step-hang the foreground `exec docker run` @@ -386,11 +401,13 @@ cleanup() { # other users' containers. (Ported from InferenceY 51ebfa88.) srun --nodelist="$SELECTED_NODELIST_SRUN" \ bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' 2>/dev/null || true + stage_native_power || true rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true echo "[${SLURM_JOB_ID}] cleanup done." } -trap cleanup INT TERM HUP +trap 'cleanup; exit 130' INT +trap 'cleanup; exit 143' TERM HUP # Force NFS cache refresh on all nodes echo "Refreshing NFS caches on all nodes..." @@ -412,6 +429,12 @@ DOCKER_ENV_COMMON=( -e SLURM_JOB_ID=\$SLURM_JOB_ID -e SLURM_JOB_NODELIST=\$SLURM_JOB_NODELIST -e NNODES=\$NNODES + -e REQUIRE_POWER=\${REQUIRE_POWER:-0} + -e POWERX_HOST_UID=\$POWERX_HOST_UID + -e POWERX_HOST_GID=\$POWERX_HOST_GID + -e POWERX_COLLECTOR_REVISION=\$POWERX_COLLECTOR_REVISION + -e POWERX_NODE_NAME=\$POWERX_NODE_NAME + -e POWERX_CLOCK_SYNCHRONIZED=\$POWERX_CLOCK_SYNCHRONIZED -e NODE_RANK=\$SLURM_PROCID -e NODE0_ADDR=\$NODE0_ADDR -e MODEL_DIR=/models @@ -601,6 +624,10 @@ set -euo pipefail echo \"Rank \$SLURM_PROCID on \$(hostname)\" +# Capture the host's synchronization state, not the container's missing D-Bus. +export POWERX_NODE_NAME=\$(hostname) +export POWERX_CLOCK_SYNCHRONIZED=\$(timedatectl show -p NTPSynchronized --value 2>/dev/null || true) + # Per-node docker privilege detection eval \"\$DOCKER_CMD_DETECT\" echo \"[docker-detect] rank \$SLURM_PROCID: DOCKER_CMD=\$DOCKER_CMD\" @@ -784,6 +811,17 @@ echo \"[rank 0] Main container exited (rc=\$DOCKER_EXIT_CODE). Stopping vllm-rou exit \$DOCKER_EXIT_CODE " +BENCHMARK_STEP_RC=$? +# Each host copies its own node-local, root-created artifacts as the runner user. +# No raw telemetry is written as root into the shared checkout. +if ! stage_native_power; then + echo 'PowerX: failed to stage native telemetry' >&2 + case "${REQUIRE_POWER:-0}" in + 1|true|TRUE|yes|YES) + if [[ "$BENCHMARK_STEP_RC" == 0 ]]; then BENCHMARK_STEP_RC=1; fi ;; + esac +fi + if [[ "${KEEP_CONTAINERS}" != "1" ]]; then srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' '"$CLIENT_CONT_NAME"' 2>/dev/null || true' @@ -794,3 +832,5 @@ if [[ "${KEEP_CONTAINERS}" != "1" ]]; then ' fi fi + +exit "$BENCHMARK_STEP_RC" diff --git a/benchmarks/multi_node/amd_utils/power.sh b/benchmarks/multi_node/amd_utils/power.sh new file mode 100644 index 000000000..18472bb72 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/power.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +start_amd_multinode_power() { + [[ "${BENCH_INPUT_LEN:-}" == 8192 && "${BENCH_OUTPUT_LEN:-}" == 1024 && + "${EVAL_ONLY:-false}" != true && "${IS_AGENTIC:-0}" != 1 && + "${IS_AGENTIC:-false}" != true && "${DRY_RUN:-0}" != 1 ]] || return 0 + local prefill_nodes_per_worker decode_nodes_per_worker prefill_nodes total_nodes role tp worker_nodes gpu_count gpu_indices + prefill_nodes_per_worker=$(( (PREFILL_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE )) + decode_nodes_per_worker=$(( (DECODE_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE )) + prefill_nodes=$(( prefill_nodes_per_worker * xP )) + total_nodes=$(( prefill_nodes + decode_nodes_per_worker * yD )) + [[ "$total_nodes" == "$NNODES" ]] || { echo 'PowerX: inconsistent AMD node topology' >&2; return 1; } + if (( NODE_RANK < prefill_nodes )); then + role=prefill; tp=$PREFILL_TP_SIZE; worker_nodes=$prefill_nodes_per_worker + else + role=decode; tp=$DECODE_TP_SIZE; worker_nodes=$decode_nodes_per_worker + fi + # Distributed tensor parallelism assigns equal local ranks to every node, + # e.g. TP12 across two nodes uses GPU0..5 on each, not 8 GPUs plus 4 GPUs. + (( tp % worker_nodes == 0 )) || { echo 'PowerX: uneven per-node TP layout' >&2; return 1; } + gpu_count=$(( tp / worker_nodes )) + gpu_indices=$(seq 0 $((gpu_count - 1)) | paste -sd, -) + export POWERX_CONTROL_DIR="${BENCHMARK_LOGS_DIR}/power-control-${SLURM_JOB_ID}" + local native_dir="/run_logs/slurm_job-${SLURM_JOB_ID}/native_power/node-${NODE_RANK}" + bash "$WS_PATH/../../native_power_collect.sh" "$native_dir" "$POWERX_CONTROL_DIR" \ + amd "$NODE_RANK" "$role" "$gpu_indices" "$total_nodes" & + POWERX_COLLECTOR_PID=$! + # EXIT remains independent of serving-engine INT/TERM handlers. + trap 'if [[ -n "${POWERX_COLLECTOR_PID:-}" ]]; then kill "$POWERX_COLLECTOR_PID" 2>/dev/null || true; wait "$POWERX_COLLECTOR_PID" 2>/dev/null || true; fi' EXIT +} + +wait_amd_multinode_power() { + local stage=$1 deadline=$((SECONDS + 60)) ready rank + [[ -n "${POWERX_CONTROL_DIR:-}" ]] || return 0 + if [[ "$stage" == done ]]; then + printf 'stop\n' > "$POWERX_CONTROL_DIR/stop" + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$POWERX_CONTROL_DIR/stop" + fi + while (( SECONDS < deadline )); do + ready=1 + for ((rank=0; rank&2; return 1 + fi + [[ -f "$POWERX_CONTROL_DIR/$stage-$rank" ]] || ready=0 + done + if [[ "$ready" == 1 ]]; then + if [[ "$stage" == done ]]; then + for ((rank=0; rank&2 + return 1 +} diff --git a/benchmarks/multi_node/amd_utils/server.sh b/benchmarks/multi_node/amd_utils/server.sh index b62ca5816..15fad699b 100755 --- a/benchmarks/multi_node/amd_utils/server.sh +++ b/benchmarks/multi_node/amd_utils/server.sh @@ -11,6 +11,14 @@ ENGINE="${ENGINE:-sglang-disagg}" WS_PATH="${WS_PATH:-${SGLANG_WS_PATH:-${VLLM_WS_PATH:-${ATOM_WS_PATH:-$(dirname "${BASH_SOURCE[0]}")}}}}" export WS_PATH ENGINE +source "$WS_PATH/power.sh" +if ! start_amd_multinode_power; then + case "${REQUIRE_POWER:-0}" in + 1|true|TRUE|yes|YES) exit 1 ;; + esac + echo 'PowerX: continuing without optional worker telemetry' >&2 +fi + echo "[DISPATCHER] ENGINE=$ENGINE WS_PATH=$WS_PATH" if [[ "$ENGINE" == "vllm-disagg" ]]; then diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 7815e5a91..729389327 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -1064,7 +1064,8 @@ print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { set +x else set -x - eval "$BENCH_CMD" + BENCHMARK_EXIT_CODE=0 + eval "$BENCH_CMD" || BENCHMARK_EXIT_CODE=$? set +x fi @@ -1371,4 +1372,4 @@ else fi echo "Script completed successfully" -exit 0 +exit "${BENCHMARK_EXIT_CODE:-0}" diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 805254c4f..5883b7b98 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -122,6 +122,16 @@ Concurrent cells serialize draft staging with a per-model lock. Each cell lets `hf download` validate or resume the existing cache before serving; a nonempty directory is not a completion signal. +## Native PowerX collection for fixed-sequence multinode runs + +The AMD SGLang/ATOM/vLLM launchers enable native SMI collection for 8192-input/1024-output runs. Every serving node starts `benchmarks/native_power_collect.sh`; the client waits for all `ready-` receipts, then requests `stop` and waits for all `done-` receipts before tearing down servers. The shared collector also accepts NVIDIA SMI for launchers that do not use the srt-slurm/DCGM contract. + +The selected active multi-node qualification uses SGLang. Inactive ATOM/vLLM multi-node exit changes are deferred; single-node ATOM/vLLM checks do not qualify those server paths. Native multi-node AgentX collection remains disabled. + +Keep `native_power/node-/gpu_metrics.csv`, the start/end device identity snapshots and `manifest.json` together. Select the actual serving GPU indices, preserve physical node counts when workers span nodes, and stage node-local files as the host runner user into `LOGS/native_power`. The result processor uses each client's formal window, validates all node/role counts and UUID membership, and reuses the shared integration/percentile math. Aggregate deployments emit whole-deployment metrics; role metrics require real separate prefill/decode pools. + +Host `timedatectl NTPSynchronized` is recorded as clock context. The shared collector accepts `yes` or `true` as synchronized; other or missing values remain unsynchronized. It does not measure the offset between nodes; common-window trace coverage is still required, and runtime clock alignment remains part of fleet qualification. Missing clock context, a replaced UUID, a missing node or an incomplete collector lifecycle makes power unavailable. Local fixtures prove the format and failure behavior, not GPU runtime or dashboard publication. + ## Native TileRT power TileRT's shared importer preserves Docker Hub image names and converts explicit registries such as `ghcr.io/team/image:tag` to Enroot's `docker://ghcr.io#team/image:tag` syntax. Existing `#` references are preserved. Valid cached squash images are reused without importing; a cache hit does not validate the registry import path. Invalid cached images are removed under the import lock before retrying the import. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 900b7a8b9..6d13ca306 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -120,6 +120,16 @@ B300 DSXE 的 Kimi-K3 AgentX 路径在 `/scratch/models` 下挂载预置目标 并发任务通过模型专用锁串行准备草稿权重。每个任务在启动服务前由 `hf download` 校验或续传现有缓存;目录非空不代表下载完成。 +## 固定序列长度多节点运行的原生 PowerX 采集 + +AMD SGLang/ATOM/vLLM launcher 为 8192 输入、1024 输出的运行启用原生 SMI 采集。每个服务节点启动 `benchmarks/native_power_collect.sh`;客户端等待全部 `ready-` 回执,然后在基准结束后请求 `stop`,等待全部 `done-` 回执,再关闭服务。共享采集器也支持 NVIDIA SMI,供未采用 srt-slurm/DCGM 契约的 launcher 使用。 + +当前所选现役多节点验证使用 SGLang。未启用的 ATOM/vLLM 多节点退出改动暂缓;单节点 ATOM/vLLM 检查不能证明这些服务路径。多节点 AgentX 原生采集仍未开启。 + +将 `native_power/node-/gpu_metrics.csv`、开始和结束时的设备身份快照及 `manifest.json` 一起保留。选择实际服务进程使用的 GPU 索引,worker 跨节点时保留真实物理节点数,并由宿主机 runner 用户将节点本地文件暂存到 `LOGS/native_power`。结果处理器使用每个客户端的正式窗口,验证全部节点、角色数量及 UUID 归属,再复用共享积分与百分位计算。聚合部署只输出全部署指标;角色指标要求实际分离的 prefill/decode 池。 + +宿主机的 `timedatectl NTPSynchronized` 状态作为时钟上下文记录。共享采集器将 `yes` 或 `true` 视为已同步;其他值或缺失值均视为未同步。它不测量节点间时钟偏移;仍需验证共同窗口的轨迹覆盖,并在集群运行验证中检查时钟对齐。缺失时钟上下文、UUID 被替换、节点缺失或采集生命周期未完成都会使功耗不可用。本地 fixture 只证明格式和失败处理行为,不能证明 GPU 运行或 dashboard 发布完成。 + ## TileRT 原生功耗 TileRT 的共享导入器保留 Docker Hub 镜像名称,并将 `ghcr.io/team/image:tag` 等显式仓库地址转换为 Enroot 的 `docker://ghcr.io#team/image:tag` 格式。已有的 `#` 地址保持不变。有效的缓存 squash 镜像会直接复用;命中缓存不能证明仓库导入路径有效。无效的缓存镜像会在持有导入锁时删除,再重新导入。 diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index b24c0f625..64162e7d7 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -108,6 +108,10 @@ The PR changelog selects representative NVIDIA and AMD coverage, not an exhausti Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. +### Power telemetry stop and validation + +The single-node AMD monitor in [`benchmark_lib.sh`](../benchmarks/benchmark_lib.sh) waits for every observed GPU to have a positive, numeric power sample at or beyond the first whole second after the stop request. `AMD_MONITOR_STOP_TIMEOUT_S` bounds the wait (default `30`; `0` skips it). Streams with missing or unsupported timestamps keep the same bounded wait for a usable sample. A timeout does not certify coverage: the aggregator still rejects an unbracketed benchmark window. AgentX cancellation skips the coverage wait and stops the monitor, including when cancellation arrives during a normal stop. + ### Native multinode telemetry `native_power_collect.sh` and `native_power_lifecycle.sh` provide per-node collection and bounded ready/stop receipts. Launchers opt into the native package under `LOGS/native_power`; this prerequisite enables no new recipe. The adapter validates serving GPU identity, synchronized clocks, collector completion, and complete formal-window coverage. It preserves per-node failures, sample counts, and collector revision in the audit. diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index a10b82e22..2ab4dcbe6 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -108,6 +108,10 @@ PR changelog 选择具有代表性的 NVIDIA 和 AMD 覆盖,并非所有受影 启动器或验证失败后仍会运行处理和功耗诊断上传,并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功,因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据;这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 +### 功耗遥测停止与校验 + +[`benchmark_lib.sh`](../benchmarks/benchmark_lib.sh) 中的单节点 AMD 监控会等待每个已观测 GPU 都记录到有效的正数功耗样本,且时间戳不早于停止请求之后的第一个整秒。`AMD_MONITOR_STOP_TIMEOUT_S` 限制等待时长(默认 `30`;设为 `0` 可跳过等待)。时间戳缺失或不支持的数据流仍在相同的截止时间内等待有效样本。超时不代表覆盖有效:聚合器仍会拒绝未被样本完整包围的基准窗口。AgentX 取消运行时会跳过覆盖等待并停止监控,即使取消发生在正常停止的等待过程中。 + ### 原生多节点遥测 `native_power_collect.sh` 和 `native_power_lifecycle.sh` 提供每节点采集及有时限的就绪/停止状态文件。启动器可使用 `LOGS/native_power` 下的原生产物;此前置改动不会启用新 recipe。适配器验证服务 GPU 身份、时钟同步、采集完成及正式窗口完整覆盖,并在审计中保留节点故障、样本数和采集器版本。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c2a59cb1b..85bce7ebd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7594,3 +7594,23 @@ - "Avoid recursive TileRT eval dispatch and preserve evaluation failures after artifact staging." - "避免 TileRT 评测分发无限递归,并在保存产物后保留评测失败状态。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3067 + +- config-keys: + - qwen3.5-fp8-mi300x-sglang + - glm5.2-fp8-mi325x-sglang-agentic-mtp + - dsr1-fp4-mi355x-sglang-mtp + - dsr1-fp4-mi355x-atom-mtp + - kimik3-fp4-mi355x-vllm-agentic-mtp + - kimik3-fp4-mi355x-atom-agentic-mtp + - qwen3.5-fp8-mi355x-sglang-disagg + - dsr1-fp8-mi355x-sglang-disagg-mtp + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp + scenario-type: + - fixed-seq-len + - agentic-coding + description: + - Collect native AMD 8K/1K worker power and retain monitor samples through the measurement end; qualify + shared runtime paths with complete selected curves across MI300X, MI325X and MI355X. Retired DSV4 + 8K/1K is excluded. + - 采集 AMD 8K/1K 工作节点功耗并保留测量终点样本;通过 MI300X、MI325X 和 MI355X 所选完整曲线验证共享运行路径,排除已退役 DSV4 8K/1K。 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3055 diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index ecaf414a8..829565d6a 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -59,6 +59,10 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ -n "${GITHUB_ACTIONS:-}" && -n "${JOB_ID:-}" ]]; then local art_dir="$GITHUB_WORKSPACE/benchmark_artifacts" mkdir -p "$art_dir" + if [[ -d "$BENCHMARK_LOGS_DIR/native_power" ]]; then + mkdir -p "$GITHUB_WORKSPACE/LOGS/native_power" + cp -r "$BENCHMARK_LOGS_DIR/native_power/". "$GITHUB_WORKSPACE/LOGS/native_power/" || true + fi cp -r "$BENCHMARK_LOGS_DIR"/slurm_job-${JOB_ID}.{out,err} "$art_dir/" 2>/dev/null || true fi # Print .err inline so failures are visible in CI output @@ -129,6 +133,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # Dynamo jobs are launched. In a follow-up PR, the location of the result file should not # depend on the runner, it should always be in the same spot in the GH workspace. + # Preserve native power evidence before cleanup, even when result processing fails. + if [[ -d "$BENCHMARK_LOGS_DIR/native_power" ]]; then + mkdir -p "$GITHUB_WORKSPACE/LOGS/native_power" + cp -r "$BENCHMARK_LOGS_DIR/native_power/". "$GITHUB_WORKSPACE/LOGS/native_power/" + export POWERX_NATIVE_DIR="$GITHUB_WORKSPACE/LOGS/native_power" + fi + # Process results from all configurations # search for "FRAMEWORK_DIFF_IF_STATEMENT #3" for this if-statement diff --git a/runners/test_amd_monitor_wiring.py b/runners/test_amd_monitor_wiring.py new file mode 100644 index 000000000..102895eb7 --- /dev/null +++ b/runners/test_amd_monitor_wiring.py @@ -0,0 +1,65 @@ +import os +import subprocess +import sys +from pathlib import Path +REPO = Path(__file__).resolve().parents[1] + +def test_amd_stop_coverage_uses_slowest_gpu_and_normalizes_milliseconds(tmp_path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text('timestamp,gpu,vcn_activity,socket_power\n' + '1700000002000,0,"[0, 0]",250\n' + '1700000001000,1,"[0, 0]",250\n' + '1700000009000,1,"[0, 0]",N/A\n') + result = subprocess.run(["bash", "-c", 'source "$1"; GPU_METRICS_CSV="$2"; _amd_monitor_min_covered_tick', + "test", str(REPO / "benchmarks/benchmark_lib.sh"), str(csv)], + capture_output=True, text=True, check=True, timeout=10) + assert result.stdout.strip() == "1700000001" + + +def test_amd_shutdown_kills_both_pipeline_processes_when_stream_dies(tmp_path): + binary = tmp_path / "bin"; binary.mkdir() + fake = binary / "amd-smi" + fake.write_text(f'''#!{sys.executable} +import json, sys, time +if "-w" in sys.argv: + print("timestamp,gpu,socket_power", flush=True) + while True: + print(str(int(time.time())) + ",0,250", flush=True); time.sleep(.1) +else: + print("[]") +''') + fake.chmod(0o755) + csv = tmp_path / "gpu_metrics.csv" + script = '''source "$1" +start_gpu_monitor --output "$2" +source_pid=$GPU_MONITOR_SOURCE_PID +sink_pid=$GPU_MONITOR_PID +kill "$sink_pid" +wait "$sink_pid" 2>/dev/null || true +AMD_MONITOR_STOP_TIMEOUT_S=0 +stop_gpu_monitor +if kill -0 "$source_pid" 2>/dev/null; then echo 'source leaked'; exit 1; fi +if kill -0 "$sink_pid" 2>/dev/null; then echo 'sink leaked'; exit 1; fi +[[ ! -p "$2.pipe.$$" ]] +''' + subprocess.run(["bash", "-c", script, "test", str(REPO / "benchmarks/benchmark_lib.sh"), str(csv)], + env={**os.environ, "PATH": f"{binary}:{os.environ['PATH']}"}, + capture_output=True, text=True, check=True, timeout=10) + + +def test_amd_multinode_selects_equal_local_tensor_ranks(tmp_path): + arguments = tmp_path / "collector.args" + # Mock only the downstream collector command; exercise actual topology routing. + script = f'''source {str(REPO / 'benchmarks/multi_node/amd_utils/power.sh')!r} +bash() {{ printf '%s\\0' "$@" > {str(arguments)!r}; }} +start_amd_multinode_power +wait "$POWERX_COLLECTOR_PID" +''' + subprocess.run(["bash", "-c", script], env={**os.environ, "BENCH_INPUT_LEN": "8192", + "BENCH_OUTPUT_LEN": "1024", "PREFILL_TP_SIZE": "12", "DECODE_TP_SIZE": "12", + "GPUS_PER_NODE": "8", "xP": "1", "yD": "1", "NNODES": "4", "NODE_RANK": "1", + "WS_PATH": str(tmp_path), "BENCHMARK_LOGS_DIR": str(tmp_path), "SLURM_JOB_ID": "123", + "IS_AGENTIC": "0", "EVAL_ONLY": "false", "DRY_RUN": "0"}, + capture_output=True, text=True, timeout=10, check=True) + args = arguments.read_bytes().decode().split("\0") + assert args[-6:-1] == ["amd", "1", "prefill", "0,1,2,3,4,5", "4"] diff --git a/runners/test_amd_power_lifecycle.py b/runners/test_amd_power_lifecycle.py new file mode 100644 index 000000000..55606aa57 --- /dev/null +++ b/runners/test_amd_power_lifecycle.py @@ -0,0 +1,124 @@ +"""Keep optional collector failures separate from AMD serving outcomes. + +Expected exit codes below follow the required-power rule the AMD scripts +implement: REQUIRE_POWER in {1, true, TRUE, yes, YES} makes a collector +failure fail the run with rc 1 but never masks a nonzero serving rc; any +other value downgrades the failure to a warning and the run rc is the +serving rc. Failures before the benchmark starts abort it when required. +""" +import os +from pathlib import Path +import shutil +import subprocess + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def test_job_slurm_forwards_require_power_into_the_container(): + job = (ROOT / 'benchmarks/multi_node/amd_utils/job.slurm').read_text() + start = job.index('DOCKER_ENV_COMMON=(') + block = job[start:job.index('\n)', start) + 2] + # Execute the submit-shell array and the node-shell expansion used by docker. + script = block + '\ndocker() { printf \'%s\\0\' "$@"; }; export -f docker;\n' + ( + 'bash -c "docker run ${DOCKER_ENV_COMMON[*]} test-image"') + result = subprocess.run(['bash', '-e', '-c', script], + env={'PATH': '/usr/bin:/bin', 'WS_PATH': '/workspace', + 'REQUIRE_POWER': 'sentinel-policy'}, capture_output=True, check=True) + args = result.stdout.decode().split('\0') + forwarded = dict(args[index + 1].split('=', 1) for index, arg in enumerate(args) if arg == '-e') + assert forwarded['REQUIRE_POWER'] == 'sentinel-policy' + + +@pytest.mark.parametrize(('phase', 'required', 'serving_rc', 'expected_rc', 'benchmark_runs'), [ + ('ready', '', 0, 0, True), + ('ready', '', 7, 7, True), + ('ready', '1', 0, 1, False), + ('ready', 'YES', 7, 1, False), + ('done', '', 7, 7, True), + ('done', '1', 0, 1, True), + ('done', 'true', 7, 7, True), +]) +def test_amd_collector_failure_respects_requirement(tmp_path, phase, required, serving_rc, expected_rc, benchmark_runs): + benchmark_root = tmp_path / 'benchmarks' + scripts = benchmark_root / 'multi_node/amd_utils' + scripts.mkdir(parents=True) + for name in ['bench.sh', 'power.sh']: + shutil.copyfile(ROOT / 'benchmarks/multi_node/amd_utils' / name, scripts / name) + (benchmark_root / 'benchmark_lib.sh').write_text('''run_benchmark_serving() { + printf 'called\\n' > "$CALL_RECEIPT" + printf '1\\n' > "$POWERX_CONTROL_DIR/done-0" + return "$SERVING_RC" + } +''') + control = tmp_path / 'control' + control.mkdir() + (control / 'ready-0').write_text('ready\n') + if phase == 'ready': + (control / 'done-0').write_text('1\n') + receipt = tmp_path / 'called' + result = subprocess.run(['bash', str(scripts / 'bench.sh'), '1', '1', '1', '1', + '/model', 'test', str(tmp_path / 'logs'), '8192', '1024', '1'], + env={**os.environ, 'POWERX_CONTROL_DIR': str(control), 'NNODES': '1', + 'REQUIRE_POWER': required, 'SERVING_RC': str(serving_rc), + 'CALL_RECEIPT': str(receipt), 'POWERX_HOST_UID': str(os.getuid()), + 'POWERX_HOST_GID': str(os.getgid())}, capture_output=True, text=True, timeout=10) + assert result.returncode == expected_rc, result.stderr + assert receipt.exists() is benchmark_runs + + +@pytest.mark.parametrize('fault', ['topology', 'uneven_tp']) +@pytest.mark.parametrize(('required', 'serving_rc', 'expected_rc', 'benchmark_runs'), [ + ('', 0, 0, True), + ('0', 7, 7, True), + ('false', 0, 0, True), + ('1', 0, 1, False), + ('true', 7, 1, False), + ('YES', 0, 1, False), +]) +def test_amd_start_failure_respects_requirement(tmp_path, fault, required, serving_rc, expected_rc, benchmark_runs): + scripts = tmp_path / 'amd_utils' + scripts.mkdir() + for name in ['server.sh', 'power.sh']: + shutil.copyfile(ROOT / 'benchmarks/multi_node/amd_utils' / name, scripts / name) + receipt = tmp_path / 'called' + (scripts / 'server_sglang.sh').write_text('''printf 'called\\n' > "$CALL_RECEIPT" +exit "$SERVING_RC" +''') + result = subprocess.run(['bash', str(scripts / 'server.sh')], + env={**os.environ, 'WS_PATH': str(scripts), + 'ENGINE': 'sglang-disagg', 'BENCH_INPUT_LEN': '8192', + 'BENCH_OUTPUT_LEN': '1024', 'EVAL_ONLY': 'false', + 'IS_AGENTIC': '0', 'DRY_RUN': '0', + 'GPUS_PER_NODE': '8', 'PREFILL_TP_SIZE': '9', + 'DECODE_TP_SIZE': '8', 'xP': '1', 'yD': '1', + 'NNODES': '2' if fault == 'topology' else '3', + 'NODE_RANK': '0', 'REQUIRE_POWER': required, + 'SERVING_RC': str(serving_rc), 'CALL_RECEIPT': str(receipt)}, + capture_output=True, text=True, timeout=5) + assert result.returncode == expected_rc, result.stderr + assert receipt.exists() is benchmark_runs + assert ('inconsistent AMD node topology' if fault == 'topology' else + 'uneven per-node TP layout') in result.stderr + + +@pytest.mark.parametrize(('required', 'serving_rc', 'staging_rc', 'expected_rc'), [ + ('', 0, 0, 0), + ('', 0, 9, 0), + ('', 7, 9, 7), + ('false', 0, 9, 0), + ('1', 0, 0, 0), + ('1', 0, 9, 1), + ('1', 7, 9, 7), + ('YES', 0, 9, 1), +]) +def test_amd_staging_failure_preserves_serving_outcome(tmp_path, required, serving_rc, staging_rc, expected_rc): + tail = (ROOT / 'benchmarks/multi_node/amd_utils/job.slurm').read_text().split( + 'BENCHMARK_STEP_RC=$?', 1)[1] + script = 'BENCHMARK_STEP_RC=$SERVING_RC\nstage_native_power() { return "$STAGING_RC"; }\n' + tail + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', script], + env={**os.environ, 'REQUIRE_POWER': required, 'SERVING_RC': str(serving_rc), + 'STAGING_RC': str(staging_rc), 'KEEP_CONTAINERS': '1'}, + capture_output=True, text=True, timeout=5) + assert result.returncode == expected_rc, result.stderr diff --git a/utils/agentic/aggregation/test_power_lifecycle.py b/utils/agentic/aggregation/test_power_lifecycle.py index 6d955af12..6b7c5f6db 100644 --- a/utils/agentic/aggregation/test_power_lifecycle.py +++ b/utils/agentic/aggregation/test_power_lifecycle.py @@ -2,6 +2,7 @@ from __future__ import annotations +import csv import json import os import re @@ -278,7 +279,9 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( start_gpu_monitor() {{ printf 'monitor-pid:%s\n' "${{BASHPID:-$$}}" >> {str(event_log)!r} }} -stop_gpu_monitor() {{ printf 'monitor-stop\n' >> {str(event_log)!r}; }} +stop_gpu_monitor() {{ + printf 'monitor-stop:%s\n' "${{AMD_MONITOR_STOP_TIMEOUT_S:-unset}}" >> {str(event_log)!r} +}} fake_replay() {{ exec {sys.executable!r} -c ' import signal, sys, time @@ -327,7 +330,404 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( assert proc.returncode == expected_rc, stderr events = _events(tmp_path) - assert events.count("monitor-stop") == 1 + stop_events = [event for event in events if event.startswith("monitor-stop")] + # Signal teardown must stop exactly once, in abort mode: the coverage wait + # is skipped by setting AMD_MONITOR_STOP_TIMEOUT_S=0 before stopping. + assert stop_events == ["monitor-stop:0"] expected_parent_event = "parent-int" if sent_signal == signal.SIGINT else "parent-term" assert expected_parent_event in events assert events[-1] == "parent-exit" + + + +@pytest.mark.parametrize( + ("sent_signal", "expected_rc"), + [(signal.SIGINT, 130), (signal.SIGTERM, 143)], +) +def test_signal_during_amd_coverage_wait_stops_monitor( + tmp_path: Path, sent_signal: signal.Signals, expected_rc: int +): + result_dir = tmp_path / "results" + result_dir.mkdir() + event_log = tmp_path / "events.log" + script = f""" +source {str(BENCHMARK_LIB)!r} +start_gpu_monitor() {{ + GPU_METRICS_CSV="$2" + printf 'timestamp,gpu,socket_power\n1,0,500\n' > "$GPU_METRICS_CSV" + command sleep 60 >/dev/null 2>&1 & + GPU_MONITOR_PID=$! + GPU_MONITOR_VENDOR=amd + printf 'monitor:%s\nlifecycle:%s\n' "$GPU_MONITOR_PID" "${{BASHPID:-$(exec sh -c 'echo "$PPID"')}}" >> {str(event_log)!r} +}} +sleep() {{ + printf 'coverage-wait\n' >> {str(event_log)!r} + command sleep "$@" +}} +_write_amd_smi_sidecar() {{ :; }} +fake_replay() {{ :; }} +trap 'printf "parent-exit\\n" >> {str(event_log)!r}' EXIT +REPLAY_CMD=fake_replay +ENABLE_AGENTX_POWER=1 +IS_MULTINODE=false +AMD_MONITOR_STOP_TIMEOUT_S=30 +run_agentic_replay_and_write_outputs {str(result_dir)!r} +exit $? +""" + proc = subprocess.Popen( + ["bash", "-c", script], + env={**os.environ, "PATH": "/usr/bin:/bin"}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + events = _events(tmp_path) if event_log.exists() else [] + if "coverage-wait" in events: + break + time.sleep(0.01) + else: + pytest.fail("AMD coverage wait did not start") + + pids = dict(event.split(":") for event in events if ":" in event) + # Signal only the lifecycle shell: signalling the whole group would + # kill the monitor directly and hide a broken cleanup handler. + os.kill(int(pids["lifecycle"]), sent_signal) + _, stderr = proc.communicate(timeout=5) + assert proc.returncode == expected_rc, stderr + assert _events(tmp_path)[-1] == "parent-exit" + with pytest.raises(ProcessLookupError): + os.kill(int(pids["monitor"]), 0) + finally: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.communicate() + + +# AMDSMI 26.2.0 `metric -p -c -t -u -w 1 --csv` header (order-faithful subset, +# measured on MI355X; mirrors test_detect_columns_amd_watch_mode_real_header). +_MI355X_WATCH_HEADER = ( + "timestamp,gpu,gfx_activity,umc_activity,mm_activity,vcn_activity," + "jpeg_activity,gfx_busy_inst_xcp_0,jpeg_busy_xcp_0,vcn_busy_xcp_0," + "socket_power,gfx_voltage,soc_voltage,mem_voltage,throttle_status," + "power_management,gfx_0_clk,mem_0_clk,edge,hotspot,mem" +) + +# amd-smi quotes list-valued cells with embedded commas; the coverage helper +# must keep the power cell at its header-relative position through them. +_WATCH_ROW_FORMAT = ( + "%s,%s,0,0,N/A,\"['N/A', 'N/A']\",\"['N/A', 'N/A']\",\"[0, 0]\"," + "\"[0, 0]\",\"[0, 0]\",%s,N/A,N/A,N/A,N/A,ENABLED,1404,2000,N/A,40,25\\n" +) + + +def _bash_single_quote(text: str) -> str: + return "'" + text.replace("'", "'\\''") + "'" + + +def _run_amd_stop( + tmp_path: Path, + *, + producer_script: str, + timeout_s: int | str, + interval: int = 1, + setup_script: str = "", +) -> subprocess.CompletedProcess[str]: + """Run the real stop_gpu_monitor against a scripted AMD telemetry producer.""" + csv_path = tmp_path / "gpu_metrics.csv" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + amd_smi = bin_dir / "amd-smi" + amd_smi.write_text("#!/bin/bash\nprintf 'gpu,total_energy_consumption\\n0,100.0\\n'\n") + amd_smi.chmod(0o755) + script = f""" +set -e +source {str(BENCHMARK_LIB)!r} +GPU_METRICS_CSV={str(csv_path)!r} +printf '%s\\n' {_bash_single_quote(_MI355X_WATCH_HEADER)} > "$GPU_METRICS_CSV" +emit_row() {{ + printf {_bash_single_quote(_WATCH_ROW_FORMAT)} "$1" "$2" "$3" >> "$GPU_METRICS_CSV" +}} +{setup_script} +( {producer_script} ) & +GPU_MONITOR_PID=$! +printf '%s\\n' "$GPU_MONITOR_PID" > {str(tmp_path / "producer.pid")!r} +GPU_MONITOR_VENDOR=amd +GPU_MONITOR_INTERVAL={interval} +AMD_MONITOR_STOP_TIMEOUT_S={timeout_s} +date +%s > {str(tmp_path / "pre.txt")!r} +stop_gpu_monitor +date +%s > {str(tmp_path / "post.txt")!r} +""" + return subprocess.run( + ["bash", "-c", script], + env={ + **os.environ, + "PATH": f"{bin_dir}:/usr/bin:/bin", + "PYTHONDONTWRITEBYTECODE": "1", + }, + capture_output=True, + text=True, + check=False, + timeout=60, + ) + + +def _min_covered_tick(csv_path: Path) -> int: + """Newest usable tick (numeric ts, power > 0) covered by every GPU.""" + newest: dict[str, float] = {} + with csv_path.open(newline="", encoding="utf-8") as f: + for row in csv.DictReader(f): + try: + timestamp = float((row.get("timestamp") or "").strip()) + power = float((row.get("socket_power") or "").strip()) + except ValueError: + continue + if timestamp > 1e12: # millisecond epoch, mirror _parse_timestamp + timestamp /= 1000.0 + gpu = (row.get("gpu") or "").strip() + if not gpu or power <= 0: + continue + newest[gpu] = max(newest.get(gpu, 0.0), timestamp) + assert newest, "no usable telemetry rows" + return int(min(newest.values())) + + +def _stop_epochs(tmp_path: Path) -> tuple[int, int]: + pre = int((tmp_path / "pre.txt").read_text().strip()) + post = int((tmp_path / "post.txt").read_text().strip()) + return pre, post + + +def _assert_producer_dead(tmp_path: Path) -> None: + producer_pid = int((tmp_path / "producer.pid").read_text().strip()) + with pytest.raises(ProcessLookupError): + os.kill(producer_pid, 0) + + +def test_amd_stop_waits_until_every_gpu_covers_stop_request(tmp_path: Path): + producer = """ +while :; do + now=$(date +%s) + emit_row "$now" 0 500 + emit_row "$now" 1 505 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s=30) + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + assert "never covered the stop request" not in result.stdout + result.stderr + pre, _ = _stop_epochs(tmp_path) + # Every GPU has a usable tick at/after the first whole second past stop + # entry, so any fractional window end before the stop is bracketed. + assert _min_covered_tick(tmp_path / "gpu_metrics.csv") >= pre + 1 + _assert_producer_dead(tmp_path) + assert duration < 10 + + +def test_amd_stop_ignores_degenerate_rows_for_coverage(tmp_path: Path): + setup = """ +stale=$(( $(date +%s) - 30 )) +emit_row "$stale" 0 500 +emit_row "$stale" 1 505 +""" + producer = """ +while :; do + now=$(date +%s) + emit_row "$now" 0 N/A + emit_row "$now" 1 N/A + sleep 0.2 +done +""" + result = _run_amd_stop( + tmp_path, + producer_script=producer, + timeout_s=2, + setup_script=setup, + ) + + assert result.returncode == 0, result.stderr + assert "never covered the stop request" in result.stderr + pre, post = _stop_epochs(tmp_path) + assert post - pre >= 2 + _assert_producer_dead(tmp_path) + + +def test_amd_stop_requires_coverage_per_gpu(tmp_path: Path): + setup = """ +stale=$(( $(date +%s) - 30 )) +emit_row "$stale" 1 505 +""" + producer = """ +while :; do + emit_row "$(date +%s)" 0 500 + sleep 0.2 +done +""" + result = _run_amd_stop( + tmp_path, + producer_script=producer, + timeout_s=2, + setup_script=setup, + ) + + assert result.returncode == 0, result.stderr + # GPU 1 never covers the stop request, so min-over-GPUs coverage times out + # even though GPU 0 keeps producing fresh usable ticks. + assert "never covered the stop request" in result.stderr + pre, post = _stop_epochs(tmp_path) + assert post - pre >= 2 + _assert_producer_dead(tmp_path) + + +def test_amd_stop_preserves_outputs_when_monitor_exits_during_wait(tmp_path: Path): + setup = """ +emit_row 1 0 500 +sleep() { + # End the real producer on the first coverage poll without a timing race. + kill "$GPU_MONITOR_PID" + wait "$GPU_MONITOR_PID" 2>/dev/null || true +} +""" + result = _run_amd_stop( + tmp_path, + producer_script="exec /bin/sleep 60", + timeout_s=3, + setup_script=setup, + ) + + assert result.returncode == 0, result.stderr + assert "AMD monitor exited before covering the stop request" in result.stderr + assert (tmp_path / "gpu_metrics_energy_end.csv").read_text() == ( + "gpu,total_energy_consumption\n0,100.0\n" + ) + _stop_epochs(tmp_path) # The caller continued after stop under set -e. + _assert_producer_dead(tmp_path) + + +def test_amd_stop_survives_non_integer_timeout(tmp_path: Path): + producer = """ +while :; do + now=$(date +%s) + emit_row "$now" 0 500 + emit_row "$now" 1 505 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s="30s") + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + # A non-integer timeout must not unwind stop_gpu_monitor via a bash + # arithmetic error (which would leak the monitor and skip tail repair + # and the energy sidecar): it warns, falls back to 30, and still waits. + assert "ignoring non-integer AMD_MONITOR_STOP_TIMEOUT_S='30s'" in result.stderr + assert "never covered the stop request" not in result.stdout + result.stderr + pre, _ = _stop_epochs(tmp_path) + assert _min_covered_tick(tmp_path / "gpu_metrics.csv") >= pre + 1 + _assert_producer_dead(tmp_path) + assert duration < 10 + + +def test_amd_stop_normalizes_millisecond_epoch_timestamps(tmp_path: Path): + producer = """ +while :; do + now=$(( $(date +%s) * 1000 + 123 )) + emit_row "$now" 0 500 + emit_row "$now" 1 505 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s=30) + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + # Raw millisecond epochs (~1.8e12) dwarf any second-scale target, so + # without normalization the poll would return + # instantly with zero tail coverage; mirrored _parse_timestamp + # normalization makes the poll wait for real coverage instead. + assert "never covered the stop request" not in result.stdout + result.stderr + pre, _ = _stop_epochs(tmp_path) + assert _min_covered_tick(tmp_path / "gpu_metrics.csv") >= pre + 1 + _assert_producer_dead(tmp_path) + assert duration < 10 + + +def test_amd_stop_bounds_wait_for_iso_timestamps(tmp_path: Path): + producer = """ +while :; do + emit_row "$(date +%Y-%m-%dT%H:%M:%S)" 0 500 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s=1, interval=30) + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + assert "never covered the stop request" in result.stderr + assert "exited before covering" not in result.stderr + assert duration < 4 + _assert_producer_dead(tmp_path) + + +def test_amd_stop_waits_for_delayed_first_sample(tmp_path: Path): + producer = """ +sleep 4 +while :; do + emit_row "$(date +%s)" 0 500 + sleep 0.2 +done +""" + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s=8) + + assert result.returncode == 0, result.stderr + pre, _ = _stop_epochs(tmp_path) + assert _min_covered_tick(tmp_path / "gpu_metrics.csv") >= pre + 1 + _assert_producer_dead(tmp_path) + + +@pytest.mark.parametrize('required', ['', '0', '1', 'true']) +@pytest.mark.parametrize('failure', ['unsupported', 'collision']) +def test_amd_fifo_failure_respects_power_requirement_and_keeps_existing_path( + tmp_path: Path, required: str, failure: str +): + script = r''' +source "$1" +amd-smi() { echo unexpected-start > "$CALLS"; } +mkfifo() { + if [[ "$FIFO_FAILURE" == collision ]]; then + printf 'another stream\n' > "$1" + fi + printf '%s\n' "$1" > "$FIFO_PATH" + return 1 +} +trap stop_gpu_monitor EXIT +set -e +start_gpu_monitor --output "$2/gpu_metrics.csv" +printf 'benchmark\n' > "$2/benchmark-called" +''' + result = subprocess.run(['bash', '-c', script, 'bash', str(BENCHMARK_LIB), str(tmp_path)], + env={**os.environ, 'PATH': '/usr/bin:/bin', 'REQUIRE_POWER': required, + 'FIFO_FAILURE': failure, 'FIFO_PATH': str(tmp_path / 'fifo-path'), + 'CALLS': str(tmp_path / 'collector-called')}, + capture_output=True, text=True, timeout=5) + is_required = required in {'1', 'true'} + assert result.returncode == int(is_required), result.stderr + assert (tmp_path / 'benchmark-called').exists() is not is_required + assert not (tmp_path / 'collector-called').exists() + fifo = Path((tmp_path / 'fifo-path').read_text().strip()) + if failure == 'collision': + assert fifo.read_text() == 'another stream\n' + else: + assert not fifo.exists() diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py index 31e563737..e4234facc 100644 --- a/utils/test_native_multinode_power.py +++ b/utils/test_native_multinode_power.py @@ -219,6 +219,20 @@ def test_native_single_role_preserves_whole_fleet_and_role_metrics(tmp_path, rol assert json.loads((tmp_path / 'power_validation_result.json').read_text())['power_valid'] +def test_native_amd_abort_skips_legacy_tail_wait(tmp_path): + result = subprocess.run(['bash', '-c', '''source "$1" +kill() { return 0; }; wait() { return 0; } +sleep() { echo unexpected-tail-wait >&2; } +_write_amd_smi_sidecar() { return 0; } +GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd AMD_MONITOR_STOP_TIMEOUT_S=0 +GPU_METRICS_CSV="$2/missing.csv" +stop_gpu_monitor +''', 'bash', str(REPO / 'benchmarks/benchmark_lib.sh'), str(tmp_path)], + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert 'unexpected-tail-wait' not in result.stderr + + def test_native_amd_abort_publishes_receipt_before_reaper_deadline(tmp_path): binary = tmp_path / 'bin' binary.mkdir() diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 8b8661043..ba334691a 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -5,6 +5,7 @@ import signal import subprocess import sys +import time from pathlib import Path import pytest @@ -1166,8 +1167,13 @@ def test_stop_gpu_monitor_drops_truncated_row_before_final_sample(self, tmp_path final_sample, ] - def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path): - """AMD stop lets the watch stream bracket the window, then snapshots energy.""" + def test_stop_gpu_monitor_amd_covers_stop_request_and_snapshots_energy(self, tmp_path): + """AMD stop returns once telemetry covers the stop entry, then snapshots energy. + + A usable tick stamped past the stop request satisfies the coverage + poll on its first pass: the legacy fixed tail sleep never runs, the + stream is not mutated, and the end-side accumulator snapshot is + written.""" fake_bin = tmp_path / "bin" fake_bin.mkdir() args_log = tmp_path / "amd_args.txt" @@ -1179,7 +1185,8 @@ def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path ) fake_amd_smi.chmod(0o755) sleep_log = tmp_path / "sleep_args.txt" - contents = "timestamp,gpu,socket_power\n1785881113,0,238\n" + covered_tick = int(time.time()) + 30 + contents = f"timestamp,gpu,socket_power\n{covered_tick},0,238\n" metrics = tmp_path / "gpu_metrics.csv" metrics.write_text(contents) benchmark_lib = Path(__file__).parents[1] / "benchmarks/benchmark_lib.sh" @@ -1187,7 +1194,7 @@ def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path source {str(benchmark_lib)!r} kill() {{ return 0; }} wait() {{ return 0; }} -sleep() {{ printf '%s\\n' "$1" > {str(sleep_log)!r}; }} +sleep() {{ printf '%s\\n' "$1" >> {str(sleep_log)!r}; }} GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd GPU_MONITOR_INTERVAL=3 @@ -1208,7 +1215,8 @@ def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path ) assert result.returncode == 0, result.stderr - assert sleep_log.read_text().strip() == "5" + assert not sleep_log.exists() + assert "never covered the stop request" not in result.stderr assert metrics.read_text() == contents assert "metric -E --csv" in args_log.read_text() energy_end = tmp_path / "gpu_metrics_energy_end.csv" @@ -1237,6 +1245,7 @@ def test_stop_gpu_monitor_amd_drops_truncated_row_without_append(self, tmp_path) GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd GPU_METRICS_CSV={str(metrics)!r} +AMD_MONITOR_STOP_TIMEOUT_S=0 stop_gpu_monitor """ env = {