diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 1b1b5f5ab4..5dc71b1eef 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -44,6 +44,9 @@ on: - 'utils/test_inject_srt_power_concurrencies.py' - 'utils/test_process_result.py' - 'utils/test_benchmark_serving.py' + - 'benchmarks/multi_node/tilert_utils/**' + - 'benchmarks/multi_node/glm5.1_fp8_b200_tilert-disagg.sh' + - 'runners/test_tilert_power_lifecycle.py' permissions: contents: read @@ -65,7 +68,7 @@ jobs: run: | cd utils uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \ - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py ../runners/test_kimik3_bh_power.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py ../runners/test_kimik3_bh_power.py ../runners/test_tilert_power_lifecycle.py -v - name: Test serving client result persistence run: | diff --git a/benchmarks/multi_node/tilert_utils/run_node.sh b/benchmarks/multi_node/tilert_utils/run_node.sh index 8927624150..0b92099c15 100755 --- a/benchmarks/multi_node/tilert_utils/run_node.sh +++ b/benchmarks/multi_node/tilert_utils/run_node.sh @@ -42,6 +42,36 @@ AGENTIC_LOGS_DIR=${AGENTIC_LOGS_DIR:-$RESULT_DIR/LOGS/agentic} mkdir -p "$BENCHMARK_LOGS_DIR" DONE_SENTINEL="$BENCHMARK_LOGS_DIR/.tilert_done.${SLURM_JOB_ID:-local}" +source "$(dirname "$0")/../../native_power_lifecycle.sh" +finish_tilert_node() { + local rc=$? pid + trap - EXIT + if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 && -n "${POWERX_COLLECTOR_PID:-}" ]]; then + if [[ "$TILERT_ROLE" == prefill || "$rc" != 0 ]]; then + powerx_stop_collectors || rc=$? + else + powerx_reap_collector || rc=$? + fi + fi + if [[ "$TILERT_ROLE" == prefill ]]; then + printf '%s\n' "$rc" > "$DONE_SENTINEL.tmp" || rc=1 + if [[ -n "${POWERX_HOST_UID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$DONE_SENTINEL.tmp" || rc=1 + fi + mv -f "$DONE_SENTINEL.tmp" "$DONE_SENTINEL" || rc=1 + fi + for pid in "${ROUTER_PID:-}" "${PREFILL_PID:-}" "${DECODE_PID:-}"; do + [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true + done + exit "$rc" +} +trap finish_tilert_node EXIT +trap 'exit 143' TERM HUP +trap 'exit 130' INT +if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then + powerx_start_collector "/powerx_native/node-$POWERX_RANK" "/powerx_control" \ + nvidia "$POWERX_RANK" "$TILERT_ROLE" "$POWERX_GPU_COUNT" 2 +fi echo "[tilert-run_node] ROLE=$TILERT_ROLE host=$(hostname) DECODE_HOST=$DECODE_HOST PREFILL_HOST=$PREFILL_HOST" log_and_run_bg() { @@ -211,6 +241,9 @@ run_bench_and_eval() { wait_for_server_ready --port "$ROUTER_PORT" \ --server-log "$BENCHMARK_LOGS_DIR/tilert_router.log" --server-pid "$ROUTER_PID" local rc=0 conc np + if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then + powerx_wait_collectors ready || return $? + fi if [[ "${EVAL_ONLY:-false}" != "true" ]]; then for conc in $CONC_LIST; do np=$(( conc * 10 )) @@ -228,7 +261,7 @@ run_bench_and_eval() { || { rc=$?; echo "[bench] WARNING: conc=$conc failed/timed out (rc=$rc)"; } done fi - run_lm_eval + run_lm_eval || rc=$? return $rc } @@ -278,13 +311,13 @@ case "$TILERT_ROLE" in sleep 5 done if [[ -f "$DONE_SENTINEL" ]]; then - echo "[decode] done sentinel received, shutting down"; kill "$DECODE_PID" 2>/dev/null || true; exit 0 + echo "[decode] done sentinel received, shutting down" + exit "$(cat "$DONE_SENTINEL")" fi echo "[decode] decode_server exited early (see $BENCHMARK_LOGS_DIR/tilert_decode.log)"; exit 1 ;; prefill) rdma_preflight || exit 1 - rm -f "$DONE_SENTINEL" if [[ "$TILERT_IS_AGENTIC" == "1" ]]; then resolve_trace_source install_agentic_deps @@ -300,8 +333,7 @@ case "$TILERT_ROLE" in else run_bench_and_eval; BENCH_RC=$? fi - touch "$DONE_SENTINEL" - kill "$ROUTER_PID" "$PREFILL_PID" 2>/dev/null || true + # EXIT drains both collectors before the decode role sees completion. exit $BENCH_RC ;; *) diff --git a/benchmarks/multi_node/tilert_utils/submit.sh b/benchmarks/multi_node/tilert_utils/submit.sh index b63e97bef7..e29ae13988 100755 --- a/benchmarks/multi_node/tilert_utils/submit.sh +++ b/benchmarks/multi_node/tilert_utils/submit.sh @@ -1,6 +1,11 @@ #!/usr/bin/env bash +set -euo pipefail set -x NODES=$(( ${PREFILL_NODES:-1} + ${DECODE_NODES:-1} )) +if [[ "${PREFILL_NODES:-1}" != 1 || "${DECODE_NODES:-1}" != 1 ]]; then + echo "TileRT native launcher requires one physical node per role" >&2 + exit 1 +fi GPUS_PER_NODE="${GPUS_PER_NODE:-${GPU_COUNT:-$(( ${PREFILL_TP:-8} > ${DECODE_TP:-8} ? ${PREFILL_TP:-8} : ${DECODE_TP:-8} ))}}" SQUASH_DIR="${B200_SQUASH_DIR:-/home/sa-shared/containers}" @@ -14,15 +19,97 @@ DECODE_IMAGE="${DECODE_IMAGE:-$IMAGE}" squash_path() { echo "$SQUASH_DIR/$(echo "$1" | sed 's/[\/:@#]/_/g').sqsh"; } DECODE_SQUASH="$(squash_path "$DECODE_IMAGE")" PREFILL_SQUASH="$(squash_path "$PREFILL_IMAGE")" +MODEL_MOUNTS="$MODEL_PATH:$MODEL_PATH" +if [[ -n "${HF_HUB_CACHE_HOST_PATH:-}" ]]; then + # HF snapshots link to sibling blobs outside the snapshot directory. + MODEL_MOUNTS="$HF_HUB_CACHE_HOST_PATH:$HF_HUB_CACHE_HOST_PATH,$MODEL_MOUNTS" +fi -salloc --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \ - --nodes="$NODES" --gres=gpu:"$GPUS_PER_NODE" --exclusive --mem=0 \ - --time="${SALLOC_TIME_LIMIT:-480}" --no-shell --job-name="$RUNNER_NAME" -JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1) - -mapfile -t HOSTS < <(scontrol show hostnames "$(squeue -j "$JOB_ID" -h -o %N)") -[[ "${#HOSTS[@]}" -ge 2 ]] || { echo "expected >=2 nodes, got: ${HOSTS[*]}"; exit 1; } +if [[ "${TILERT_IN_ALLOCATION:-0}" != 1 ]]; then + # Run inside the allocation returned by this request. Looking up a runner + # name can attach to an older job; salloc supplies the authoritative ID. + export TILERT_IN_ALLOCATION=1 + exec salloc --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \ + --nodes="$NODES" --gres=gpu:"$GPUS_PER_NODE" --exclusive --mem=0 \ + --time="${SALLOC_TIME_LIMIT:-480}" --job-name="$RUNNER_NAME" \ + "$BASH" "$0" "$@" +fi +JOB_ID="${SLURM_JOB_ID:?salloc did not provide its allocation ID}" +mapfile -t HOSTS < <(scontrol show hostnames "${SLURM_JOB_NODELIST:?salloc did not provide its nodes}") +[[ "${#HOSTS[@]}" -eq 2 ]] || { echo "expected 2 nodes, got: ${HOSTS[*]}"; exit 1; } export DECODE_HOST="${HOSTS[0]}" PREFILL_HOST="${HOSTS[1]}" +export POWERX_NATIVE_ENABLED=0 +if [[ "${REQUIRE_POWER:-0}" =~ ^(1|true|TRUE|yes|YES)$ && "$ISL" == 8192 && "$OSL" == 1024 && "${IS_AGENTIC:-0}" != 1 && "${SCENARIO_TYPE:-}" != agentic-coding && "${EVAL_ONLY:-false}" != true ]]; then + export POWERX_NATIVE_ENABLED=1 +fi +export SLURM_JOB_ID="$JOB_ID" +POWERX_MOUNTS="" +POWERX_ENV="POWERX_NATIVE_ENABLED" +if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + export POWERX_HOST_UID="$(id -u)" POWERX_HOST_GID="$(id -g)" + export POWERX_COLLECTOR_REVISION="$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" + export POWERX_RAW_ROOT="${POWERX_RAW_ROOT:-/tmp/inferencex-native-$JOB_ID}" + export POWERX_CONTROL_ROOT="$GITHUB_WORKSPACE/LOGS/power_control-$JOB_ID" + mkdir -p "$POWERX_CONTROL_ROOT" "$GITHUB_WORKSPACE/LOGS/native_power" + chmod 777 "$POWERX_CONTROL_ROOT" + srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 mkdir -p "$POWERX_RAW_ROOT" + srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 chmod 777 "$POWERX_RAW_ROOT" + POWERX_MOUNTS=",$POWERX_RAW_ROOT:/powerx_native,$POWERX_CONTROL_ROOT:/powerx_control" + POWERX_ENV+=",CUDA_VISIBLE_DEVICES,POWERX_HOST_UID,POWERX_HOST_GID,POWERX_COLLECTOR_REVISION,POWERX_NODE_NAME,POWERX_CLOCK_SYNCHRONIZED,POWERX_RANK,POWERX_GPU_COUNT" +fi +rm -f "${BENCHMARK_LOGS_DIR:-$GITHUB_WORKSPACE}/.tilert_done.$JOB_ID" +# Keep node-local receipts inside the allocation until both serving steps drain. +# A caught cancellation exits through the same staging path as normal completion. +wait_owned_step() { + local pid="$1" deadline=$((SECONDS + ${TILERT_DECODE_DRAIN:-60})) rc=0 + while kill -0 "$pid" 2>/dev/null; do + if (( SECONDS >= deadline )); then + echo "[submit] step $pid did not drain before timeout" >&2 + kill -TERM "$pid" 2>/dev/null || true + sleep 2 + kill -KILL "$pid" 2>/dev/null || true + rc=1 + break + fi + sleep 1 + done + wait "$pid" || rc=$? + return "$rc" +} + +finish_tilert_submit() { + local rc=$? step_rc pid role_rank + trap - EXIT + trap '' TERM HUP INT + if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + printf 'stop\n' > "$POWERX_CONTROL_ROOT/stop" || { [[ "$rc" != 0 ]] || rc=1; } + fi + if [[ "$rc" != 0 ]]; then + for pid in "${PREFILL_SRUN_PID:-}" "${DECODE_SRUN_PID:-}"; do + [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true + done + fi + for pid in "${PREFILL_SRUN_PID:-}" "${DECODE_SRUN_PID:-}"; do + [[ -n "$pid" ]] || continue + step_rc=0 + wait_owned_step "$pid" || step_rc=$? + [[ "$rc" != 0 ]] || rc=$step_rc + done + if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + for role_rank in 0 1; do + # A failed or missing rank must not suppress the other rank's audit. + srun --jobid="$JOB_ID" --nodelist="${HOSTS[$role_rank]}" --ntasks=1 \ + bash -c 'cp -R "$POWERX_RAW_ROOT/node-$1" "$GITHUB_WORKSPACE/LOGS/native_power/"' bash "$role_rank" & + step_rc=0 + wait_owned_step "$!" || step_rc=$? + [[ "$rc" != 0 ]] || rc=$step_rc + done + fi + exit "$rc" +} +trap finish_tilert_submit EXIT +trap 'exit 143' TERM HUP +trap 'exit 130' INT import_image() { local image_ref="$1" squash_file="$2" host="$3" @@ -43,6 +130,15 @@ mkdir -p "$TILERT_WEIGHTS_DIR" run_role() { local role="$1" host="$2" squash_file="$3" + local rank=0 gpu_count="${DECODE_TP:-8}" + if [[ "$role" == prefill ]]; then rank=1; gpu_count="${PREFILL_TP:-8}"; fi + if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + export POWERX_RANK="$rank" POWERX_GPU_COUNT="$gpu_count" POWERX_NODE_NAME="$host" + export CUDA_VISIBLE_DEVICES="$(seq -s, 0 "$((gpu_count - 1))")" + export POWERX_CLOCK_SYNCHRONIZED + POWERX_CLOCK_SYNCHRONIZED=$(srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \ + bash -c 'timedatectl show -p NTPSynchronized --value 2>/dev/null || echo false') + fi # The tilert image bakes no NVIDIA_VISIBLE_DEVICES (unlike vllm-openai), and # enroot's nvidia hook only injects the driver when it is set — without it the # decode container has no libcuda and torch dies with "Found no NVIDIA driver". @@ -50,10 +146,11 @@ run_role() { # Exported here (not in --export) because the capabilities value contains a # comma, which srun's --export parsing would split on. export NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=compute,utility - srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \ + exec srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \ --container-image="$squash_file" \ - --container-mounts="$GITHUB_WORKSPACE:/workspace,$MODEL_PATH:$MODEL_PATH,$TILERT_WEIGHTS_DIR:$TILERT_WEIGHTS_DIR" \ + --container-mounts="$GITHUB_WORKSPACE:/workspace,$MODEL_MOUNTS,$TILERT_WEIGHTS_DIR:$TILERT_WEIGHTS_DIR$POWERX_MOUNTS" \ --container-workdir=/workspace --no-container-entrypoint \ + --container-env="$POWERX_ENV" \ --export=ALL,TILERT_ROLE="$role",DECODE_HOST="$DECODE_HOST",PREFILL_HOST="$PREFILL_HOST",PORT="${PORT:-8888}" \ bash "/workspace/benchmarks/multi_node/tilert_utils/run_node.sh" } @@ -61,14 +158,10 @@ run_role() { run_role decode "$DECODE_HOST" "$DECODE_SQUASH" & DECODE_SRUN_PID=$! -run_role prefill "$PREFILL_HOST" "$PREFILL_SQUASH" -PREFILL_RC=$? - -for _ in $(seq 1 "${TILERT_DECODE_DRAIN:-60}"); do - kill -0 "$DECODE_SRUN_PID" 2>/dev/null || break - sleep 1 -done -kill -0 "$DECODE_SRUN_PID" 2>/dev/null && { echo "[submit] decode srun did not exit on its own, killing it"; kill "$DECODE_SRUN_PID" 2>/dev/null; } -wait "$DECODE_SRUN_PID" 2>/dev/null || true - +# Both roles run as owned children so a signal interrupts the shell's wait and +# reaches EXIT cleanup immediately; run_role execs srun to preserve that PID. +run_role prefill "$PREFILL_HOST" "$PREFILL_SQUASH" & +PREFILL_SRUN_PID=$! +PREFILL_RC=0 +wait "$PREFILL_SRUN_PID" || PREFILL_RC=$? exit "$PREFILL_RC" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index cbabca1613..a71fc4f670 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -10273,6 +10273,7 @@ glm5.1-fp8-b200-tilert: - "DECODE_NODES=1" - isl: 8192 osl: 1024 + require-power: true search-space: - spec-decoding: "mtp" conc-list: [1] diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 581e5d09ac..21475c30a4 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -114,6 +114,12 @@ The runner-name prefix is load-bearing: workflow routing uses `launch_${RUNNER_N 6. Verify every runner is **Idle** in [repository runner settings](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners) before adding it to sweep traffic. 7. Verify launcher mounts for `_work`, HF cache, staged weights, and squash images from a compute node. Root containers must not leave root-owned files in the shared workspace. +## Native TileRT power + +For GLM-5.1 on B200 Nscale, `MODEL_PATH` can select an existing shared checkpoint instead of the default `/scratch/models/GLM-5.1-FP8`. When it selects an HF snapshot, also set `HF_HUB_CACHE_HOST_PATH` to the existing cache root; TileRT mounts that root at the same absolute path so snapshot links to sibling blobs remain readable. Keep `TILERT_WEIGHTS_DIR` pointed at the separately converted decode weights. + +Only fixed 8192/1024 `glm5.1-fp8-b200-tilert` requires native power. TileRT runs inside its returned `salloc` allocation, retains both role exit codes and drains collectors before staging audits. Exactly one physical node per role is supported. Other sequence lengths, AgentX and eval-only do not enable this collector. Hardware qualification and publication remain pending. + ## Register an srt-slurm recipe Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md). Checked-in recipes: [`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index aeff271a8e..638807434f 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -114,6 +114,12 @@ runner 名称前缀是关键契约：workflow 通过 `launch_${RUNNER_NAME%%_*}. 6. 将 runner 加入 sweep 流量前，在[仓库 runner 设置页](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners)确认每个 runner 都是 **Idle**。 7. 从计算节点验证 launcher 对 `_work`、HF cache、预置权重和 squash 镜像的挂载。root 容器不得在共享 workspace 留下 root 所有的文件。 +## TileRT 原生功耗 + +B200 Nscale 的 GLM-5.1 可用 `MODEL_PATH` 指定已有共享权重，覆盖默认的 `/scratch/models/GLM-5.1-FP8`。若指定 HF snapshot，还需把 `HF_HUB_CACHE_HOST_PATH` 设为现有缓存根目录；TileRT 按相同绝对路径挂载整个缓存，使 snapshot 指向同级 blobs 的软链接可读。`TILERT_WEIGHTS_DIR` 仍指向单独转换的 decode 权重。 + +仅固定 8192/1024 的 `glm5.1-fp8-b200-tilert` 要求原生功耗。TileRT 在 `salloc` 返回的分配内运行，保留两个角色的退出码，并在保存审计数据前等待采集器排空。每个角色仅支持一个物理节点。其他序列长度、AgentX 和 eval-only 不启用此采集器。硬件资格验证与发布仍待完成。 + ## 注册 srt-slurm 配方 映射来源：[`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md)。检入的配方：[`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/)。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1cc4df6fc7..d3ed48d1f7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7441,3 +7441,20 @@ - "Replace the removed decode-only alias --cuda-graph-max-bs with its canonical name --cuda-graph-max-bs-decode, retaining the same min(2*CONC, 64) cap. The 20260910 image includes sgl-project/sglang#38375, so argparse interprets the removed spelling as an ambiguous prefix of the decode and prefill options and exits before serving. Leave --cuda-graph-max-bs-prefill unset: it controls a separate prefill token-capture ceiling, not request concurrency." - "Switch DSA prefill and decode from tilelang to triton (sgl-project/sglang#30575 Fast Triton Sparse MLA). The 20260910 image already contains that backend; kv-cache-dtype stays fp8_e4m3, which the Triton DSA path requires." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 + +- config-keys: + - glm5.1-fp8-b200-tilert + scenario-type: + - fixed-seq-len + description: + - Require native power only for TileRT 8k1k; keep native launches inside the owned + allocation and retain both role exit statuses. + - 仅对 TileRT 8k1k 要求原生功耗；原生启动始终使用本任务分配，并保留两个角色退出状态。 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3057 + +- config-keys: + - glm5.1-fp8-b200-tilert + description: + - "Preserve an explicit prepared GLM-5.1 model path and mount shared HF cache blobs in native TileRT containers." + - "保留显式指定的 GLM-5.1 预备模型路径，并在原生 TileRT 容器中挂载共享 HF 缓存 blobs。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3057 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 960e33e9dc..5f822ec530 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -56,7 +56,7 @@ elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/scratch/models/GLM-5-FP8" export SRT_SLURM_MODEL_PREFIX="glm5-fp8" elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/scratch/models/GLM-5.1-FP8" + export MODEL_PATH="${MODEL_PATH:-/scratch/models/GLM-5.1-FP8}" export SRT_SLURM_MODEL_PREFIX="glm5.1-fp8" elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/scratch/models/GLM-5-NVFP4" diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index f0d6b8ae96..fb95bd7ae4 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -41,6 +41,10 @@ if [[ "$IS_MULTINODE" != "true" ]]; then run_compat_launcher fi +if [[ "$FRAMEWORK" == "tilert" && "${IS_AGENTIC:-0}" != "1" ]]; then + run_compat_launcher +fi + if [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then export MODEL_PATH="${MODEL_PATH:-$NSCALE_MODEL_ROOT/DeepSeek-V4-Pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" diff --git a/runners/test_tilert_power_lifecycle.py b/runners/test_tilert_power_lifecycle.py new file mode 100644 index 0000000000..973e7595ed --- /dev/null +++ b/runners/test_tilert_power_lifecycle.py @@ -0,0 +1,158 @@ +"""CPU-only lifecycle checks with controlled external collector/Slurm processes.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize('prepared_path', ['', '/shared/hf/hub/snapshots/revision']) +def test_b200_tilert_preserves_prepared_model_path(prepared_path): + env = {**os.environ, 'MODEL_PREFIX': 'glm5.1', 'PRECISION': 'fp8', + 'FRAMEWORK': 'tilert', 'IS_MULTINODE': 'true', 'SCENARIO_SUBDIR': '', + 'EXP_NAME': 'glm5.1_8k1k', 'GITHUB_WORKSPACE': str(ROOT), + 'MODEL_PATH': prepared_path} + result = subprocess.run( + ['bash', '-c', 'exec() { printf "%s\\n" "$MODEL_PATH"; exit; }; source "$1"', + 'bash', str(ROOT / 'runners/launch_b200-nscale-compat.sh')], + env=env, capture_output=True, text=True, timeout=10, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == (prepared_path or '/scratch/models/GLM-5.1-FP8') + + +@pytest.mark.parametrize(('decode_rc', 'interruption', 'enabled'), + [(0, None, True), (9, None, True), (0, 'TERM', True), + (0, 'HUP', True), (0, 'INT', True), (0, 'hang', True), + (0, None, False)]) +def test_tilert_submit_keeps_decode_status_and_stages_both_roles(tmp_path, decode_rc, interruption, enabled): + repo, bindir = tmp_path / 'repo', tmp_path / 'bin' + for path in (repo, bindir): + path.mkdir() + commands = { + 'salloc':'#!' + sys.executable + '\n' + r''' +import json,os,pathlib,subprocess,sys +args=sys.argv[1:] +command=args[next(i for i, arg in enumerate(args) if not arg.startswith('--')):] +if '/' not in command[0]: + # Slurm resolves its command before exec; do not silently skip EACCES. + command[0]=next(str(pathlib.Path(p)/command[0]) for p in os.environ['PATH'].split(os.pathsep) + if (pathlib.Path(p)/command[0]).exists()) +try: + result=subprocess.run(command,env={**os.environ,'SLURM_JOB_ID':'123','SLURM_JOB_NODELIST':'node-[a-b]'}) +finally: + staged=pathlib.Path(os.environ['GITHUB_WORKSPACE'])/'LOGS/native_power' + pathlib.Path(os.environ['RELEASE_RECEIPT']).write_text(json.dumps( + [json.loads(p.read_text()) for p in sorted(staged.glob('node-*/manifest.json'))])) +sys.exit(result.returncode) +''', + 'squeue':'#!/bin/sh\necho unexpected job-name lookup >&2\nexit 98\n', + 'scontrol':'#!/bin/sh\nprintf "node-a\\nnode-b\\n"\n', + 'scancel':'#!/bin/sh\necho unexpected cancellation >&2\nexit 98\n', + 'git':'#!/bin/sh\necho 0123456789012345678901234567890123456789\n', + 'srun':'#!' + sys.executable + '\n' + r''' +import json, os, pathlib, signal, subprocess, sys, time +args=sys.argv[1:] +if any(a.startswith('--container-image=') for a in args): + cache=os.environ['HF_HUB_CACHE_HOST_PATH'] + mounts=next(a.split('=',1)[1] for a in args if a.startswith('--container-mounts=')) + if cache: + assert f'{cache}:{cache}' in mounts.split(',') + assert f"{os.environ['MODEL_PATH']}:{os.environ['MODEL_PATH']}" in mounts.split(',') + if os.environ['POWERX_NATIVE_ENABLED'] != '1': + assert not any('/powerx_native' in a for a in args) + sys.exit(0) + rank=os.environ['POWERX_RANK'] + out=pathlib.Path(os.environ['POWERX_RAW_ROOT']) / f'node-{rank}' + out.mkdir(parents=True,exist_ok=True) + mode=os.environ.get('INTERRUPTION') + manifest={'rank':int(rank),'synthetic':True} + def finish(signum, frame): + assert pathlib.Path(os.environ['POWERX_CONTROL_ROOT'],'stop').exists() + manifest['terminated']=True + (out/'manifest.json').write_text(json.dumps(manifest)) + sys.exit(143) + signal.signal(signal.SIGTERM, signal.SIG_IGN if mode=='hang' and rank=='0' else finish) + (out/'manifest.json').write_text(json.dumps(manifest)) + if mode: + if rank=='1': + deadline=time.monotonic()+5 + while not (out.parent/'node-0/manifest.json').exists(): + assert time.monotonic()