From c3040728c629feac18cdd26aca7cdbc246b03b73 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:23:08 -0700 Subject: [PATCH 01/21] fix: preserve complete request outcomes and diagnostic batches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留完整请求结果状态与诊断批次，拆分 Slurm 生命周期和原生功耗采集。 --- .../workflows/benchmark-multinode-tmpl.yml | 26 +-- .github/workflows/benchmark-tmpl.yml | 4 +- docs/results-and-ingestion.md | 12 +- docs/results-and-ingestion_zh.md | 12 +- infx/bench_serving/benchmark_outcome.py | 31 +++ infx/bench_serving/benchmark_serving.py | 17 +- infx/results/fixed_sequence.py | 109 ++++++++- infx/results/power/__init__.py | 1 + infx/results/power/audit.py | 50 ++++ perf-changelog.yaml | 11 + utils/test_process_result.py | 217 +++++++++++++++++- 11 files changed, 444 insertions(+), 46 deletions(-) create mode 100644 infx/bench_serving/benchmark_outcome.py create mode 100644 infx/results/power/audit.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index bae33211b..80b0842f8 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -470,7 +470,7 @@ jobs: fi - name: Process result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ always() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} env: RUNNER_TYPE: ${{ inputs.runner }} run: | @@ -480,29 +480,10 @@ jobs: export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA" fi - # Process each result file - for result_file in ${RESULT_FILENAME}_*.json; do - if [ -f "$result_file" ]; then - echo "Processing $result_file" - # Extract GPU count, prefill_gpus and decode_gpus from filename for tp_size calculation - gpus=$(echo "$result_file" | sed -n "s/.*_gpus_\([0-9][0-9]*\).*\.json/\1/p") - prefill_gpus=$(echo "$result_file" | sed -n "s/.*_ctx_\([0-9]*\).*\.json/\1/p") - decode_gpus=$(echo "$result_file" | sed -n "s/.*_gen_\([0-9]*\).*\.json/\1/p") - - if [ -n "$gpus" ]; then - # Non-disagg results lack _ctx_/_gen_ fields; all GPUs belong to the single worker - if [ -z "$prefill_gpus" ] || [ -z "$decode_gpus" ]; then - prefill_gpus=$gpus - decode_gpus=0 - fi - echo "Extracted: gpus=$gpus, prefill_gpus=$prefill_gpus, decode_gpus=$decode_gpus" - RESULT_FILENAME=${result_file%.json} IS_MULTINODE=true PREFILL_GPUS="$prefill_gpus" DECODE_GPUS="$decode_gpus" python3 utils/process_result.py - fi - fi - done + python3 utils/process_result.py --all - name: Upload result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bmk_${{ env.RESULT_FILENAME }} @@ -518,6 +499,7 @@ jobs: agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json LOGS/power/** + result_processing_${{ env.RESULT_FILENAME }}.json LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json LOGS/agentic/**/agentic_power_timezone_offset.txt diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index b08ef0775..4d1b817ea 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -395,14 +395,14 @@ jobs: fi - name: Process result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ always() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} env: RUNNER_TYPE: ${{ inputs.runner }} run: | python3 utils/process_result.py - name: Upload result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bmk_${{ env.RESULT_FILENAME }} diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index daba2715b..a1e357b30 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -92,10 +92,20 @@ The fixed-sequence transformer requires runner, framework, precision, speculativ | Latency and interactivity | Each benchmark input key ending in `ms` is converted from milliseconds to seconds with `_ms` removed. Keys containing `tpot` also produce an `intvty` reciprocal. | | Optional runtime metadata | `router` as exactly `{name, version}`, `kv_p2p_transfer`, and measured power patched from `gpu_metrics.csv` when available | -Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physical GPU count. Multinode per-GPU denominators use the declared prefill and decode GPU counts. Invalid or missing required metadata fails transformation. Power aggregation is explicitly best effort and cannot fail the benchmark aggregate. +Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physical GPU count. Multinode per-GPU denominators use the declared prefill and decode GPU counts. Invalid or missing required metadata fails transformation. Power aggregation is best effort by default; `REQUIRE_POWER=1` fails the job after preserving available results and audits when power validation fails. InferenceX-app treats routing fields as columns or config dimensions and stores numeric measurements in `benchmark_results.metrics` JSONB. The mapper supports v1 shared topology, v2 split prefill/decode topology, and nested v3 AgentX metrics. Unknown numeric metrics are retained and warned about, which permits schema growth without silently losing numeric data. +### Fixed-sequence outcomes and PowerX audits + +The serving client records `benchmark_outcome` before saving its raw result. It retains the existing maximum request-failure rate of 5%, including the requested/completed/failed counts. The processor verifies this record, copies it to the aggregate, and returns failure even when telemetry is valid. Zero successful requests retain a diagnostic aggregate without fabricated reciprocal latency. Legacy results without outcome metadata remain distinguishable; power validity alone never establishes benchmark success or answer quality. + +`power_invalid_reasons` and `power_audit` carry a bounded summary alongside numeric metrics. The summary includes the available measurement window, expected/observed GPU counts, sampling diagnostics, observed device identifiers and producer pin. Its `source` names the retained `power_validation_*.json` sidecar. Device identifiers retain the collector's semantics; local SMI indices are not physical UUID proof. + +For multinode fixed-sequence jobs, `utils/process_result.py --all` processes every available result before returning failure. It accepts `_c_gpus_...`, `_conc_gpus_...`, and AMD `_concurrency__req_rate__gpus_...` filenames, including `inf` request rates. It compares result concurrencies with `CONC_LIST`, rejects duplicate or contradictory point identities, and records omissions/errors in `result_processing_.json`. Aggregate workers pass `AGGREGATE_GPUS` with zero role GPU counts to telemetry validation; separate prefill/decode energy remains absent. Existing layout fields remain compatible, and `num_aggregate_gpu` marks the shared allocation. + +Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index cc5c9e302..6a054d5f0 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -92,10 +92,20 @@ shape: array of benchmark row objects | 延迟和交互性 | 基准输入中每个以 `ms` 结尾的键都会从毫秒换算为秒，并移除 `_ms`。包含 `tpot` 的键还会产生其倒数 `intvty`。 | | 可选运行时元数据 | 形式必须精确为 `{name, version}` 的 `router`、`kv_p2p_transfer`，以及在可用时由 `gpu_metrics.csv` 补入的实测功耗 | -单节点 GPU 数为 `tp * pp * pcp_size`。DCP 不会增加物理 GPU 数。多节点每 GPU 指标的分母使用声明的 prefill 和 decode GPU 数。无效或缺失的必需元数据会使转换失败。功耗聚合明确采用尽力而为模式，不能导致基准聚合失败。 +单节点 GPU 数为 `tp * pp * pcp_size`。DCP 不会增加物理 GPU 数。多节点每 GPU 指标的分母使用声明的 prefill 和 decode GPU 数。无效或缺失的必需元数据会使转换失败。功耗聚合默认尽力而为；当设置 `REQUIRE_POWER=1` 时，功耗验证失败会在保留已有结果和审计后使任务失败。 InferenceX-app 将路由字段作为列或配置维度，并把数值测量存入 `benchmark_results.metrics` JSONB。映射器支持共享拓扑的 v1、拆分 prefill/decode 拓扑的 v2，以及嵌套 AgentX 指标的 v3。未知数值指标会被保留并产生警告，因此架构可以扩展，同时不会无提示地丢失数值数据。 +### 固定序列基准结果状态与 PowerX 审计 + +服务客户端在保存原始结果前写入 `benchmark_outcome`，保留现有的 5% 最大请求失败率，以及请求总数、完成数和失败数。处理器检查该记录并复制到聚合结果中；即使遥测有效，请求失败率超限仍返回失败。零成功请求会保留诊断聚合结果，但不会生成不存在的延迟倒数。没有状态元数据的历史结果仍可区分；功耗有效不能证明基准成功或答案质量。 + +`power_invalid_reasons` 和 `power_audit` 在数值指标旁携带有界摘要，包括可用的测量窗口、预期与观测 GPU 数、采样诊断、观测设备标识和生产者版本。`source` 指向保留的 `power_validation_*.json` 工件名称。设备标识保留采集器原有语义，本地 SMI 序号不是物理 UUID 的证明。 + +对于多节点固定序列任务，`utils/process_result.py --all` 先处理所有已有结果，再返回失败。它接受 `_c_gpus_...`、`_conc_gpus_...` 和 AMD 的 `_concurrency__req_rate__gpus_...` 文件名，也支持 `inf` 请求速率。它将结果并发度与 `CONC_LIST` 比较，拒绝重复或矛盾的点身份，并将遗漏和错误记录到 `result_processing_.json`。共享工作池通过 `AGGREGATE_GPUS` 及零值角色 GPU 数进行遥测验证；独立的 prefill/decode 能耗保持缺失。现有布局字段保持兼容，`num_aggregate_gpu` 明确标记共享分配。 + +启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 + ## 评测工件 ### 单配置身份和收集 diff --git a/infx/bench_serving/benchmark_outcome.py b/infx/bench_serving/benchmark_outcome.py new file mode 100644 index 000000000..e14dca470 --- /dev/null +++ b/infx/bench_serving/benchmark_outcome.py @@ -0,0 +1,31 @@ +"""The serving client's existing request-failure gate, preserved with results.""" + +from __future__ import annotations + +from typing import Literal, TypedDict + + +MAX_FAILURE_RATE = 0.05 + + +class BenchmarkOutcome(TypedDict): + status: Literal["passed", "failed"] + requested: int + completed: int + failed: int + max_failure_rate: float + + +def benchmark_outcome(requested: int, completed: int) -> BenchmarkOutcome: + """Record the same five-percent gate before cleanup or artifact upload.""" + if (type(requested) is not int or type(completed) is not int + or requested <= 0 or not 0 <= completed <= requested): + raise ValueError("Benchmark request counts must satisfy 0 <= completed <= requested > 0") + failed = requested - completed + return { + "status": "failed" if failed / requested > MAX_FAILURE_RATE else "passed", + "requested": requested, + "completed": completed, + "failed": failed, + "max_failure_rate": MAX_FAILURE_RATE, + } diff --git a/infx/bench_serving/benchmark_serving.py b/infx/bench_serving/benchmark_serving.py index 82dfec973..1619d28ae 100644 --- a/infx/bench_serving/benchmark_serving.py +++ b/infx/bench_serving/benchmark_serving.py @@ -62,6 +62,7 @@ from argparse import ArgumentParser as FlexibleArgumentParser from .benchmark_utils import convert_to_pytorch_benchmark_format +from .benchmark_outcome import benchmark_outcome from .encoding_dsv4 import encode_messages as dsv4_encode_messages MILLISECONDS_TO_SECONDS_CONVERSION = 1000 @@ -898,6 +899,11 @@ def main(args: argparse.Namespace): lora_modules=args.lora_modules, )) + # Preserve the request gate before writing results: cleanup or a surrounding + # launcher must not turn an error-containing benchmark into a passed point. + outcome = benchmark_outcome(args.num_prompts, benchmark_result["completed"]) + benchmark_result["benchmark_outcome"] = outcome + # Save config and results to json if args.save_result: result_json: Dict[str, Any] = {} @@ -957,14 +963,11 @@ def main(args: argparse.Namespace): json.dump(result_json, outfile) save_to_pytorch_benchmark_format(args, result_json, file_name) - max_failure_rate = 0.05 - completed = benchmark_result["completed"] - failure_rate = 1 - completed / args.num_prompts - if failure_rate > max_failure_rate: + if outcome["status"] == "failed": raise SystemExit( - f"FAIL: request failure rate {failure_rate:.1%} exceeds " - f"{max_failure_rate:.0%} threshold " - f"({completed}/{args.num_prompts} completed)" + f"FAIL: request failure rate {outcome['failed'] / outcome['requested']:.1%} exceeds " + f"{outcome['max_failure_rate']:.0%} threshold " + f"({outcome['completed']}/{outcome['requested']} completed)" ) diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index ef6ad2b8e..f6fa76102 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -3,7 +3,9 @@ from __future__ import annotations import json +import math import os +import re import sys from collections.abc import Iterable, Mapping from pathlib import Path @@ -12,6 +14,7 @@ from .metadata import parse_component_metadata from .power import ALL_POWER_METRIC_KEYS, POWER_METRIC_SCHEMA_VERSION, with_power_metrics from .topology import Parallelism, validate_parallelism +from infx.bench_serving.benchmark_outcome import benchmark_outcome _BASE_ENV_VARS = ( @@ -49,6 +52,14 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s 'isl': int(env['ISL']), 'osl': int(env['OSL']), } + if 'benchmark_outcome' in benchmark: + outcome = benchmark['benchmark_outcome'] + expected = benchmark_outcome(outcome['requested'], outcome['completed']) + if (outcome != expected + or benchmark.get('completed', expected['completed']) != expected['completed'] + or benchmark.get('num_prompts', expected['requested']) != expected['requested']): + raise ValueError("Benchmark outcome does not match the recorded request counts and gate") + data['benchmark_outcome'] = expected router = parse_component_metadata(env.get('ROUTER_METADATA'), 'ROUTER_METADATA') if router is not None: @@ -61,9 +72,6 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s is_multinode = env.get('IS_MULTINODE', 'false').lower() == 'true' if is_multinode: - # TODO: Eventually will have to have a separate condition in here for multinode disagg and - # multinode agg. For now, just assume that multinode implies disagg. - multinode_vars = ['PREFILL_GPUS', 'DECODE_GPUS', 'PREFILL_NUM_WORKERS', 'PREFILL_TP', 'PREFILL_EP', 'PREFILL_DP_ATTN', 'DECODE_NUM_WORKERS', 'DECODE_TP', 'DECODE_EP', 'DECODE_DP_ATTN'] @@ -76,6 +84,14 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s ) prefill_gpus = int(env['PREFILL_GPUS']) decode_gpus = int(env['DECODE_GPUS']) + aggregate_gpus = int(env.get('AGGREGATE_GPUS', '0')) + if aggregate_gpus: + if aggregate_gpus < 0 or prefill_gpus or decode_gpus or disagg: + raise ValueError("Aggregate GPUs require non-disaggregated topology without role GPUs") + # Preserve the existing multinode layout fields for consumers, but + # expose the actual shared role and validate telemetry as `agg`. + prefill_gpus = aggregate_gpus + data['num_aggregate_gpu'] = aggregate_gpus prefill_num_workers = int(env['PREFILL_NUM_WORKERS']) prefill = Parallelism( tp=int(env['PREFILL_TP']), @@ -154,9 +170,9 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s data = data | single_node_data for key, value in benchmark.items(): - if key.endswith('ms'): + if key.endswith('ms') and math.isfinite(float(value)): data[key.replace('_ms', '')] = float(value) / 1000.0 - if 'tpot' in key: + if 'tpot' in key and math.isfinite(float(value)) and float(value) > 0: data[key.replace('_ms', '').replace( 'tpot', 'intvty')] = 1000.0 / float(value) return data @@ -221,7 +237,8 @@ def aggregate_power_result( source = Path(env.get('POWER_ARTIFACT_DIR', 'LOGS/power')) prefill_gpus = int(env['PREFILL_GPUS']) decode_gpus = int(env['DECODE_GPUS']) - expected_num_gpus = prefill_gpus + decode_gpus + aggregate_gpus = int(env.get('AGGREGATE_GPUS', '0')) + expected_num_gpus = prefill_gpus + decode_gpus + aggregate_gpus else: candidates = [env.get('GPU_METRICS_CSV'), 'gpu_metrics.csv', '/workspace/gpu_metrics.csv'] source = next( @@ -237,6 +254,7 @@ def aggregate_power_result( source, bench_path, agg_path, prefill_gpus=prefill_gpus, decode_gpus=decode_gpus, + aggregate_gpus=aggregate_gpus, expected_producer_sha=env.get('POWER_PRODUCER_SHA') or None, logs_root=Path(env.get('POWER_RESULT_ROOT', 'LOGS')), validation_result=validation_path, @@ -265,8 +283,7 @@ def aggregate_power_result( return int(require_power) -def main() -> int: - env = os.environ +def process_result(env: Mapping[str, str]) -> int: require_environment(env, _BASE_ENV_VARS) result_filename = env['RESULT_FILENAME'] bench_path = Path(f'{result_filename}.json') @@ -277,10 +294,86 @@ def main() -> int: with open(agg_path, 'w') as f: json.dump(data, f, indent=2) status = aggregate_power_result(env, bench_path, agg_path) + validation_path = Path(f'power_validation_{result_filename}.json') + from .power.audit import audit_summary + + result = json.loads(agg_path.read_text()) + try: + validation = json.loads(validation_path.read_text()) + result.update(audit_summary(validation, validation_path.name)) + except (OSError, ValueError, TypeError) as exc: + print(f'[process_result] audit summary unavailable: {exc}', file=sys.stderr) + result['power_invalid_reasons'] = ['validation_artifact_unavailable'] + # A required run must preserve its audit as well as numeric metrics. + status = max(status, int(env.get('REQUIRE_POWER', '').lower() in {'1', 'true', 'yes'})) + agg_path.write_text(json.dumps(result, indent=2)) with open(agg_path) as f: print(json.dumps(json.load(f), indent=2)) + return max(status, int(data.get('benchmark_outcome', {}).get('status') == 'failed')) + + +def process_multinode_results(env: Mapping[str, str]) -> int: + """Process every available point and preserve sweep omissions before failing.""" + expected = {int(value) for value in env['CONC_LIST'].split()} + if not expected or min(expected) <= 0: + raise ValueError('CONC_LIST must contain positive concurrencies') + points: list[dict[str, Any]] = [] + observed: set[int] = set() + status = 0 + for path in sorted(Path('.').glob(f"{env['RESULT_FILENAME']}_*.json")): + point: dict[str, Any] = {'source': path.name} + try: + match = re.search( + r'_(?:c|conc|concurrency_)(\d+)(?:_req_rate_[^_]+)?_gpus_(\d+)' + r'(?:_ctx_(\d+)_gen_(\d+))?$', path.stem) + if match is None: + raise ValueError('Result filename lacks concurrency and physical GPU counts') + concurrency, total = int(match[1]), int(match[2]) + raw = json.loads(path.read_text()) + if raw['max_concurrency'] != concurrency: + raise ValueError('Result concurrency does not match its filename') + if concurrency in observed: + raise ValueError('Duplicate concurrency result within one recipe') + observed.add(concurrency) + point['concurrency'] = concurrency + point_env = {**env, 'RESULT_FILENAME': path.stem, 'IS_MULTINODE': 'true'} + disagg = env['DISAGG'].lower() == 'true' + # Some disaggregated recipe groups include a shared-worker point. + # Its zero decode workers and filename must both describe aggregate + # execution; the group-level flag cannot manufacture role energy. + aggregate = not disagg or int(env['DECODE_NUM_WORKERS']) == 0 + if aggregate: + if match[3] is not None and (int(match[3]) != total or int(match[4]) != 0): + raise ValueError('Aggregate result contains separate role GPU counts') + point_env.update(DISAGG='false', PREFILL_GPUS='0', DECODE_GPUS='0', + AGGREGATE_GPUS=str(total)) + else: + if match[3] is None: + raise ValueError('Disaggregated result lacks prefill/decode GPU counts') + prefill, decode = int(match[3]), int(match[4]) + if prefill + decode != total: + raise ValueError('Role GPU counts do not equal total GPU count') + point_env.update(PREFILL_GPUS=str(prefill), DECODE_GPUS=str(decode), AGGREGATE_GPUS='0') + point['exit_code'] = process_result(point_env) + except (OSError, ValueError, KeyError, TypeError) as exc: + point.update(exit_code=1, error=str(exc)) + print(f'[process_result] {path}: {exc}', file=sys.stderr) + status = max(status, point['exit_code']) + points.append(point) + missing, unexpected = sorted(expected - observed), sorted(observed - expected) + status = max(status, int(bool(missing or unexpected))) + summary = {'expected_concurrencies': sorted(expected), + 'missing_concurrencies': missing, 'unexpected_concurrencies': unexpected, + 'points': points, 'exit_code': status} + Path(f"result_processing_{env['RESULT_FILENAME']}.json").write_text(json.dumps(summary, indent=2)) + if missing or unexpected: + print(f'[process_result] incomplete sweep: missing={missing}, unexpected={unexpected}', file=sys.stderr) return status +def main() -> int: + return process_multinode_results(os.environ) if sys.argv[1:] == ['--all'] else process_result(os.environ) + + if __name__ == '__main__': sys.exit(main()) diff --git a/infx/results/power/__init__.py b/infx/results/power/__init__.py index f0e1da95c..8007f5a91 100644 --- a/infx/results/power/__init__.py +++ b/infx/results/power/__init__.py @@ -59,6 +59,7 @@ def with_power_metrics( data["power_metric_schema_version"] = schema_version data["power_valid"] = int(power_valid) data.pop("power_invalid_reasons", None) + data.pop("power_audit", None) if power_valid: for key, value in metrics.items(): if value is None or not math.isfinite(value): diff --git a/infx/results/power/audit.py b/infx/results/power/audit.py new file mode 100644 index 000000000..6463976cf --- /dev/null +++ b/infx/results/power/audit.py @@ -0,0 +1,50 @@ +"""Bounded public audit metadata from retained power-validation sidecars.""" + +from __future__ import annotations + +import math +import re +from collections.abc import Mapping +from typing import Any + + +def audit_summary(validation: Mapping[str, Any], source: str) -> dict[str, Any]: + """Project the shared app audit contract without publishing raw telemetry.""" + audit: dict[str, Any] = {"source": source} + window = validation.get("benchmark_window") or {} + producer = validation.get("producer") or {} + fields = { + "window_start_unix": window.get("start_time_unix"), + "window_end_unix": window.get("end_time_unix"), + "expected_gpu_count": validation.get("expected_gpu_count"), + "observed_gpu_count": validation.get("observed_gpu_count"), + } + counts = validation.get("per_gpu_sample_counts") or {} + if counts: + fields["sample_count"] = sum(counts.values()) + gaps = validation.get("per_gpu_max_sample_gap_s") or {} + finite_gaps = [value for value in gaps.values() + if type(value) in (int, float) and math.isfinite(value)] + if finite_gaps: + fields["max_sample_gap_s"] = max(finite_gaps) + for key, value in fields.items(): + if type(value) in (int, float) and math.isfinite(value) and value >= 0: + audit[key] = value + for target, original in (("producer_sha", "producer_git_commit"), + ("exporter_image_sha256", "exporter_image_sha256")): + value = producer.get(original) + if isinstance(value, str) and 0 < len(value) <= 128: + audit[target] = value + ids = validation.get("observed_gpu_ids") + if ids is None: + ids = list((validation.get("per_gpu_role") or {}).keys()) + if ids: + audit["observed_gpu_ids"] = list(dict.fromkeys( + str(value) for value in ids if 0 < len(str(value)) <= 128 + ))[:1024] + return { + "power_invalid_reasons": [reason for reason in validation.get("reasons", []) + if isinstance(reason, str) + and re.fullmatch(r"[a-z][a-z0-9_]{0,63}", reason)][:32], + "power_audit": audit, + } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 88c163461..f3b85de24 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7399,3 +7399,14 @@ - "Disable adaptive verification in the EVAL_ONLY DSpark config as well. It trims verification requests on device, which the ROCm DeepseekV4IndexerBackend does not support, so the eval-only engine refused to start (run 34651830283, c32). Evals keep real block rejection; throughput settings are unchanged." - "EVAL_ONLY 的 DSpark 配置同样关闭自适应验证：它会在设备端裁剪验证请求，而 ROCm 的 DeepseekV4IndexerBackend 不支持该操作，导致仅评测引擎拒绝启动（运行 34651830283，c32）。评测仍保留真实块拒绝采样；吞吐设置不变。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Preserve request failure outcomes, retain all available fixed-sequence diagnostics, and reject incomplete concurrency batches before publishing benchmark artifacts." + - "保留请求失败状态及全部可用的固定序列诊断信息；发布基准产物前拒绝缺失并发点的批次。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 485021b36..d14d6a87a 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -28,12 +28,13 @@ def test_workflow_processes_results_through_compatibility_entrypoint( step = next(step for job in workflow["jobs"].values() for step in job.get("steps", []) if step.get("name", "").startswith("Process result")) multinode = workflow_name == "benchmark-multinode-tmpl.yml" - stem = "fixture_gpus_28_ctx_20_gen_8" if multinode else "fixture" + stem = "fixture_conc64_gpus_28_ctx_20_gen_8" if multinode else "fixture" (tmp_path / f"{stem}.json").write_text(json.dumps({ **sample_benchmark_result, "total_token_throughput": 56, "output_throughput": 28, })) env = {**(multinode_env_vars if multinode else single_node_env_vars), "RESULT_FILENAME": "fixture" if multinode else stem, "POWER_PRODUCER_SHA": "", + "CONC_LIST": "64", "PATH": f"{Path(sys.executable).parent}:{os.environ['PATH']}"} result = subprocess.run(["bash", "-euo", "pipefail", "-c", step["run"]], cwd=tmp_path, env=env, capture_output=True, text=True, timeout=10) @@ -845,7 +846,7 @@ def test_missing_csv_does_not_break_process_result(self, tmp_path, single_node_e assert "avg_power_w" not in patched assert "joules_per_output_token" not in patched assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] validation = json.loads( (tmp_path / "power_validation_benchmark_result.json").read_text() @@ -876,7 +877,7 @@ def test_missing_bench_timestamps_does_not_patch(self, tmp_path, single_node_env assert "avg_power_w" not in patched assert "joules_per_output_token" not in patched assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] def test_expected_gpu_count_mismatch_is_invalid(self, tmp_path, single_node_env_vars): """TP/PP/PCP topology is checked against the observed device IDs.""" @@ -902,7 +903,7 @@ def test_expected_gpu_count_mismatch_is_invalid(self, tmp_path, single_node_env_ assert result.returncode == 0, f"Script failed: {result.stderr}" patched = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] assert "total_gpu_energy_j" not in patched def test_require_power_propagates_validation_failure( @@ -1009,7 +1010,7 @@ def test_internal_aggregation_error_is_always_auditable( agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 0 - assert "power_invalid_reasons" not in agg + assert agg["power_invalid_reasons"] == ["aggregation_internal_error"] validation = json.loads( (tmp_path / "power_validation_benchmark_result.json").read_text() ) @@ -1416,3 +1417,209 @@ def test_strict_mode_passes_on_valid_package(self, tmp_path, power_env): assert result.returncode == 0, f"Script failed: {result.stderr}" agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert agg["power_valid"] == 1 + + +@pytest.mark.parametrize('completed,status', [(100, 'passed'), (95, 'passed'), (94, 'failed')]) +def test_request_outcome_preserves_existing_failure_threshold(completed, status): + from infx.bench_serving.benchmark_outcome import benchmark_outcome + + outcome = benchmark_outcome(100, completed) + assert outcome['status'] == status + assert outcome['failed'] == 100 - completed + assert outcome['max_failure_rate'] == 0.05 + + +def test_failed_client_is_preserved_with_valid_power(tmp_path, single_node_env_vars): + start, end = 1_700_000_100.0, 1_700_000_110.0 + TestPowerAggregationIntegration._write_nvidia_csv(tmp_path / 'gpu_metrics.csv', start, end, num_gpus=2) + outcome = {'status': 'failed', 'requested': 100, 'completed': 94, 'failed': 6, + 'max_failure_rate': 0.05} + raw = {'model_id': 'fixture', 'max_concurrency': 4, 'total_token_throughput': 500, + 'output_throughput': 100, 'benchmark_start_time_unix': start, + 'benchmark_end_time_unix': end, 'duration': 10, 'completed': 94, + 'total_input_tokens': 8192, 'total_output_tokens': 1024, + 'benchmark_outcome': outcome} + result = run_script(tmp_path, {**single_node_env_vars, 'TP': '2'}, raw) + assert result.returncode == 1, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['benchmark_outcome'] == outcome + assert aggregate['power_valid'] == 1 + assert aggregate['power_invalid_reasons'] == [] + assert aggregate['power_audit']['window_start_unix'] == start + assert aggregate['power_audit']['expected_gpu_count'] == 2 + assert aggregate['power_audit']['observed_gpu_count'] == 2 + assert aggregate['power_audit']['source'] == 'power_validation_benchmark_result.json' + + +def test_request_outcome_cannot_disagree_with_raw_counts(single_node_env_vars, sample_benchmark_result): + from infx.results.fixed_sequence import build_result + + raw = {**sample_benchmark_result, 'completed': 94, + 'benchmark_outcome': {'status': 'passed', 'requested': 100, 'completed': 100, + 'failed': 0, 'max_failure_rate': 0.05}} + with pytest.raises(ValueError, match='request counts and gate'): + build_result(raw, single_node_env_vars) + + +def test_multinode_aggregate_role_through_result_processor(tmp_path, multinode_env_vars): + pkg = build_package(tmp_path, bench_extra=TestMultinodePower.BENCH_EXTRA) + manifest_path = pkg.power_dir / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + for device in manifest['expected_devices']: + for assignment in device['assignments']: + assignment.update(worker_role='agg', het_group=None) + manifest_path.write_text(json.dumps(manifest)) + env = {**multinode_env_vars, 'DISAGG': 'false', 'PREFILL_GPUS': '0', + 'DECODE_GPUS': '0', 'AGGREGATE_GPUS': '4', 'POWER_PRODUCER_SHA': PRODUCER_SHA, + 'REQUIRE_POWER': '1'} + result = run_script(tmp_path, env, json.loads(pkg.original_result.read_text())) + assert result.returncode == 0, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['power_valid'] == 1 + assert aggregate['num_aggregate_gpu'] == 4 + assert aggregate['avg_power_w'] == 350 + assert aggregate['power_audit']['producer_sha'] == PRODUCER_SHA + assert set(ROLE_METRIC_KEYS).isdisjoint(aggregate) + + +@pytest.mark.parametrize('conc_token,rate_suffix', [ + ('c', ''), ('conc', ''), + ('concurrency_', '_req_rate_1'), ('concurrency_', '_req_rate_inf'), +]) +@pytest.mark.parametrize('expected_concs,missing', [('4 8 16', [8]), ('4 16', [])]) +def test_multinode_batch_preserves_points_and_checks_completeness( + tmp_path, multinode_env_vars, sample_benchmark_result, conc_token, rate_suffix, + expected_concs, missing, +): + for conc in (4, 16): + (tmp_path / f'run_recipe_{conc_token}{conc}{rate_suffix}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': conc})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': expected_concs, 'REQUIRE_POWER': '0'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == int(bool(missing)), result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == missing + assert receipt['unexpected_concurrencies'] == [] + assert len(receipt['points']) == 2 + for conc in (4, 16): + stem = f'run_recipe_{conc_token}{conc}{rate_suffix}_gpus_4_ctx_2_gen_2' + aggregate = json.loads((tmp_path / f'agg_{stem}.json').read_text()) + assert aggregate['power_valid'] == 0 + assert aggregate['power_invalid_reasons'] + assert (tmp_path / f'power_validation_{stem}.json').is_file() + + +@pytest.mark.parametrize('role_suffix', ['', '_ctx_4_gen_0']) +def test_multinode_batch_normalizes_legacy_zero_decode_aggregate( + tmp_path, multinode_env_vars, role_suffix, +): + pkg = build_package(tmp_path, bench_extra=TestMultinodePower.BENCH_EXTRA) + manifest_path = pkg.power_dir / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + for device in manifest['expected_devices']: + for assignment in device['assignments']: + assignment.update(worker_role='agg', het_group=None) + manifest_path.write_text(json.dumps(manifest)) + stem = f'run_recipe_c4_gpus_4{role_suffix}' + (tmp_path / f'{stem}.json').write_text(pkg.original_result.read_text()) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4', 'DECODE_NUM_WORKERS': '0', 'REQUIRE_POWER': '1', + 'POWER_PRODUCER_SHA': PRODUCER_SHA} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 0, result.stderr + aggregate = json.loads((tmp_path / f'agg_{stem}.json').read_text()) + assert aggregate['disagg'] is False + assert aggregate['num_aggregate_gpu'] == 4 + assert aggregate['num_decode_gpu'] == 0 + assert aggregate['decode_num_workers'] == 0 + assert aggregate['power_valid'] == 1 + assert aggregate['avg_power_w'] == 350 + assert set(ROLE_METRIC_KEYS).isdisjoint(aggregate) + + +@pytest.mark.parametrize('decode_workers,role_suffix', [ + ('0', '_ctx_2_gen_2'), ('0', '_ctx_3_gen_0'), ('1', ''), +]) +def test_multinode_batch_rejects_misdeclared_aggregate_role_counts( + tmp_path, multinode_env_vars, sample_benchmark_result, decode_workers, role_suffix, +): + stem = f'run_recipe_c4_gpus_4{role_suffix}' + (tmp_path / f'{stem}.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': 4})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4', 'DECODE_NUM_WORKERS': decode_workers} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['points'][0]['exit_code'] == 1 + assert not (tmp_path / f'agg_{stem}.json').exists() + + +def test_zero_successful_requests_preserves_diagnostic_result(tmp_path, single_node_env_vars, sample_benchmark_result): + raw = {**sample_benchmark_result, 'tpot_p50_ms': 0, 'tpot_p99_ms': float('nan'), + 'completed': 0, 'num_prompts': 4, + 'benchmark_outcome': {'status': 'failed', 'requested': 4, 'completed': 0, + 'failed': 4, 'max_failure_rate': 0.05}} + result = run_script(tmp_path, single_node_env_vars, raw) + assert result.returncode == 1, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['benchmark_outcome']['failed'] == 4 + assert aggregate['power_valid'] == 0 + assert 'intvty_p50' not in aggregate + assert 'tpot_p99' not in aggregate + + +@pytest.mark.parametrize('extra_conc,error', [(4, 'Duplicate concurrency'), (8, None)]) +def test_multinode_batch_rejects_extra_results_without_losing_other_points( + tmp_path, multinode_env_vars, sample_benchmark_result, extra_conc, error, +): + for label, conc in [('a', 4), ('b', extra_conc), ('c', 16)]: + (tmp_path / f'run_{label}_conc{conc}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': conc})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4 16'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [] + assert receipt['unexpected_concurrencies'] == ([] if error else [8]) + if error: + assert error in receipt['points'][1]['error'] + assert (tmp_path / 'agg_run_a_conc4_gpus_4_ctx_2_gen_2.json').is_file() + assert (tmp_path / 'agg_run_c_conc16_gpus_4_ctx_2_gen_2.json').is_file() + + +def test_multinode_empty_sweep_records_every_missing_point(tmp_path, multinode_env_vars): + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4 8'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [4, 8] + assert receipt['points'] == [] + + +def test_public_power_audit_bounds_text_and_device_identifiers(): + from infx.results.power.audit import audit_summary + + summary = audit_summary({ + 'producer': {'producer_git_commit': 'x' * 129, 'exporter_image_sha256': 'a' * 64}, + 'observed_gpu_ids': ['gpu0', 'gpu0', 'x' * 129] + [f'gpu{i}' for i in range(1, 1025)], + 'reasons': ['window_missing', ''] + [f'reason_{i}' for i in range(40)], + }, 'power_validation_run.json') + assert len(summary['power_invalid_reasons']) == 32 + assert '' not in summary['power_invalid_reasons'] + audit = summary['power_audit'] + assert 'producer_sha' not in audit + assert audit['exporter_image_sha256'] == 'a' * 64 + assert len(audit['observed_gpu_ids']) == 1024 + assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] From 57ae800333ac5d85c6cfb422e1b336a442cc2716 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:23:55 -0700 Subject: [PATCH 02/21] feat: add scoped fixed-sequence power requirements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为固定序列场景增加默认关闭的功耗要求，并贯通矩阵及工作流；保持 AgentX 和评估隔离。 --- .github/workflows/e2e-tests.yml | 4 +-- .github/workflows/run-sweep.yml | 3 ++ docs/configuration-procedures.md | 2 ++ docs/configuration-procedures_zh.md | 2 ++ infx/matrix/generate.py | 7 +++++ infx/matrix/validation.py | 5 ++++ .../test_generate_sweep_configs.py | 29 +++++++++++++++++++ 7 files changed, 50 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 0718ef718..f3dc97527 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -368,7 +368,7 @@ jobs: decode-dp-attn: ${{ matrix.config.decode.dp-attn }} decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} run-eval: false - require-power: ${{ inputs.require-power }} + require-power: ${{ inputs.require-power || (matrix.config['require-power'] == true && matrix.config['eval-only'] != true) }} power-producer-sha: ${{ inputs.power-producer-sha }} ref: ${{ inputs.ref }} @@ -692,7 +692,7 @@ jobs: spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} run-eval: false - require-power: ${{ inputs.require-power }} + require-power: ${{ inputs.require-power || (matrix.config['require-power'] == true && matrix.config['eval-only'] != true) }} ref: ${{ inputs.ref }} test-sweep-evals: diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 747017020..88afbba16 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -413,6 +413,7 @@ jobs: klaud-run: &klaud-run ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login == 'Klaud-Cold' && (startsWith(github.head_ref, 'klaud/auto-') || startsWith(github.head_ref, 'klaude/auto-')) }} exp-name: ${{ matrix.config.exp-name }} recipe-fingerprint: ${{ matrix.config['recipe-fingerprint'] || '' }} + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} isl: ${{ matrix.config.isl }} osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} @@ -474,6 +475,7 @@ jobs: kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} exp-name: ${{ matrix.config.exp-name }} recipe-fingerprint: ${{ matrix.config['recipe-fingerprint'] || '' }} + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} @@ -540,6 +542,7 @@ jobs: klaud-run: *klaud-run exp-name: ${{ matrix.config.exp-name }} recipe-fingerprint: ${{ matrix.config['recipe-fingerprint'] || '' }} + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} isl: ${{ matrix.config.isl }} osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index d853f6493..581e5d09a 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -88,6 +88,8 @@ Sources: [`configs/CONFIGS.md`](../configs/CONFIGS.md), [`validation.py`](../uti 6. For srt-slurm, update recipe and master entry together. For llm-d, update the llm-d recipe/orchestration and master entry together. 7. Append the trigger entry, generate only the affected key first, and inspect every emitted point. +Fixed-sequence `8192/1024` scenarios may set `require-power: true` to opt into validated measured power. The matrix passes this flag to standard sweeps and manual E2E throughput jobs; eval-only and AgentX rows do not inherit it. Omit the field to preserve existing behavior. Enable it only alongside the corresponding runtime and result adapter, then qualify the complete selected scope. + ## Register and set up a runner Setup source: [`utils/runner_setup/RUNNER_SETUP.md`](../utils/runner_setup/RUNNER_SETUP.md). Config source: [`configs/CONFIGS.md#runners`](../configs/CONFIGS.md#runners). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 883f1f8e2..aeff271a8 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -88,6 +88,8 @@ STP（Single Token Prediction，单 Token 预测）是每次前向传播生成 6. srt-slurm 必须同时更新配方和主条目；llm-d 必须同时更新 llm-d 配方/编排和主条目。 7. 追加触发条目，先只生成受影响的 key，并检查每个生成点。 +固定序列 `8192/1024` 场景可设置 `require-power: true`，要求经过验证的实测功耗。矩阵将此标记传递给标准 sweep 和手动 E2E 吞吐作业；eval-only 和 AgentX 行不继承该标记。省略此字段可保留现有行为。仅在对应 runtime 和结果适配器同时交付时启用，然后验证完整选定范围。 + ## 注册并设置 runner 设置来源：[`utils/runner_setup/RUNNER_SETUP.md`](../utils/runner_setup/RUNNER_SETUP.md)。配置来源：[`configs/CONFIGS.md#runners`](../configs/CONFIGS.md#runners)。 diff --git a/infx/matrix/generate.py b/infx/matrix/generate.py index 55c46da58..ee6166045 100644 --- a/infx/matrix/generate.py +++ b/infx/matrix/generate.py @@ -136,6 +136,7 @@ def smoke_entries(entries: list[dict]) -> list[dict]: if not row.get('run-eval'): continue row = {**row, 'eval-only': True} + row.pop(Fields.REQUIRE_POWER.value, None) if row.get('prefill') is not None: row['conc'] = [row['eval-conc']] row['eval-all-concs'] = False @@ -808,6 +809,9 @@ def _fixed_sequence_entries( is_multinode = config.get(Fields.MULTINODE.value, False) disagg = config.get(Fields.DISAGG.value, False) isl, osl = sequence[Fields.ISL.value], sequence[Fields.OSL.value] + require_power = sequence.get(Fields.REQUIRE_POWER.value, False) + if require_power and (isl, osl) != (8192, 1024): + raise ValueError("require-power rollout supports only fixed-sequence 8192/1024") model_code = config[Fields.MODEL_PREFIX.value] spec_decoding = benchmark.get(Fields.SPEC_DECODING.value, "none") if is_multinode: @@ -831,6 +835,8 @@ def _fixed_sequence_entries( Fields.ISL.value: isl, Fields.OSL.value: osl, } + if require_power: + entry[Fields.REQUIRE_POWER.value] = True if is_multinode: entry.update({ Fields.SPEC_DECODING.value: spec_decoding, @@ -1316,6 +1322,7 @@ def select_matrix_evals( rows = [row for row in rows if row.get(Fields.RUN_EVAL.value, False)] for row in rows: row[Fields.EVAL_ONLY.value] = True + row.pop(Fields.REQUIRE_POWER.value, None) return rows diff --git a/infx/matrix/validation.py b/infx/matrix/validation.py index a3614b0bc..8ea0acffb 100644 --- a/infx/matrix/validation.py +++ b/infx/matrix/validation.py @@ -76,6 +76,7 @@ class Fields(Enum): NUM_NODES = 'num-nodes' NODE_COUNT = 'node-count' DURATION = 'duration' + REQUIRE_POWER = 'require-power' # Matrix entry fields CONC = 'conc' @@ -168,6 +169,7 @@ class SingleNodeMatrixEntry(BaseModel): runner: str isl: int osl: int + require_power: bool = Field(default=False, alias=Fields.REQUIRE_POWER.value, strict=True) tp: int pp: int = Field(gt=0, strict=True) dcp_size: int = Field(alias=Fields.DCP_SIZE.value, gt=0, strict=True) @@ -268,6 +270,7 @@ class MultiNodeMatrixEntry(BaseModel): node_count: int = Field(alias=Fields.NODE_COUNT.value, gt=0, strict=True) isl: int osl: int + require_power: bool = Field(default=False, alias=Fields.REQUIRE_POWER.value, strict=True) prefill: WorkerConfig decode: WorkerConfig conc: List[int] @@ -629,6 +632,7 @@ class SingleNodeSeqLenConfig(BaseModel): isl: int osl: int + require_power: bool = Field(default=False, alias=Fields.REQUIRE_POWER.value, strict=True) search_space: List[SingleNodeSearchSpaceEntry] = Field( alias=Fields.SEARCH_SPACE.value) @@ -639,6 +643,7 @@ class MultiNodeSeqLenConfig(BaseModel): isl: int osl: int + require_power: bool = Field(default=False, alias=Fields.REQUIRE_POWER.value, strict=True) search_space: List[MultiNodeSearchSpaceEntry] = Field( alias=Fields.SEARCH_SPACE.value) diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 67699d254..eeae0b83d 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -3241,3 +3241,32 @@ def test_empty_matrix_has_no_jobs(self, split_e2e_configs): output = split_e2e_configs([]) assert output and all(rows == [] for rows in output.values()) + + +@pytest.mark.parametrize("multinode", [False, True]) +def test_require_power_is_scoped_to_one_fixed_sequence(multinode, sample_single_node_config, + sample_multinode_config, sample_runner_config): + from infx.matrix.generate import expand_full_sweep, select_matrix_evals + from infx.matrix.validation import MultiNodeSeqLenConfig, SingleNodeSeqLenConfig + + config = sample_multinode_config if multinode else sample_single_node_config + entry = next(iter(config.values())) + sequences = entry["scenarios"]["fixed-seq-len"] + if multinode: + sequences.append(copy.deepcopy(sequences[0])) + sequences[-1]["isl"] = 8192 + before = expand_full_sweep(config, sample_runner_config) + assert all("require-power" not in row for row in before) + sequences[-1]["require-power"] = True + schema = MultiNodeSeqLenConfig if multinode else SingleNodeSeqLenConfig + schema.model_validate(sequences[-1]) + after = expand_full_sweep(config, sample_runner_config) + assert len(before) == len(after) + for original, row in zip(before, after): + assert row == ({**original, "require-power": True} if original["isl"] == 8192 else original) + evals = select_matrix_evals(copy.deepcopy(after), mode="subset") + assert evals + assert all("require-power" not in row for row in evals) + sequences[0]["require-power"] = True + with pytest.raises(ValueError, match="only fixed-sequence 8192/1024"): + expand_full_sweep(config, sample_runner_config) From b44efcd09eb7fb63061adb1e883ae4802b28241f Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:24:13 -0700 Subject: [PATCH 03/21] feat: separate native multinode power collection contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 拆分原生多节点功耗采集契约、UTC 上下文和生命周期屏障，不启用新路径。 --- .../workflows/benchmark-multinode-tmpl.yml | 1 + .github/workflows/benchmark-tmpl.yml | 4 + benchmarks/benchmark_lib.sh | 7 +- benchmarks/native_power_collect.sh | 73 +++++ benchmarks/native_power_lifecycle.sh | 77 +++++ docs/results-and-ingestion.md | 6 + docs/results-and-ingestion_zh.md | 6 + infx/results/fixed_sequence.py | 15 + infx/results/power/native_multinode.py | 286 ++++++++++++++++++ infx/results/power/single_node.py | 59 +++- perf-changelog.yaml | 11 + runners/test_native_collector_barriers.py | 46 +++ runners/test_native_collector_receipts.py | 16 + utils/test_aggregate_power.py | 255 +++++++++++++++- utils/test_native_multinode_power.py | 201 ++++++++++++ 15 files changed, 1045 insertions(+), 18 deletions(-) create mode 100644 benchmarks/native_power_collect.sh create mode 100644 benchmarks/native_power_lifecycle.sh create mode 100644 infx/results/power/native_multinode.py create mode 100644 runners/test_native_collector_barriers.py create mode 100644 runners/test_native_collector_receipts.py create mode 100644 utils/test_native_multinode_power.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 80b0842f8..e9f3a3c90 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -499,6 +499,7 @@ jobs: agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json LOGS/power/** + LOGS/native_power/** result_processing_${{ env.RESULT_FILENAME }}.json LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 4d1b817ea..1da933f63 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -445,11 +445,13 @@ jobs: name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }} path: | gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json if-no-files-found: ignore @@ -462,12 +464,14 @@ jobs: ${{ env.RESULT_FILENAME }}.json agg_${{ env.RESULT_FILENAME }}.json gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json results/agentic_power_window.json results/agentic_power_timezone_offset.txt diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 305a118ef..9d99a4cc9 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -177,6 +177,7 @@ start_gpu_monitor() { GPU_METRICS_CSV="$output" GPU_MONITOR_INTERVAL="$interval" export GPU_METRICS_CSV + printf '{"timestamp_timezone":"UTC"}\n' > "${output%.csv}_context.json" if command -v nvidia-smi &>/dev/null; then GPU_MONITOR_VENDOR="nvidia" @@ -185,7 +186,7 @@ start_gpu_monitor() { rm -f "${output%.csv}_identity.csv" echo "[GPU Monitor] Warning: NVIDIA identity sidecar failed" >&2 fi - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv -l "$interval" > "$output" 2>/dev/null & GPU_MONITOR_PID=$! echo "[GPU Monitor] Started NVIDIA (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)" @@ -196,7 +197,7 @@ start_gpu_monitor() { # Python; measured on MI355X: trailing ticks were lost at kill without it). # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated # headers, and flush every row so killing the pipe cannot discard buffered samples. - PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ + TZ=UTC PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ | awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" & GPU_MONITOR_PID=$! # Hardware energy-accumulator + identity snapshots; the end-side twin in @@ -232,7 +233,7 @@ stop_gpu_monitor() { case "$GPU_MONITOR_VENDOR" in nvidia) if _repair_truncated_gpu_metrics_tail; then - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv,noheader >> "$GPU_METRICS_CSV" 2>/dev/null || echo "[GPU Monitor] Warning: final NVIDIA sample failed" >&2 fi diff --git a/benchmarks/native_power_collect.sh b/benchmarks/native_power_collect.sh new file mode 100644 index 000000000..69df07951 --- /dev/null +++ b/benchmarks/native_power_collect.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# One native SMI collector per serving node. Raw files stay on node-local scratch; +# the launcher stages them as its host user after containers stop. +set -uo pipefail +power_dir=$1 +control_dir=$2 +vendor=$3 +rank=$4 +role=$5 +gpu_indices=$6 +num_nodes=$7 +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" +source "$repo_root/benchmarks/benchmark_lib.sh" +mkdir -p "$power_dir" +collector_rc=0 +finished=0 + +write_control() { + local path="$control_dir/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + # The control directory is created by the host user before Docker starts. + # Containers must not strand root-owned files in the shared runner tree. + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +finish() { + local incoming_rc=$? + [[ "$finished" == 0 ]] || return + if [[ "$incoming_rc" != 0 ]]; then collector_rc=$incoming_rc; fi + if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi + stop_gpu_monitor + if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json + else + nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \ + > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1 + fi + python3 -m infx.results.power.native_multinode end --directory "$power_dir" \ + --collector-exit-code "$collector_rc" || collector_rc=1 + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown -R "$POWERX_HOST_UID:$POWERX_HOST_GID" "$power_dir" || collector_rc=1 + fi + write_control "done-$rank" "$collector_rc" + finished=1 +} +trap finish EXIT +trap 'collector_rc=130; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 130' INT +trap 'collector_rc=143; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 143' TERM HUP + +case "${POWERX_CLOCK_SYNCHRONIZED:-false}" in + yes|true) clock_synchronized=true ;; + *) clock_synchronized=false ;; +esac + +python3 -m infx.results.power.native_multinode begin --directory "$power_dir" \ + --vendor "$vendor" --rank "$rank" --role "$role" --gpu-indices "$gpu_indices" \ + --num-nodes "$num_nodes" --clock-synchronized "$clock_synchronized" || exit 1 +start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 +[[ "$GPU_MONITOR_VENDOR" == "$vendor" ]] || exit 1 +if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json +fi +write_control "ready-$rank" ready +while [[ ! -f "$control_dir/stop" ]]; do + _background_process_is_running "$GPU_MONITOR_PID" || exit 1 + sleep 1 & + wait $! || true +done diff --git a/benchmarks/native_power_lifecycle.sh b/benchmarks/native_power_lifecycle.sh new file mode 100644 index 000000000..b24d1dada --- /dev/null +++ b/benchmarks/native_power_lifecycle.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Shared barriers for native collectors. Launchers own host scratch and mounts. + +powerx_start_collector() { + local power_dir="$1" control_dir="$2" vendor="$3" rank="$4" role="$5" gpus="$6" nodes="$7" + local indices + indices=$(seq -s, 0 "$((gpus - 1))") || return 1 + POWERX_CONTROL_DIR="$control_dir" + POWERX_NUM_NODES="$nodes" + bash "$(dirname "${BASH_SOURCE[0]}")/native_power_collect.sh" \ + "$power_dir" "$control_dir" "$vendor" "$rank" "$role" "$indices" "$nodes" & + POWERX_COLLECTOR_PID=$! +} + +powerx_write_control() { + local path="$POWERX_CONTROL_DIR/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +powerx_wait_collectors() { + local phase="$1" deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rank pending failed + while :; do + pending=0 + failed=0 + for ((rank=0; rank&2 + return 1 + fi + if [[ ! -f "$POWERX_CONTROL_DIR/$phase-$rank" ]]; then + pending=1 + elif [[ "$phase" == done && "$(cat "$POWERX_CONTROL_DIR/done-$rank")" != 0 ]]; then + failed=1 + fi + done + if [[ "$pending" == 0 ]]; then + [[ "$failed" == 0 ]] || echo "One or more PowerX collectors failed" >&2 + return "$failed" + fi + if (( SECONDS >= deadline )); then + echo "Timed out waiting for PowerX $phase receipts" >&2 + return 1 + fi + sleep 1 + done +} + +powerx_stop_collectors() { + local rc=0 + powerx_write_control stop stop || rc=$? + powerx_wait_collectors done || rc=$? + powerx_reap_collector || rc=$? + return "$rc" +} + +powerx_reap_collector() { + [[ -n "${POWERX_COLLECTOR_PID:-}" ]] || return 0 + local deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rc=0 + while kill -0 "$POWERX_COLLECTOR_PID" 2>/dev/null; do + if (( SECONDS >= deadline )); then + kill -TERM "$POWERX_COLLECTOR_PID" 2>/dev/null || true + sleep 2 + kill -KILL "$POWERX_COLLECTOR_PID" 2>/dev/null || true + rc=1 + break + fi + sleep 1 + done + wait "$POWERX_COLLECTOR_PID" || rc=$? + POWERX_COLLECTOR_PID="" + return "$rc" +} diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index a1e357b30..6da11624d 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -106,6 +106,12 @@ For multinode fixed-sequence jobs, `utils/process_result.py --all` processes eve Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. +### Native multinode telemetry + +`native_power_collect.sh` and `native_power_lifecycle.sh` provide per-node collection and bounded ready/stop receipts. Launchers opt into the native package under `LOGS/native_power`; this prerequisite enables no new recipe. The adapter validates serving GPU identity, synchronized clocks, collector completion, and complete formal-window coverage. It preserves per-node failures, sample counts, and collector revision in the audit. + +SMI collection records UTC context beside each CSV for portable replay. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index 6a054d5f0..fd3fa2a2d 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -106,6 +106,12 @@ InferenceX-app 将路由字段作为列或配置维度，并把数值测量存 启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 +### 原生多节点遥测 + +`native_power_collect.sh` 和 `native_power_lifecycle.sh` 提供每节点采集及有时限的就绪/停止状态文件。启动器可使用 `LOGS/native_power` 下的原生产物；此前置改动不会启用新 recipe。适配器验证服务 GPU 身份、时钟同步、采集完成及正式窗口完整覆盖，并在审计中保留节点故障、样本数和采集器版本。 + +SMI 采集在 CSV 旁记录 UTC 上下文以支持跨环境回放。正式窗口外的无效样本不能构成覆盖；`boundary_degenerate_rows` 保留其逐 GPU 计数。 + ## 评测工件 ### 单配置身份和收集 diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index f6fa76102..5ae8a65ee 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -248,6 +248,21 @@ def aggregate_power_result( expected_num_gpus = int(env['TP']) * int(env.get('PP_SIZE', '1')) * int(env.get('PCP_SIZE', '1')) try: if is_multinode: + native_dir = Path(env.get('POWERX_NATIVE_DIR', 'LOGS/native_power')) + if env.get('POWERX_NATIVE_DIR') or native_dir.is_dir(): + if source.is_dir() and source != native_dir: + raise ValueError('Both native and SRT power packages are present') + source = native_dir + from .power.native_multinode import run + + return run( + native_dir, bench_path, agg_path, + expected_prefill_gpus=prefill_gpus, + expected_decode_gpus=decode_gpus, + expected_aggregate_gpus=aggregate_gpus, + validation_result=validation_path, + require_power=require_power, + ) from .power.multinode import run return run( diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py new file mode 100644 index 000000000..1ffae5bd9 --- /dev/null +++ b/infx/results/power/native_multinode.py @@ -0,0 +1,286 @@ +"""Validate native SMI traces from each node of one fixed-sequence deployment. + +This format is owned by InferenceX. It does not claim the srt-slurm/DCGM wire +contract. The launcher records real serving-device membership and synchronized +host clocks; the normal result processor binds every trace to the client window. +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import os +import socket +import time +from datetime import timezone +from pathlib import Path + +from . import ALL_POWER_METRIC_KEYS +from .common import ( + _load_benchmark_data, _write_json_atomic, benchmark_window_payload, + patch_power_metrics, +) +from .single_node import _derived_metrics, _detect_columns, _parse_timestamp, integrate_power + + +def _identity(path: Path, vendor: str) -> dict[str, str]: + """Map the vendor enumeration index to a physical UUID; never use PCI alone.""" + if vendor == "nvidia": + with path.open(newline="") as stream: + rows = [] + for row in csv.DictReader(stream, skipinitialspace=True): + if any(not isinstance(k, str) or not isinstance(v, str) for k, v in row.items()): + raise ValueError("invalid_device_identity") + rows.append({k.strip().lower(): v.strip() for k, v in row.items()}) + elif vendor == "amd": + payload = json.loads(path.read_text()) + rows = [] + + def visit(value: object) -> None: + if isinstance(value, list): + for item in value: + visit(item) + elif isinstance(value, dict): + row = {str(k).lower(): v for k, v in value.items()} + if "gpu" in row and "uuid" in row: + rows.append(row) + else: + for item in value.values(): + visit(item) + + visit(payload) + else: + raise ValueError("unsupported_native_vendor") + identities = {} + for row in rows: + index = str(row.get("index", row.get("gpu", ""))).strip() + uuid = str(row.get("uuid", "")).strip() + if not index.isdigit() or not uuid or uuid.lower() in {"n/a", "none", "null"}: + raise ValueError("invalid_device_identity") + if index in identities or uuid in identities.values(): + raise ValueError("duplicate_device_identity") + identities[index] = uuid + if not identities: + raise ValueError("device_identity_missing") + return identities + + +def record_begin(directory: Path, *, vendor: str, node: str, rank: int, + role: str, gpu_indices: list[int], num_nodes: int, job_id: str, + clock_synchronized: bool, revision: str) -> None: + if (role not in {"prefill", "decode", "aggregate"} or rank < 0 or + num_nodes <= rank or not gpu_indices or min(gpu_indices) < 0 or + len(set(gpu_indices)) != len(gpu_indices)): + raise ValueError("invalid_native_topology") + directory.mkdir(parents=True, exist_ok=True) + _write_json_atomic(directory / "manifest.json", { + "schema_version": 1, "collector": "inferencex-native-smi", + "vendor": vendor, "node": node, "rank": rank, "role": role, + "selected_gpu_indices": gpu_indices, "expected_num_nodes": num_nodes, + "job_id": job_id, "collector_revision": revision, + "clock_source": "utc_ntp", "clock_synchronized": clock_synchronized, + "clock_observation": "timedatectl NTPSynchronized on serving host; no measured clock offset", + "collection_start_unix": time.time(), "lifecycle": "collecting", + }) + + +def record_end(directory: Path, *, collector_exit_code: int) -> None: + manifest = json.loads((directory / "manifest.json").read_text()) + manifest.update(collection_end_unix=time.time(), + collector_exit_code=collector_exit_code, + lifecycle="complete" if collector_exit_code == 0 else "failed") + _write_json_atomic(directory / "manifest.json", manifest) + + +def run(power_dir: Path, bench_result: Path, agg_result: Path, *, + expected_prefill_gpus: int, expected_decode_gpus: int, expected_aggregate_gpus: int = 0, + validation_result: Path | None = None, require_power: bool = False) -> int: + validation_result = validation_result or bench_result.with_name( + f"power_validation_{bench_result.stem}.json") + benchmark, reasons = _load_benchmark_data(bench_result) + expected_gpus = expected_prefill_gpus + expected_decode_gpus + expected_aggregate_gpus + if expected_aggregate_gpus and (expected_prefill_gpus or expected_decode_gpus): + reasons.append("native_mixed_aggregate_role_topology") + roles: dict[str, list[str]] = {"prefill": [], "decode": [], "aggregate": []} + receipts = [] + node_errors = [] + samples = [] + ranks = [] + nodes = [] + jobs = set() + revisions = set() + expected_nodes = set() + paths = sorted(power_dir.glob("node-*/manifest.json")) + if not paths: + reasons.append("native_manifests_missing") + if expected_gpus <= 0 or min(expected_prefill_gpus, expected_decode_gpus, expected_aggregate_gpus) < 0: + reasons.append("invalid_expected_gpu_count") + for path in paths: + try: + manifest = json.loads(path.read_text()) + rank = manifest["rank"] + role = manifest["role"] + if (manifest.get("schema_version") != 1 or + manifest.get("collector") != "inferencex-native-smi" or + type(rank) is not int or rank < 0 or role not in roles or + path.parent.name != f"node-{rank}"): + raise ValueError("invalid_native_manifest") + if (not isinstance(manifest.get("node"), str) or not manifest["node"] or + not isinstance(manifest.get("job_id"), str) or + not isinstance(manifest.get("collector_revision"), str) or + type(manifest.get("expected_num_nodes")) is not int or + manifest["expected_num_nodes"] <= 0): + raise ValueError("invalid_native_run_identity") + ranks.append(rank) + nodes.append(manifest["node"]) + expected_nodes.add(manifest["expected_num_nodes"]) + jobs.add(manifest["job_id"]) + revisions.add(manifest["collector_revision"]) + if (manifest.get("lifecycle") != "complete" or + manifest.get("collector_exit_code") != 0): + reasons.append("native_collector_incomplete") + if (manifest.get("clock_source") != "utc_ntp" or + manifest.get("clock_synchronized") is not True): + reasons.append("native_clock_not_synchronized") + start, end = manifest["collection_start_unix"], manifest["collection_end_unix"] + if (not all(type(x) in (int, float) and math.isfinite(x) for x in (start, end)) + or end <= start or (benchmark is not None and + (start > benchmark.start_unix or end < benchmark.end_unix))): + reasons.append("native_collection_window_mismatch") + selected = manifest["selected_gpu_indices"] + if (not isinstance(selected, list) or not selected or + any(type(i) is not int or i < 0 for i in selected) or + len(set(selected)) != len(selected)): + raise ValueError("invalid_native_gpu_selection") + suffix = "csv" if manifest["vendor"] == "nvidia" else "json" + stem = "gpu_metrics_identity" if suffix == "csv" else "gpu_metrics_devices" + first_path = path.parent / f"{stem}.{suffix}" + last_path = path.parent / f"{stem}_end.{suffix}" + first = _identity(first_path, manifest["vendor"]) + last = _identity(last_path, manifest["vendor"]) + selected_ids = {str(i): first[str(i)] for i in selected} + if any(last.get(i) != uuid for i, uuid in selected_ids.items()): + reasons.append("native_device_identity_changed") + previous = {uuid for members in roles.values() for uuid in members} + if previous.intersection(selected_ids.values()): + reasons.append("native_duplicate_physical_gpu") + roles[role].extend(selected_ids.values()) + csv_path = path.parent / "gpu_metrics.csv" + with csv_path.open(newline="") as stream: + reader = csv.DictReader(stream, skipinitialspace=True) + reader.fieldnames = [c.strip() for c in (reader.fieldnames or [])] + t_col, p_col, g_col = _detect_columns(reader.fieldnames) + if not all((t_col, p_col, g_col)): + raise ValueError("native_telemetry_columns_missing") + for row in reader: + gpu = (row.get(g_col) or "").strip() + if gpu not in first: + reasons.append("native_unknown_device_index") + continue + if gpu not in selected_ids: + continue + timestamp = _parse_timestamp((row.get(t_col) or ""), naive_timezone=timezone.utc) + # Invalid rows are retained for the common validator to reject. + samples.append((timestamp, selected_ids[gpu], row.get(p_col) or "")) + receipts.append({**manifest, "physical_gpu_ids": selected_ids, + "manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "telemetry_sha256": hashlib.sha256(csv_path.read_bytes()).hexdigest(), + "identity_sha256": hashlib.sha256(first_path.read_bytes()).hexdigest(), + "identity_end_sha256": hashlib.sha256(last_path.read_bytes()).hexdigest()}) + except (OSError, ValueError, KeyError, TypeError, csv.Error) as exc: + reasons.append("native_node_invalid") + node_errors.append({"node": path.parent.name, "type": type(exc).__name__, "detail": str(exc)}) + if (len(expected_nodes) != 1 or not expected_nodes or + type(next(iter(expected_nodes), None)) is not int or + set(ranks) != set(range(next(iter(expected_nodes), 0))) or + len(set(nodes)) != len(nodes) or len(ranks) != len(set(ranks))): + reasons.append("native_node_topology_mismatch") + if len(jobs) != 1 or "" in jobs or len(revisions) != 1 or "" in revisions: + reasons.append("native_run_identity_mismatch") + if expected_aggregate_gpus: + if roles["prefill"] or roles["decode"] or len(roles["aggregate"]) != expected_aggregate_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif expected_decode_gpus: + if roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or len(roles["decode"]) != expected_decode_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif roles["decode"] or len(roles["aggregate"] + roles["prefill"]) != expected_prefill_gpus: + reasons.append("native_role_gpu_count_mismatch") + combined_path = validation_result.with_name(f"{validation_result.stem}_native.csv") + combined_path.parent.mkdir(parents=True, exist_ok=True) + with combined_path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["timestamp", "gpu", "power"]) + writer.writerows(samples) + integration = None + if benchmark is not None: + integration = integrate_power(combined_path, start_unix=benchmark.start_unix, + end_unix=benchmark.end_unix, expected_num_gpus=expected_gpus) + reasons.extend(integration.invalid_reasons) + metrics = {} + if not reasons and integration is not None and benchmark is not None: + metrics = _derived_metrics(integration, benchmark) + if expected_decode_gpus: + prefill = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["prefill"]) + decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) + metrics.update(prefill_gpu_energy_j=prefill, decode_gpu_energy_j=decode, + prefill_avg_power_w=prefill / benchmark.integration_duration_s / expected_prefill_gpus, + decode_avg_power_w=decode / benchmark.integration_duration_s / expected_decode_gpus, + prefill_joules_per_input_token=prefill / benchmark.total_input_tokens, + decode_joules_per_output_token=decode / benchmark.total_output_tokens) + valid = not reasons + try: + patch_power_metrics(agg_result, metric_keys=ALL_POWER_METRIC_KEYS, power_valid=valid, metrics=metrics) + except (OSError, ValueError): + reasons.append("aggregate_result_unwritable") + valid, metrics = False, {} + audit = {"schema_version": 1, "telemetry_kind": "native_multinode_smi", + "power_valid": valid, "reasons": list(dict.fromkeys(reasons)), + "benchmark_result": str(bench_result), + "benchmark_result_sha256": hashlib.sha256(bench_result.read_bytes()).hexdigest() if bench_result.is_file() else None, + "benchmark_window": benchmark_window_payload(benchmark), + "expected_gpu_count": expected_gpus, "nodes": receipts, + "node_errors": node_errors, + "observed_gpu_count": integration.observed_num_gpus if integration else 0, + "per_gpu_role": {uuid: role for role, uuids in roles.items() for uuid in uuids}, + "per_gpu_sample_counts": integration.per_gpu_sample_counts if integration else {}, + "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s if integration else {}, + "producer": {"name": "inferencex-native-smi", "revisions": sorted(revisions), + "producer_git_commit": next(iter(revisions)) if len(revisions) == 1 else None}, + "integration_method": "per_device_trapezoidal_with_linear_boundary_interpolation", + "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear", + "metrics": metrics} + _write_json_atomic(validation_result, audit) + return int(require_power and not valid) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + begin = sub.add_parser("begin") + begin.add_argument("--directory", type=Path, required=True) + begin.add_argument("--vendor", choices=("amd", "nvidia"), required=True) + begin.add_argument("--node", default=os.environ.get("POWERX_NODE_NAME", socket.gethostname())) + begin.add_argument("--rank", type=int, required=True) + begin.add_argument("--role", choices=("prefill", "decode", "aggregate"), required=True) + begin.add_argument("--gpu-indices", required=True) + begin.add_argument("--num-nodes", type=int, required=True) + begin.add_argument("--job-id", default=os.environ.get("SLURM_JOB_ID", "")) + begin.add_argument("--revision", default=os.environ.get("POWERX_COLLECTOR_REVISION", "")) + begin.add_argument("--clock-synchronized", choices=("true", "false"), default="false") + end = sub.add_parser("end") + end.add_argument("--directory", type=Path, required=True) + end.add_argument("--collector-exit-code", type=int, required=True) + args = vars(parser.parse_args()) + action = args.pop("action") + if action == "begin": + args["gpu_indices"] = [int(i) for i in args["gpu_indices"].split(",")] + args["clock_synchronized"] = args["clock_synchronized"] == "true" + record_begin(**args) + else: + record_end(**args) + + +if __name__ == "__main__": + main() diff --git a/infx/results/power/single_node.py b/infx/results/power/single_node.py index 54669a1a8..4ce1bcaaf 100644 --- a/infx/results/power/single_node.py +++ b/infx/results/power/single_node.py @@ -9,7 +9,11 @@ aggregate and a validation sidecar, but does not fail the benchmark. Power studies can set ``REQUIRE_POWER=1`` to fail after those audit artifacts exist. The aggregate carries numeric ``power_valid`` (1/0) for metric ingestion; the -sidecar is the canonical source for boolean validity and reason codes. +sidecar is the canonical source for boolean validity and reason codes. Rows in +the ingest band but outside the formal window whose power is missing, +non-finite, or <= 0 are teardown noise: they are skipped and counted in the +sidecar's ``boundary_degenerate_rows`` instead of poisoning validity or faking +window bracketing. """ from __future__ import annotations @@ -22,7 +26,7 @@ import os import re import sys -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from statistics import mean @@ -65,6 +69,10 @@ class PowerIntegration: per_gpu_max_sample_gap_s: dict[str, float] per_gpu_energy_j: dict[str, float] device_issues: dict[str, list[str]] + # Rows in the ingest band but outside the formal window whose power was + # missing/N-A/non-finite/<=0, skipped and counted per GPU; "unknown" + # buckets rows without a GPU identity. + boundary_degenerate_rows: dict[str, int] = field(default_factory=dict) avg_power_w: float | None = None p75_power_w: float | None = None p75_total_gpu_power_w: float | None = None @@ -78,7 +86,7 @@ def observed_num_gpus(self) -> int: return len(self.observed_gpu_ids) -def _parse_timestamp(value: str) -> float | None: +def _parse_timestamp(value: str, *, naive_timezone: timezone | None = None) -> float | None: """Best-effort timestamp parse to Unix epoch seconds (local wall clock). Handles the formats observed in practice: @@ -96,7 +104,7 @@ def _parse_timestamp(value: str) -> float | None: # nvidia-smi: "YYYY/MM/DD HH:MM:SS.ffffff" for fmt in ("%Y/%m/%d %H:%M:%S.%f", "%Y/%m/%d %H:%M:%S"): try: - return datetime.strptime(value, fmt).timestamp() + return datetime.strptime(value, fmt).replace(tzinfo=naive_timezone).timestamp() except ValueError: pass # ISO 8601 (amd-smi variants). fromisoformat tolerates 'T' or space separator @@ -108,7 +116,7 @@ def _parse_timestamp(value: str) -> float | None: return None if dt.tzinfo is None: # Treat naive timestamps as local time (matches nvidia-smi convention). - return dt.timestamp() + return dt.replace(tzinfo=naive_timezone).timestamp() return dt.astimezone(timezone.utc).timestamp() @@ -146,6 +154,16 @@ def _detect_columns(header: list[str]) -> tuple[str | None, str | None, str | No return timestamp_col, power_col, gpu_col +def _telemetry_timezone(csv_path: Path) -> timezone | None: + context = csv_path.with_name(f"{csv_path.stem}_context.json") + if not context.is_file(): + return None + payload = json.loads(context.read_text()) + if not isinstance(payload, dict) or payload.get("timestamp_timezone") != "UTC": + raise ValueError("unsupported_telemetry_timezone") + return timezone.utc + + def aggregate_power( csv_path: Path, start_unix: float, @@ -162,6 +180,7 @@ def aggregate_power( return None try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [c.strip() for c in (reader.fieldnames or [])] @@ -189,7 +208,7 @@ def aggregate_power( for row in reader: ts_raw = (row.get(timestamp_col) or "").strip() pw_raw = (row.get(power_col) or "").strip() - ts = _parse_timestamp(ts_raw) + ts = _parse_timestamp(ts_raw, naive_timezone=timestamp_timezone) pw = _parse_power(pw_raw) if ts is None or pw is None: continue @@ -204,7 +223,7 @@ def aggregate_power( if gpu_id: per_sample_gpus.setdefault(bucket, set()).add(gpu_id) gpu_keys.add(gpu_id) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): return None if not per_sample_total: @@ -237,6 +256,7 @@ def _empty_integration( *, expected_num_gpus: int | None, reasons: list[str], + boundary_degenerate_rows: dict[str, int] | None = None, ) -> PowerIntegration: """Build an invalid integration result when no device data is available.""" return PowerIntegration( @@ -248,6 +268,7 @@ def _empty_integration( per_gpu_max_sample_gap_s={}, per_gpu_energy_j={}, device_issues={}, + boundary_degenerate_rows=boundary_degenerate_rows or {}, ) @@ -308,8 +329,10 @@ def integrate_power( # expose timestamps at lower resolution than their sampling cadence, so # duplicate-timestamp readings are averaged rather than treated as corrupt. raw_samples: dict[str, dict[float, list[float]]] = {} + boundary_degenerate: dict[str, int] = {} saw_missing_gpu_identity = False try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -332,7 +355,7 @@ def integrate_power( ) for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) if timestamp is None or not math.isfinite(timestamp): _append_reason(reasons, "invalid_timestamp_sample") continue @@ -348,6 +371,15 @@ def integrate_power( power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() + if (power is None or not math.isfinite(power) or power <= 0.0) and ( + timestamp < start_unix or timestamp > end_unix + ): + # SMI teardown rows can carry N/A or 0 W cells: outside the + # formal window they are counted, never used to satisfy + # bracketing or to poison in-window validity. + key = gpu_id or "unknown" + boundary_degenerate[key] = boundary_degenerate.get(key, 0) + 1 + continue if power is None: _append_reason(reasons, "invalid_power_sample") continue @@ -359,11 +391,12 @@ def integrate_power( continue values = raw_samples.setdefault(gpu_id, {}).setdefault(timestamp, []) values.append(power) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): _append_reason(reasons, "telemetry_file_unreadable") return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) if saw_missing_gpu_identity: @@ -373,6 +406,7 @@ def integrate_power( return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) observed_gpu_ids = tuple(sorted(raw_samples, key=_gpu_sort_key)) @@ -450,6 +484,7 @@ def integrate_power( per_gpu_max_sample_gap_s=per_gpu_max_sample_gap_s, per_gpu_energy_j=per_gpu_energy_j, device_issues=device_issues, + boundary_degenerate_rows=boundary_degenerate, avg_power_w=avg_power_w, p75_power_w=p75_total / len(observed_gpu_ids) if p75_total is not None else None, p75_total_gpu_power_w=p75_total, @@ -485,6 +520,7 @@ def _read_energy_snapshot(path: Path) -> dict[str, float] | None: def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float]]]: """Group every parseable (timestamp, watt) sample per GPU, sorted in time.""" samples: dict[str, list[tuple[float, float]]] = {} + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -493,7 +529,7 @@ def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float] if not timestamp_col or not power_col or not gpu_col: return {} for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() if timestamp is None or power is None or not gpu_id: @@ -679,6 +715,7 @@ def _validation_payload( "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s, "per_gpu_energy_j": integration.per_gpu_energy_j, "device_issues": integration.device_issues, + "boundary_degenerate_rows": integration.boundary_degenerate_rows, "accumulator_check": accumulator_check, "metrics": audit_metrics(metrics), } @@ -754,7 +791,7 @@ def run( try: accumulator_check = cross_check_accumulator(csv_path) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): accumulator_check = {"available": False, "reason": "cross_check_error"} try: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f3b85de24..10db254b5 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7410,3 +7410,14 @@ - "Preserve request failure outcomes, retain all available fixed-sequence diagnostics, and reject incomplete concurrency batches before publishing benchmark artifacts." - "保留请求失败状态及全部可用的固定序列诊断信息；发布基准产物前拒绝缺失并发点的批次。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Retain UTC SMI telemetry context and validate native multinode collector artifacts without enabling new recipes." + - "保留 UTC SMI 遥测上下文并验证原生多节点采集产物，不启用新 recipe。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 diff --git a/runners/test_native_collector_barriers.py b/runners/test_native_collector_barriers.py new file mode 100644 index 000000000..986016a78 --- /dev/null +++ b/runners/test_native_collector_barriers.py @@ -0,0 +1,46 @@ +"""CPU-only lifecycle checks with controlled external collector/Slurm processes.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +LIFECYCLE = ROOT / 'benchmarks/native_power_lifecycle.sh' +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' + + +@pytest.mark.parametrize('failed_rank', [None, 1]) +def test_stop_waits_for_every_collector_and_retains_failure(tmp_path, failed_rank): + command = ''' +source "$1" +export POWERX_CONTROL_DIR="$2" POWERX_NUM_NODES=2 POWERX_BARRIER_TIMEOUT_S=5 +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.1; echo 0 > "$2/done-0") & +POWERX_COLLECTOR_PID=$! +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.2; echo "$3" > "$2/done-1") & +remote_pid=$! +powerx_stop_collectors +rc=$? +wait "$remote_pid" +exit "$rc" +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(LIFECYCLE), + str(tmp_path), '1' if failed_rank else '0'], + capture_output=True, text=True, timeout=10) + assert result.returncode == (1 if failed_rank else 0), result.stderr + assert (tmp_path / 'done-0').read_text().strip() == '0' + assert (tmp_path / 'done-1').read_text().strip() == ('1' if failed_rank else '0') + + +def test_ready_barrier_rejects_collector_that_already_stopped(tmp_path): + (tmp_path / 'ready-0').write_text('ready') + (tmp_path / 'done-0').write_text('1') + result = subprocess.run(['bash', '-c', 'source "$1"; POWERX_CONTROL_DIR="$2"; ' + 'POWERX_NUM_NODES=1; powerx_wait_collectors ready', + 'bash', str(LIFECYCLE), str(tmp_path)], capture_output=True, text=True) + assert result.returncode == 1 + assert 'stopped before benchmark readiness' in result.stderr + + diff --git a/runners/test_native_collector_receipts.py b/runners/test_native_collector_receipts.py new file mode 100644 index 000000000..b981f57ce --- /dev/null +++ b/runners/test_native_collector_receipts.py @@ -0,0 +1,16 @@ +import subprocess +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] + +def test_native_control_receipt_is_owned_before_publication(tmp_path): + source = (ROOT / 'benchmarks/native_power_collect.sh').read_text() + function = source[source.index('write_control() {'):source.index('\nfinish() {')] + result = subprocess.run(['bash', '-c', function + ''' +control_dir=$1 +POWERX_HOST_UID=1000 POWERX_HOST_GID=1000 +chown() { [[ ! -e "$control_dir/done-0" ]]; } +write_control done-0 7 +''', 'bash', str(tmp_path)], capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert (tmp_path / 'done-0').read_text() == '7\n' + assert not list(tmp_path.glob('*.tmp')) diff --git a/utils/test_aggregate_power.py b/utils/test_aggregate_power.py index aa3bf23f9..5b8b76121 100644 --- a/utils/test_aggregate_power.py +++ b/utils/test_aggregate_power.py @@ -574,6 +574,201 @@ def test_run_rejects_malformed_telemetry_inside_window( assert audit["reasons"] == [expected_reason] + +# AMDSMI 26.2.0 `metric -p -c -t -u -w 1 --csv` header (order-faithful subset, +# measured on MI355X; see test_detect_columns_amd_watch_mode_real_header). +_MI355X_WATCH_HEADER = ( + "timestamp,gpu,gfx_activity,umc_activity,mm_activity,vcn_activity," + "jpeg_activity,gfx_busy_inst_xcp_0,jpeg_busy_xcp_0,vcn_busy_xcp_0," + "socket_power,gfx_voltage,soc_voltage,mem_voltage,throttle_status," + "power_management,gfx_0_clk,mem_0_clk,edge,hotspot,mem" +) + + +def _mi355x_watch_row(timestamp: int, gpu: int, socket_power: str) -> str: + """One data row in the shape captured from run 32433563482 (conc1): + integer-second epoch, quoted list cells with embedded commas, N/A cells, + and a trailing carriage return.""" + return ( + f"{timestamp},{gpu},0,0,N/A,\"['N/A', 'N/A', 'N/A', 'N/A']\"," + "\"['N/A', 'N/A']\",\"[0, 0, 0, 0, 0, 0, 0, 0]\",\"[0, 0]\",\"[0, 0]\"," + f"{socket_power},N/A,N/A,N/A,N/A,ENABLED,1404,2000,N/A,40,25\r" + ) + + +def test_integrate_power_skips_na_power_rows_outside_window(tmp_path: Path): + """N/A-power teardown rows past the window end are counted, not poisonous.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-2, 13): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},N/A,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + assert result.total_gpu_energy_j == pytest.approx(10_000.0) + + +def test_integrate_power_does_not_bracket_with_zero_power_tail(tmp_path: Path): + """A 0 W teardown row past the window end must not fake end bracketing. + + Legacy behavior accepted the 0 W row as a valid boundary sample, corrupting + the end interpolation with a bogus value; this intentionally flips that + case to an explicit benchmark_window_not_bracketed failure.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + # Good rows stop at end-4; the only post-end sample per GPU has power 0. + for offset in range(-1, 7): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},0,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "benchmark_window_not_bracketed" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + + +def test_integrate_power_preserves_boundary_counts_without_usable_samples(tmp_path: Path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text( + "timestamp,gpu,socket_power\n" + "1699999999,0,N/A\n" + "1700000011,0,0\n" + "1700000011,1,N/A\n", + encoding="utf-8", + ) + + result = integrate_power( + csv, + start_unix=1_700_000_000, + end_unix=1_700_000_010, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "no_usable_power_samples" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 2, "1": 1} + + +def test_integrate_power_keeps_zero_power_semantics_inside_window(tmp_path: Path): + """Frozen legacy behavior: an in-window 0 W sample still integrates.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-1, 12): + watts = "0.0" if offset == 5 else "500.0" + lines.append(f"{base + offset},0,{watts},65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=1, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {} + # Trapezoids dip to 0 at t=5: 8 x 500 + 2 x 250 = 4500 J. + assert result.total_gpu_energy_j == pytest.approx(4_500.0) + + +def test_run_writes_boundary_degenerate_rows_to_sidecar(tmp_path: Path): + base = 1_700_000_000.0 + csv = tmp_path / "gpu_metrics.csv" + _write_constant_window_samples( + csv, + start=base, + end=base + 10, + watts_per_gpu=500.0, + num_gpus=2, + ) + bench = tmp_path / "bench.json" + agg = tmp_path / "agg.json" + validation = tmp_path / "power_validation.json" + _write_bench_result( + bench, + start=base, + end=base + 10, + duration=10.0, + total_output=2_000, + total_input=10_000, + ) + agg.write_text(json.dumps({"hw": "mi355x"}), encoding="utf-8") + + exit_code = run(csv, bench, agg, expected_num_gpus=2, validation_result=validation) + + assert exit_code == 0 + audit = json.loads(validation.read_text()) + # Present-and-empty for clean streams: readers can rely on the key. + assert audit["boundary_degenerate_rows"] == {} + + +def test_integrate_power_regression_mi355x_integer_ticks_end_gap(tmp_path: Path): + """Run-32433563482 conc1 regression: amd-smi integer-second ticks stop 4 s + before the fractional aiperf window end (last tick 1787277605 vs end + ...609.157497), so bracketing fails and the producer-side telemetry loss is + attributed as benchmark_window_not_bracketed on every GPU. + + The retrieved artifact's trailing rows all carry valid socket_power + (254-264 W) with N/A activity/voltage cells; the N/A- and 0-power teardown + rows appended past the window end are the documented synthetic degenerate + shapes, asserting they are counted rather than used for bracketing.""" + csv = tmp_path / "gpu_metrics.csv" + start = 1_787_277_560.155891 + end = 1_787_277_609.157497 + last_tick = 1_787_277_605 + powers = [259, 255, 263, 264, 256, 254, 259, 259] + lines = [_MI355X_WATCH_HEADER] + for tick in range(1_787_277_555, last_tick + 1): + for gpu in range(8): + lines.append(_mi355x_watch_row(tick, gpu, str(powers[gpu]))) + # amd-smi watch mode emits a blank line between tick groups. + lines.append("") + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_610, gpu, "N/A")) + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_611, gpu, "0")) + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=start, + end_unix=end, + expected_num_gpus=8, + ) + + assert result.power_valid is False + assert result.invalid_reasons == ("benchmark_window_not_bracketed",) + assert result.device_issues == { + str(gpu): ["benchmark_window_not_bracketed"] for gpu in range(8) + } + assert result.boundary_degenerate_rows == {str(gpu): 2 for gpu in range(8)} + + def test_run_patches_agg_with_power_and_joules(tmp_path: Path): base = 1_700_000_000.0 csv = tmp_path / "gpu_metrics.csv" @@ -708,6 +903,7 @@ def test_power_replacement_removes_stale_metrics(tmp_path, patch_validated_power path.write_text(json.dumps({ "hw": "fixture", "avg_power_w": 99, "total_gpu_energy_j": 50, "power_invalid_reasons": ["stale"], + "power_audit": {"source": "previous-run.json"}, })) patch_validated_power(path, power_valid=valid, metrics={ "avg_power_w": 12.34567, "joules_per_output_token": 0.12345678, @@ -1433,12 +1629,12 @@ def test_power_percentiles_uses_synchronized_total_not_device_percentiles(tmp_pa def test_power_percentiles_weights_time_and_clips_the_validated_window(tmp_path): csv_path = tmp_path / "power.csv" - # Dense readings near the high end must not bias a uniform linear ramp. - _write_amd_csv(csv_path, [(0, 0, 0), (1, 0, 100), (1.9, 0, 190), (2, 0, 200)]) + # Dense readings must not bias the uniform 150-250 W ramp inside the window. + _write_amd_csv(csv_path, [(0, 0, 100), (1, 0, 200), (1.9, 0, 290), (2, 0, 300)]) result = integrate_power(csv_path, start_unix=0.5, end_unix=1.5, expected_num_gpus=1) assert result.power_valid - assert result.p75_power_w == pytest.approx(125) - assert result.p90_power_w == pytest.approx(140) + assert result.p75_power_w == pytest.approx(225) + assert result.p90_power_w == pytest.approx(240) def test_power_percentiles_is_withheld_for_invalid_telemetry(tmp_path): @@ -1462,3 +1658,54 @@ def test_power_percentiles_aligns_asynchronous_gpu_samples(tmp_path): assert result.p75_power_w == pytest.approx(300) assert result.p90_total_gpu_power_w == pytest.approx(600) assert result.p90_power_w == pytest.approx(300) + + +@pytest.mark.parametrize('step_name', ['Upload GPU metrics', 'Upload power audit bundle']) +@pytest.mark.parametrize('directory', ['', 'results']) +def test_uploaded_telemetry_replays_in_a_different_timezone(tmp_path, step_name, directory): + import os + import shutil + import yaml + + repo = Path(__file__).resolve().parents[1] + source = tmp_path / 'source' + telemetry = source / directory + telemetry.mkdir(parents=True) + (telemetry / 'gpu_metrics.csv').write_text( + 'timestamp,index,power.draw [W]\n' + '2024/01/01 00:00:00.000,0,100 W\n' + '2024/01/01 00:00:01.000,0,100 W\n' + '2024/01/01 00:00:02.000,0,100 W\n') + (telemetry / 'gpu_metrics_context.json').write_text('{"timestamp_timezone":"UTC"}') + workflow = yaml.safe_load((repo / '.github/workflows/benchmark-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job['steps'] + if step.get('name') == step_name) + archive = tmp_path / 'downloaded' + for pattern in step['with']['path'].splitlines(): + for file in source.glob(pattern): + destination = archive / file.relative_to(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(file, destination) + result = subprocess.run( + [sys.executable, '-c', """ +import sys, time +from pathlib import Path +from infx.results.power.single_node import integrate_power +time.tzset() +result = integrate_power(Path(sys.argv[1]), start_unix=1704067200, end_unix=1704067202, + expected_num_gpus=1) +assert result.power_valid, result.invalid_reasons +assert result.total_gpu_energy_j == 200 +""", str(archive / directory / 'gpu_metrics.csv')], + cwd=tmp_path, env={**os.environ, 'TZ': 'Etc/GMT+8', 'PYTHONPATH': str(repo)}, + capture_output=True, text=True, timeout=10, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize('context', ['{"timestamp_timezone":"PST"}', '{}', '{invalid']) +def test_legacy_average_rejects_invalid_telemetry_context(tmp_path, context): + csv = tmp_path / 'gpu_metrics.csv' + csv.write_text('timestamp,index,power.draw [W]\n1,0,100\n2,0,100\n') + csv.with_name('gpu_metrics_context.json').write_text(context) + assert aggregate_power(csv, 1, 2) is None diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py new file mode 100644 index 000000000..0aeee2511 --- /dev/null +++ b/utils/test_native_multinode_power.py @@ -0,0 +1,201 @@ +"""Native collector acceptance uses independent, hand-computed two-node traces.""" +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from infx.results.power.native_multinode import record_begin, record_end, run +from infx.results.power.single_node import integrate_power + +REPO = Path(__file__).resolve().parents[1] + + +def _package(tmp_path, vendor="amd"): + root = tmp_path / "native_power" + for rank, role, watts in ((0, "prefill", 100), (1, "decode", 300)): + node = root / f"node-{rank}" + record_begin(node, vendor=vendor, node=f"host-{rank}", rank=rank, role=role, + gpu_indices=[0], num_nodes=2, job_id="job-123", revision="revision-abc", + clock_synchronized=True) + record_end(node, collector_exit_code=0) + manifest = json.loads((node / "manifest.json").read_text()) + manifest.update(collection_start_unix=0, collection_end_unix=5) + (node / "manifest.json").write_text(json.dumps(manifest)) + for ending in ("", "_end"): + if vendor == "amd": + (node / f"gpu_metrics_devices{ending}.json").write_text(json.dumps([ + {"gpu": 0, "uuid": f"uuid-{rank}"}, {"gpu": 1, "uuid": f"unused-{rank}"}])) + else: + (node / f"gpu_metrics_identity{ending}.csv").write_text( + f"index, uuid, pci.bus_id\n0, uuid-{rank}, 0000:01:00.0\n1, unused-{rank}, 0000:02:00.0\n") + # The spare physical GPU is visible but does not belong to the server. + (node / "gpu_metrics.csv").write_text("timestamp,gpu,power\n" + "".join( + f"{tick},0,{watts}\n{tick},1,900\n" for tick in range(5))) + bench = tmp_path / "result.json" + bench.write_text(json.dumps({"benchmark_start_time_unix": 1, "benchmark_end_time_unix": 3, + "duration": 2, "completed": 2, "total_input_tokens": 20, + "total_output_tokens": 10})) + agg = tmp_path / "agg.json" + agg.write_text(json.dumps({"avg_power_w": 999, "prefill_gpu_energy_j": 999})) + return root, bench, agg + + +@pytest.mark.parametrize("vendor", ["amd", "nvidia"]) +def test_native_whole_fleet_and_role_energy_use_only_serving_devices(tmp_path, vendor): + root, bench, agg = _package(tmp_path, vendor) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 1 + assert actual["total_gpu_energy_j"] == 800 + assert actual["avg_power_w"] == 200 + assert actual["p90_power_w"] == 200 + assert actual["p75_power_w"] == 200 + assert actual["joules_per_successful_query"] == 400 + assert actual["prefill_gpu_energy_j"] == 200 + assert actual["decode_gpu_energy_j"] == 600 + assert actual["prefill_joules_per_input_token"] == 10 + assert actual["decode_joules_per_output_token"] == 60 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert audit["observed_gpu_count"] == 2 + assert audit["per_gpu_role"] == {"uuid-0": "prefill", "uuid-1": "decode"} + assert len(audit["nodes"][0]["telemetry_sha256"]) == 64 + from infx.results.power.audit import audit_summary + summary = audit_summary(audit, "power_validation_result.json")["power_audit"] + assert summary["sample_count"] == 10 # Five samples on each of two participating GPUs. + assert summary["producer_sha"] == "revision-abc" + + +def test_native_parse_failure_has_a_public_reason_and_retained_detail(tmp_path): + from infx.results.power.audit import audit_summary + root, bench, agg = _package(tmp_path) + (root / "node-1/manifest.json").write_text("not JSON") + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + summary = audit_summary(audit, "power_validation_result.json") + assert "native_node_invalid" in summary["power_invalid_reasons"] + assert audit["node_errors"][0]["node"] == "node-1" + assert audit["node_errors"][0]["detail"] + + +@pytest.mark.parametrize(("field", "value", "reason"), [ + ("clock_synchronized", False, "native_clock_not_synchronized"), + ("lifecycle", "collecting", "native_collector_incomplete"), + ("collection_end_unix", 2, "native_collection_window_mismatch"), + ("job_id", "other-job", "native_run_identity_mismatch"), + ("role", "prefill", "native_role_gpu_count_mismatch"), + ("expected_num_nodes", 3, "native_node_topology_mismatch"), +]) +def test_native_invalid_evidence_clears_stale_metrics_and_writes_audit(tmp_path, field, value, reason): + root, bench, agg = _package(tmp_path) + path = root / "node-1/manifest.json" + manifest = json.loads(path.read_text()); manifest[field] = value + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 0 + assert "avg_power_w" not in actual + assert "prefill_gpu_energy_j" not in actual + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert reason in audit["reasons"] + + +def test_native_device_replacement_cannot_preserve_validity(tmp_path): + root, bench, agg = _package(tmp_path) + (root / "node-1/gpu_metrics_devices_end.json").write_text('[{"gpu":0,"uuid":"replacement"}]') + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + assert "native_device_identity_changed" in json.loads((tmp_path / "power_validation_result.json").read_text())["reasons"] + + +def test_native_aggregate_nodes_do_not_invent_prefill_decode_metrics(tmp_path): + root, bench, agg = _package(tmp_path) + for path in root.glob("*/manifest.json"): + manifest = json.loads(path.read_text()); manifest["role"] = "aggregate" + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=0, expected_decode_gpus=0, + expected_aggregate_gpus=2, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["avg_power_w"] == 200 + assert "prefill_gpu_energy_j" not in actual + + +def test_utc_context_replays_in_a_different_timezone(tmp_path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text("timestamp,index,power.draw [W]\n2026/01/01 00:00:00,0,100\n2026/01/01 00:00:02,0,100\n") + (tmp_path / "gpu_metrics_context.json").write_text('{"timestamp_timezone":"UTC"}') + script = "from pathlib import Path; from infx.results.power.single_node import integrate_power; " + \ + f"r=integrate_power(Path({str(csv)!r}),start_unix=1767225600,end_unix=1767225602,expected_num_gpus=1); assert r.power_valid; assert r.total_gpu_energy_j == 200" + subprocess.run([sys.executable, "-c", script], cwd=REPO, + env={**os.environ, "TZ": "America/Los_Angeles"}, check=True, timeout=10) + + +@pytest.mark.parametrize("clock_value, synchronized", [ + ("yes", True), ("true", True), ("no", False), ("false", False), ("", False), +]) +def test_native_supervisor_reaps_monitor_and_writes_completion(tmp_path, clock_value, synchronized): + binary = tmp_path / "bin"; binary.mkdir() + (binary / "python3").symlink_to(sys.executable) + fake = binary / "nvidia-smi" + fake.write_text(f'''#!{sys.executable} +import datetime, os, sys, time +if any("index,uuid" in arg for arg in sys.argv): + print("index, uuid, pci.bus_id"); print("0, gpu-0, 0000:01:00.0") +else: + with open({str(tmp_path / 'monitor.pid')!r}, "w") as stream: stream.write(str(os.getpid())) + if "-l" in sys.argv: print("timestamp,index,power.draw [W]", flush=True) + while True: + print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y/%m/%d %H:%M:%S.%f") + ",0,100", flush=True) + if "-l" not in sys.argv: break + time.sleep(0.1) +''') + fake.chmod(0o755) + control = tmp_path / "control"; control.mkdir() + node = tmp_path / "node-0" + process = subprocess.Popen(["bash", str(REPO / "benchmarks/native_power_collect.sh"), + str(node), str(control), "nvidia", "0", "aggregate", "0", "1"], + env={**os.environ, "PATH": f"{binary}:{os.environ['PATH']}", + "SLURM_JOB_ID": "test-job", "POWERX_COLLECTOR_REVISION": "revision", + "POWERX_CLOCK_SYNCHRONIZED": clock_value}, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + try: + deadline = time.monotonic() + 10 + while not (control / "ready-0").exists(): + assert process.poll() is None + assert time.monotonic() < deadline + time.sleep(0.02) + (control / "stop").write_text("stop") + stdout, stderr = process.communicate(timeout=10) + assert process.returncode == 0, (stdout, stderr) + assert (control / "done-0").read_text().strip() == "0" + manifest = json.loads((node / "manifest.json").read_text()) + assert manifest["lifecycle"] == "complete" + assert manifest["clock_synchronized"] is synchronized + assert (node / "gpu_metrics_identity_end.csv").exists() + finally: + if process.poll() is None: + process.terminate(); process.communicate(timeout=10) + + +def test_result_processor_discovers_staged_native_package(tmp_path, monkeypatch): + from infx.results.fixed_sequence import aggregate_power_result + + root, bench, agg = _package(tmp_path) + logs = tmp_path / "LOGS" + logs.mkdir() + root.rename(logs / "native_power") + monkeypatch.chdir(tmp_path) + env = {"IS_MULTINODE": "true", "PREFILL_GPUS": "1", "DECODE_GPUS": "1", + "RESULT_FILENAME": "result", "REQUIRE_POWER": "1"} + assert aggregate_power_result(env, bench, agg) == 0 + assert json.loads(agg.read_text())["total_gpu_energy_j"] == 800 + assert (tmp_path / "power_validation_result.json").is_file() + + # Two competing formats must not silently select one package. + (logs / "power").mkdir() + assert aggregate_power_result(env, bench, agg) == 1 + invalid = json.loads(agg.read_text()) + assert invalid["power_valid"] == 0 + assert "total_gpu_energy_j" not in invalid From ec4a58fde5eb541ff307ded48b3b2ea4f220f082 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:24:14 -0700 Subject: [PATCH 04/21] fix: verify Slurm completion and exit llm-d workers normally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验证 Slurm 完成状态并让 llm-d 工作进程正常退出，先保存诊断证据。 --- .../workflows/benchmark-multinode-tmpl.yml | 1 + .github/workflows/benchmark-tmpl.yml | 1 + benchmarks/multi_node/llm-d/job.slurm | 26 ++---- benchmarks/multi_node/llm-d/server.sh | 46 ++++++---- docs/results-and-ingestion.md | 4 + docs/results-and-ingestion_zh.md | 4 + perf-changelog.yaml | 12 +++ runners/launch_b200-nscale-compat.sh | 6 +- runners/launch_b200-nscale-slurm.sh | 7 +- runners/launch_b300-dsxe.sh | 6 +- runners/launch_gb200-nv.sh | 12 ++- runners/launch_gb300-nv.sh | 6 +- runners/launch_h100-dgxc-slurm.sh | 6 +- runners/launch_h200-dgxc-slurm.sh | 7 +- runners/launch_mi355x-amds.sh | 5 ++ runners/slurm_utils.sh | 29 +++++- runners/test_llmd_lifecycle.py | 90 +++++++++++++++++++ runners/test_slurm_utils.py | 57 ++++++++++++ 18 files changed, 278 insertions(+), 47 deletions(-) create mode 100644 runners/test_llmd_lifecycle.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index bae33211b..cfbaefce8 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -517,6 +517,7 @@ jobs: ${{ env.RESULT_FILENAME }}_*.json agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json + slurm_job_*_outcome.txt LOGS/power/** LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index b08ef0775..d788a8023 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -467,6 +467,7 @@ jobs: gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json + slurm_job_*_outcome.txt results/gpu_metrics*.csv results/gpu_metrics_identity.json results/agentic_power_window.json diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm index 536507bee..e3338c31d 100644 --- a/benchmarks/multi_node/llm-d/job.slurm +++ b/benchmarks/multi_node/llm-d/job.slurm @@ -63,25 +63,11 @@ export DOCKER_CONT_NAME : "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}" DOCKER_MOUNT_PATH="/workspace" -cleanup() { - echo "[${SLURM_JOB_ID}] cleanup on $(hostname)" - [[ -n "${WATCHER_PID:-}" ]] && kill "$WATCHER_PID" 2>/dev/null || true -} -trap cleanup INT TERM HUP EXIT - -# Coordinator-done watcher. server.sh on the decode coordinator writes -# this marker after the bench finishes; we then scancel the allocation -# from outside the container (the image has no SLURM client tools). -# Without this, workers `wait` on local vLLM forever and the job runs -# to TIME_LIMIT. +# Workers exit through server.sh after reading the coordinator's final status. +# Normal completion must not cancel an otherwise successful allocation. BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" rm -f "$BENCH_DONE_MARKER" -( - while [[ ! -f "$BENCH_DONE_MARKER" ]]; do sleep 5; done - echo "[${SLURM_JOB_ID}] coordinator finished; scancel'ing job" - scancel "$SLURM_JOB_ID" 2>/dev/null || true -) & -WATCHER_PID=$! +main_rc=0 # Container engine: 'docker' (default) for clusters where the SLURM # user can talk to /var/run/docker.sock (e.g. h200-dgxc-slurm); 'pyxis' @@ -173,7 +159,7 @@ exec docker run --rm \ ${DOCKER_MOUNT_PATH}/benchmarks/multi_node/llm-d/server.sh \ 2>&1 | tee /benchmark_logs/slurm_job-'\"\$SLURM_JOB_ID\"'_rank_'\"\$SLURM_PROCID\"'.log ' -" +" || main_rc=$? srun bash -c "docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f" || true @@ -257,9 +243,11 @@ export MODEL_DIR=/models export BENCHMARK_LOGS_DIR=/benchmark_logs '"$DOCKER_MOUNT_PATH"'/benchmarks/multi_node/llm-d/server.sh \ 2>&1 | tee /benchmark_logs/slurm_job-${SLURM_JOB_ID}_rank_${SLURM_PROCID}.log -' +' || main_rc=$? else echo "Unsupported LLMD_CONTAINER_ENGINE: $LLMD_CONTAINER_ENGINE (expected docker|pyxis)" >&2 exit 1 fi + +exit "$main_rc" diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 89894b210..b38940a85 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -114,6 +114,27 @@ else DP_ADDR="$DECODE_DP_ADDR" fi +# One coordinator publishes the benchmark status; workers leave normally when +# it finishes so Slurm can distinguish success from cancellation. +BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" +BENCH_RC=0 + +finish_llmd_node() { + local rc=$? pid + trap - EXIT + if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then + printf '%s\n' "$rc" > "$BENCH_DONE_MARKER.tmp" && + mv -f "$BENCH_DONE_MARKER.tmp" "$BENCH_DONE_MARKER" || rc=1 + fi + for pid in "${ENVOY_PID:-}" "${EPP_PID:-}" "${SIDECAR_PID:-}" "${VLLM_PID:-}"; do + [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true + done + exit "$rc" +} +trap finish_llmd_node EXIT +trap 'exit 143' TERM HUP +trap 'exit 130' INT + DP_SIZE_LOCAL="$GPUS_PER_NODE" START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) @@ -327,11 +348,7 @@ fi # ================================================================ # Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval # ================================================================ -if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then - - # Release the allocation whenever the coordinator exits. - BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" - trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT +if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then # ---- Write endpoints.yaml (file-discovery) ---- # namespace must match EPP's --pool-namespace (file-discovery filters by it; @@ -582,10 +599,8 @@ PY ) fi - # Non-fatal: a failed or timed-out conc point must not abort the sweep - # or (under set -e) skip the allocation release below. The EXIT trap - # releases the allocation regardless, but continuing here lets a - # multi-conc sweep record every point it can. + # Continue collecting available points after a failure, retaining the + # nonzero verdict for the coordinator's final status and worker shutdown. run_benchmark_serving \ --bench-serving-dir /workspace \ --tokenizer /models \ @@ -600,7 +615,7 @@ PY --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \ --result-dir "$BENCHMARK_LOGS_DIR/" \ "${bench_extra_args[@]}" \ - || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" + || { BENCH_RC=$?; echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$BENCH_RC)"; } done fi @@ -631,10 +646,11 @@ PY ) fi - # Signal job.slurm (outside the container, where scancel exists) to release - # the allocation; without it workers wait until TIME_LIMIT. - touch "$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" + exit "$BENCH_RC" else - # Workers (prefill leader, prefill/decode workers): keep vLLM alive. - wait + while [[ ! -f "$BENCH_DONE_MARKER" ]]; do + kill -0 "$VLLM_PID" 2>/dev/null || exit 1 + sleep 2 + done + exit "$(cat "$BENCH_DONE_MARKER")" fi diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index daba2715b..2fcc7c039 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -96,6 +96,10 @@ Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physica InferenceX-app treats routing fields as columns or config dimensions and stores numeric measurements in `benchmark_results.metrics` JSONB. The mapper supports v1 shared topology, v2 split prefill/decode topology, and nested v3 AgentX metrics. Unknown numeric metrics are retained and warned about, which permits schema growth without silently losing numeric data. +### Slurm completion receipts + +Shared Slurm waiting verifies the terminal allocation state and exit code, consulting `scontrol` when `sacct` is missing or non-terminal and retaining `slurm_job_*_outcome.txt`. Launchers stage available evidence before returning failure. llm-d workers exit using the coordinator’s atomically published status-bearing completion marker; normal completion no longer cancels the allocation. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index cc5c9e302..edb2bb3a3 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -96,6 +96,10 @@ shape: array of benchmark row objects InferenceX-app 将路由字段作为列或配置维度，并把数值测量存入 `benchmark_results.metrics` JSONB。映射器支持共享拓扑的 v1、拆分 prefill/decode 拓扑的 v2，以及嵌套 AgentX 指标的 v3。未知数值指标会被保留并产生警告，因此架构可以扩展，同时不会无提示地丢失数值数据。 +### Slurm 完成状态文件 + +共享 Slurm 等待逻辑检查分配的最终状态和退出码；当 `sacct` 记录缺失或尚未进入最终状态时查询 `scontrol`，并保留 `slurm_job_*_outcome.txt`。启动器先保存已有证据再返回失败。llm-d 工作进程根据协调进程原子发布的完成状态退出；正常结束不再取消 Slurm 分配。 + ## 评测工件 ### 单配置身份和收集 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 88c163461..ebc4bdcce 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7399,3 +7399,15 @@ - "Disable adaptive verification in the EVAL_ONLY DSpark config as well. It trims verification requests on device, which the ROCm DeepseekV4IndexerBackend does not support, so the eval-only engine refused to start (run 34651830283, c32). Evals keep real block rejection; throughput settings are unchanged." - "EVAL_ONLY 的 DSpark 配置同样关闭自适应验证：它会在设备端裁剪验证请求，而 ROCm 的 DeepseekV4IndexerBackend 不支持该操作，导致仅评测引擎拒绝启动（运行 34651830283，c32）。评测仍保留真实块拒绝采样；吞吐设置不变。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + - dsv4-fp4-gb200-llmd-vllm + scenario-type: + - fixed-seq-len + description: + - "Verify terminal Slurm receipts and exit llm-d workers normally after preserving available evidence." + - "验证 Slurm 最终状态，并在保留已有证据后正常退出 llm-d 工作进程。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 960e33e9d..dd634bb53 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -449,10 +449,12 @@ EOF tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID + SRT_JOB_RC=0 + verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -504,6 +506,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else SQUASH_FILE="/data/home/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index b5e0ef627..62cd88d27 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -373,11 +373,12 @@ LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" # Waits for the log file to appear, fails fast if the job dies first, then # streams until the job leaves the queue. -stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -448,3 +449,5 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index c0c959d25..e68951f73 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -338,10 +338,12 @@ echo "Tailing LOG_FILE: $LOG_FILE" tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID +SRT_JOB_RC=0 +verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -393,6 +395,8 @@ for i in 1 2 3 4 5; do done find . -name '.nfs*' -delete 2>/dev/null || true +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else # HF_HUB_CACHE is set to help with dataset download inside the container # for eval jobs. diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index b6090491b..4a695aa9f 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -158,7 +158,8 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then trap 'bundle_server_logs "$BENCHMARK_LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true' EXIT INT TERM HUP LOG_FILE="${BENCHMARK_LOGS_DIR}/slurm_job-${JOB_ID}.out" - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + SRT_JOB_RC=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? while IFS= read -r -d '' result_file; do copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 @@ -173,7 +174,7 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then fi scancel "$JOB_ID" 2>/dev/null || true - exit 0 + exit "$SRT_JOB_RC" fi # MODEL_PATH: Override with pre-downloaded paths on GB200 runner @@ -822,11 +823,12 @@ trap 'exit 143' TERM HUP LOGS_DIR="outputs/$JOB_ID/logs" LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" -stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ -d "$LOGS_DIR" ]; then @@ -908,3 +910,5 @@ fi if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then copy_eval_artifacts "$LOGS_DIR/eval_results" "$GITHUB_WORKSPACE" || exit 1 fi + +exit "$SRT_JOB_RC" diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 9249e91f2..fede5b1f6 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -669,10 +669,12 @@ echo "Tailing LOG_FILE: $LOG_FILE" tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID +SRT_JOB_RC=0 +verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ -d "$LOGS_DIR" ]; then @@ -745,3 +747,5 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index af43c740c..de50907f6 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -195,10 +195,12 @@ EOF tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID + SRT_JOB_RC=0 + verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -244,6 +246,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else HF_HUB_CACHE_MOUNT="/mnt/nfs/sa-shared/gharunners/hf-hub-cache/" diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 95f519fd3..dbc2d5f58 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -348,11 +348,12 @@ EOF LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" trap 'rc=$?; bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT INT TERM HUP - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + SRT_JOB_RC=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -422,6 +423,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index accd85021..6e07dad91 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -123,6 +123,10 @@ if [[ "$IS_MULTINODE" == "true" ]]; then wait $POLL_PID + source "$GITHUB_WORKSPACE/runners/slurm_utils.sh" + slurm_outcome_rc=0 + verify_slurm_job_completion "$JOB_ID" || slurm_outcome_rc=$? + set -x # FIXME: The below is bad and is a result of the indirection of the ways in which @@ -249,6 +253,7 @@ PY sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true # Log preservation and cleanup handled by EXIT trap (cleanup_and_save_logs) + exit "$slurm_outcome_rc" else diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 8a6462f12..a8fe5b0d7 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -45,6 +45,32 @@ slurm_job_is_active() { squeue -j "$job_id" --noheader 2>/dev/null | grep -q "$job_id" } +verify_slurm_job_completion() { + local job_id="$1" records record state="" exit_code="" + # Disappearance from squeue only means the job is no longer active. Read + # the allocation's terminal record, not a successful batch/extern step. + records=$(sacct -j "$job_id" --noheader --parsable2 --format=JobIDRaw,State,ExitCode 2>/dev/null) || records="" + record=$(printf '%s\n' "$records" | awk -F'|' -v job="$job_id" '$1 == job {print; exit}') + if [[ -n "$record" ]]; then + IFS='|' read -r _ state exit_code <<< "$record" + fi + case "$state" in + COMPLETED|FAILED|CANCELLED*|TIMEOUT|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED|DEADLINE|BOOT_FAIL) ;; + *) + # Accounting may be missing or still report RUNNING after squeue + # empties. Ask the controller before judging the final outcome. + record=$(scontrol show job "$job_id" --oneliner 2>/dev/null) || record="" + state=$(printf '%s\n' "$record" | sed -n 's/.*JobState=\([^ ]*\).*/\1/p') + exit_code=$(printf '%s\n' "$record" | sed -n 's/.*ExitCode=\([^ ]*\).*/\1/p') + ;; + esac + printf '%s\n' "$record" > "${GITHUB_WORKSPACE:-.}/slurm_job_${job_id}_outcome.txt" + if [[ "$state" != "COMPLETED" || "$exit_code" != "0:0" ]]; then + echo "ERROR: Slurm job $job_id ended with state=${state:-unknown} exit=${exit_code:-unknown}" >&2 + return 1 + fi +} + stream_slurm_job_log() { local job_id="$1" local log_file="$2" @@ -52,7 +78,7 @@ stream_slurm_job_log() { while [[ ! -f "$log_file" ]]; do if ! slurm_job_is_active "$job_id"; then echo "ERROR: job $job_id failed before creating $log_file" >&2 - scontrol show job "$job_id" || true + verify_slurm_job_completion "$job_id" || true return 1 fi sleep 5 @@ -68,6 +94,7 @@ stream_slurm_job_log() { echo "Tailing $log_file" tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null wait "$poll_pid" + verify_slurm_job_completion "$job_id" } copy_to_workspace() { diff --git a/runners/test_llmd_lifecycle.py b/runners/test_llmd_lifecycle.py new file mode 100644 index 000000000..0b2145249 --- /dev/null +++ b/runners/test_llmd_lifecycle.py @@ -0,0 +1,90 @@ +"""Exercise llm-d shutdown with local stand-ins for the Slurm/container boundary.""" +import os +from pathlib import Path +import re +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' +SERVER = ROOT / 'benchmarks/multi_node/llm-d/server.sh' + + +@pytest.mark.parametrize('main_rc', [0, 7]) +def test_llmd_completed_coordinator_does_not_cancel_allocation(tmp_path, main_rc): + cwd = tmp_path / 'repo/benchmarks/multi_node/llm-d' + cwd.mkdir(parents=True) + model, logs, bindir = tmp_path / 'model', tmp_path / 'logs', tmp_path / 'bin' + for path in (model, logs, bindir): + path.mkdir() + squash = tmp_path / 'image.sqsh' + squash.write_text('fixture image') + commands = { + 'scontrol': '#!/bin/sh\nprintf "node-a\\nnode-b\\n"\n', + 'scancel': '#!/bin/sh\necho unexpected-cancel >> "$CANCEL_RECEIPT"\n', + 'sleep': '#!/bin/sh\n/bin/sleep 0.01\n', + 'srun': '#!' + sys.executable + '\n' + r''' +import os, pathlib, subprocess, sys, time +args = sys.argv[1:] +if any(a.startswith('--container-image=') for a in args): + marker = pathlib.Path(os.environ['BENCHMARK_LOGS_DIR']) / ('.bench_done.' + os.environ['SLURM_JOB_ID']) + marker.write_text(os.environ['MAIN_RC'] + '\n') + time.sleep(0.2) + sys.exit(int(os.environ['MAIN_RC'])) +if 'ip route' in ' '.join(args): + print('127.0.0.1') + sys.exit(0) +args = [arg for arg in args if not arg.startswith('--')] +sys.exit(subprocess.run(args).returncode) +''', + } + for name, script in commands.items(): + path = bindir / name + path.write_text(script) + path.chmod(0o755) + defaults = dict.fromkeys('PREFILL_WORKERS DECODE_WORKERS PREFILL_DP_SIZE DECODE_DP_SIZE ' + 'BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_MAX_CONCURRENCY ' + 'BENCH_REQUEST_RATE BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER ' + 'RUN_EVAL EVAL_ONLY EVAL_CONC EVAL_FRAMEWORK EVAL_LIMIT EVAL_SUITE ' + 'SWEBENCH_GEN_MODE SWEBENCH_USE_MODAL MODAL_TOKEN_ID MODAL_TOKEN_SECRET ' + 'IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX ' + 'RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE'.split(), '1') + env = {**os.environ, **defaults, 'PATH': str(bindir) + os.pathsep + os.environ['PATH'], + 'SLURM_JOB_ID': 'local-fixture', 'SLURM_JOB_NODELIST': 'node-[a-b]', + 'NUM_NODES': '2', 'PREFILL_NODES': '1', 'DECODE_NODES': '1', 'GPUS_PER_NODE': '2', + 'MODEL_DIR': str(model), 'MODEL_NAME': 'fixture', 'BENCHMARK_LOGS_DIR': str(logs), + 'LLMD_CONTAINER_ENGINE': 'pyxis', 'LLMD_SQUASH_FILE': str(squash), + 'MAIN_RC': str(main_rc), 'CANCEL_RECEIPT': str(tmp_path / 'cancelled')} + result = subprocess.run(['bash', str(JOB)], cwd=cwd, env=env, + capture_output=True, text=True, timeout=10) + assert result.returncode == main_rc, result.stderr + result.stdout + assert not (tmp_path / 'cancelled').exists() + + +@pytest.mark.parametrize('node_rc', [0, 7]) +def test_llmd_node_records_status_and_stops_owned_server(tmp_path, node_rc): + function = re.search(r'^finish_llmd_node\(\) \{\n.*?^\}', + SERVER.read_text(), flags=re.MULTILINE | re.DOTALL).group() + command = function + r''' +NODE_RANK=1 PREFILL_NODES=1 +BENCH_DONE_MARKER="$1/done" +# Observe the publication boundary after shell redirection opens its target, +# but before printf can write the status seen by workers. +printf() { + if [[ -e "$BENCH_DONE_MARKER" ]]; then + echo 'worker can observe an incomplete status' >&2 + return 1 + fi + builtin printf "$@" +} +sleep 60 & +VLLM_PID=$! +trap finish_llmd_node EXIT +exit "$2" +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(tmp_path), str(node_rc)], + capture_output=True, text=True, timeout=5) + assert result.returncode == node_rc, result.stderr + assert (tmp_path / 'done').read_text().strip() == str(node_rc) diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 31fa6f218..64a89b8cb 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -631,3 +631,60 @@ def test_mi355x_agentic_model_mount_and_routing( script = f"benchmarks/single_node/agentic/{prefix}_fp4_mi355x_vllm_mtp.sh" assert args[-2] == script assert (REPO_ROOT / script).is_file() + + +@pytest.mark.parametrize('state,exit_code,expected', [ + ('COMPLETED', '0:0', 0), ('FAILED', '1:0', 1), ('TIMEOUT', '0:15', 1), + ('COMPLETED', '1:0', 1), ('CANCELLED', '0:15', 1), +]) +def test_slurm_terminal_allocation_status_is_required(tmp_path, state, exit_code, expected): + result = run_bash( + f'source "$1"; export GITHUB_WORKSPACE="$2"; ' + f'sacct() {{ printf "42|{state}|{exit_code}\\n42.batch|COMPLETED|0:0\\n"; }}; ' + 'scontrol() { echo "JobId=42 JobState=COMPLETED ExitCode=0:0"; }; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == f'42|{state}|{exit_code}' + + +@pytest.mark.parametrize('accounting_state,controller_state,exit_code,expected', [ + ('', 'COMPLETED', '0:0', 0), + ('RUNNING', 'COMPLETED', '0:0', 0), + ('COMPLETING', 'COMPLETED', '0:0', 0), + ('RUNNING', 'FAILED', '1:0', 1), + ('RUNNING', 'RUNNING', '0:0', 1), +]) +def test_slurm_recent_completion_falls_back_to_controller( + tmp_path, accounting_state, controller_state, exit_code, expected, +): + record = f'42|{accounting_state}|0:0' if accounting_state else '' + controller = f'JobId=42 JobState={controller_state} ExitCode={exit_code}' + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; ' + f'sacct() {{ echo "{record}"; }}; ' + f'scontrol() {{ echo "{controller}"; }}; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == controller + + +def test_slurm_unknown_terminal_state_is_not_success(tmp_path): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; sacct() { return 1; }; ' + 'scontrol() { return 1; }; verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == 1 + assert 'state=unknown' in result.stderr + + +def test_slurm_exit_before_log_retains_terminal_receipt(tmp_path): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; ' + 'slurm_job_is_active() { return 1; }; ' + 'sacct() { printf "42|FAILED|1:0\\n"; }; ' + 'stream_slurm_job_log 42 "$2/missing.log"', SLURM_UTILS, tmp_path, + ) + assert result.returncode == 1 + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == '42|FAILED|1:0' From 85986f4cefd085906873a6e63a9e6fee8d04e646 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:24:14 -0700 Subject: [PATCH 05/21] fix: verify Slurm completion and exit llm-d workers normally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验证 Slurm 完成状态并让 llm-d 工作进程正常退出，先保存诊断证据。 --- .../workflows/benchmark-multinode-tmpl.yml | 1 + .github/workflows/benchmark-tmpl.yml | 1 + benchmarks/multi_node/llm-d/job.slurm | 26 ++---- benchmarks/multi_node/llm-d/server.sh | 46 ++++++---- docs/results-and-ingestion.md | 4 + docs/results-and-ingestion_zh.md | 4 + perf-changelog.yaml | 12 +++ runners/launch_b200-nscale-compat.sh | 6 +- runners/launch_b200-nscale-slurm.sh | 7 +- runners/launch_b300-dsxe.sh | 6 +- runners/launch_gb200-nv.sh | 12 ++- runners/launch_gb300-nv.sh | 6 +- runners/launch_h100-dgxc-slurm.sh | 6 +- runners/launch_h200-dgxc-slurm.sh | 7 +- runners/launch_mi355x-amds.sh | 5 ++ runners/slurm_utils.sh | 29 +++++- runners/test_llmd_lifecycle.py | 90 +++++++++++++++++++ runners/test_slurm_utils.py | 57 ++++++++++++ 18 files changed, 278 insertions(+), 47 deletions(-) create mode 100644 runners/test_llmd_lifecycle.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index bae33211b..cfbaefce8 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -517,6 +517,7 @@ jobs: ${{ env.RESULT_FILENAME }}_*.json agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json + slurm_job_*_outcome.txt LOGS/power/** LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index b08ef0775..d788a8023 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -467,6 +467,7 @@ jobs: gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json + slurm_job_*_outcome.txt results/gpu_metrics*.csv results/gpu_metrics_identity.json results/agentic_power_window.json diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm index 536507bee..e3338c31d 100644 --- a/benchmarks/multi_node/llm-d/job.slurm +++ b/benchmarks/multi_node/llm-d/job.slurm @@ -63,25 +63,11 @@ export DOCKER_CONT_NAME : "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}" DOCKER_MOUNT_PATH="/workspace" -cleanup() { - echo "[${SLURM_JOB_ID}] cleanup on $(hostname)" - [[ -n "${WATCHER_PID:-}" ]] && kill "$WATCHER_PID" 2>/dev/null || true -} -trap cleanup INT TERM HUP EXIT - -# Coordinator-done watcher. server.sh on the decode coordinator writes -# this marker after the bench finishes; we then scancel the allocation -# from outside the container (the image has no SLURM client tools). -# Without this, workers `wait` on local vLLM forever and the job runs -# to TIME_LIMIT. +# Workers exit through server.sh after reading the coordinator's final status. +# Normal completion must not cancel an otherwise successful allocation. BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" rm -f "$BENCH_DONE_MARKER" -( - while [[ ! -f "$BENCH_DONE_MARKER" ]]; do sleep 5; done - echo "[${SLURM_JOB_ID}] coordinator finished; scancel'ing job" - scancel "$SLURM_JOB_ID" 2>/dev/null || true -) & -WATCHER_PID=$! +main_rc=0 # Container engine: 'docker' (default) for clusters where the SLURM # user can talk to /var/run/docker.sock (e.g. h200-dgxc-slurm); 'pyxis' @@ -173,7 +159,7 @@ exec docker run --rm \ ${DOCKER_MOUNT_PATH}/benchmarks/multi_node/llm-d/server.sh \ 2>&1 | tee /benchmark_logs/slurm_job-'\"\$SLURM_JOB_ID\"'_rank_'\"\$SLURM_PROCID\"'.log ' -" +" || main_rc=$? srun bash -c "docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f" || true @@ -257,9 +243,11 @@ export MODEL_DIR=/models export BENCHMARK_LOGS_DIR=/benchmark_logs '"$DOCKER_MOUNT_PATH"'/benchmarks/multi_node/llm-d/server.sh \ 2>&1 | tee /benchmark_logs/slurm_job-${SLURM_JOB_ID}_rank_${SLURM_PROCID}.log -' +' || main_rc=$? else echo "Unsupported LLMD_CONTAINER_ENGINE: $LLMD_CONTAINER_ENGINE (expected docker|pyxis)" >&2 exit 1 fi + +exit "$main_rc" diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 89894b210..b38940a85 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -114,6 +114,27 @@ else DP_ADDR="$DECODE_DP_ADDR" fi +# One coordinator publishes the benchmark status; workers leave normally when +# it finishes so Slurm can distinguish success from cancellation. +BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" +BENCH_RC=0 + +finish_llmd_node() { + local rc=$? pid + trap - EXIT + if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then + printf '%s\n' "$rc" > "$BENCH_DONE_MARKER.tmp" && + mv -f "$BENCH_DONE_MARKER.tmp" "$BENCH_DONE_MARKER" || rc=1 + fi + for pid in "${ENVOY_PID:-}" "${EPP_PID:-}" "${SIDECAR_PID:-}" "${VLLM_PID:-}"; do + [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true + done + exit "$rc" +} +trap finish_llmd_node EXIT +trap 'exit 143' TERM HUP +trap 'exit 130' INT + DP_SIZE_LOCAL="$GPUS_PER_NODE" START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) @@ -327,11 +348,7 @@ fi # ================================================================ # Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval # ================================================================ -if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then - - # Release the allocation whenever the coordinator exits. - BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" - trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT +if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then # ---- Write endpoints.yaml (file-discovery) ---- # namespace must match EPP's --pool-namespace (file-discovery filters by it; @@ -582,10 +599,8 @@ PY ) fi - # Non-fatal: a failed or timed-out conc point must not abort the sweep - # or (under set -e) skip the allocation release below. The EXIT trap - # releases the allocation regardless, but continuing here lets a - # multi-conc sweep record every point it can. + # Continue collecting available points after a failure, retaining the + # nonzero verdict for the coordinator's final status and worker shutdown. run_benchmark_serving \ --bench-serving-dir /workspace \ --tokenizer /models \ @@ -600,7 +615,7 @@ PY --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \ --result-dir "$BENCHMARK_LOGS_DIR/" \ "${bench_extra_args[@]}" \ - || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" + || { BENCH_RC=$?; echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$BENCH_RC)"; } done fi @@ -631,10 +646,11 @@ PY ) fi - # Signal job.slurm (outside the container, where scancel exists) to release - # the allocation; without it workers wait until TIME_LIMIT. - touch "$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" + exit "$BENCH_RC" else - # Workers (prefill leader, prefill/decode workers): keep vLLM alive. - wait + while [[ ! -f "$BENCH_DONE_MARKER" ]]; do + kill -0 "$VLLM_PID" 2>/dev/null || exit 1 + sleep 2 + done + exit "$(cat "$BENCH_DONE_MARKER")" fi diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index daba2715b..2fcc7c039 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -96,6 +96,10 @@ Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physica InferenceX-app treats routing fields as columns or config dimensions and stores numeric measurements in `benchmark_results.metrics` JSONB. The mapper supports v1 shared topology, v2 split prefill/decode topology, and nested v3 AgentX metrics. Unknown numeric metrics are retained and warned about, which permits schema growth without silently losing numeric data. +### Slurm completion receipts + +Shared Slurm waiting verifies the terminal allocation state and exit code, consulting `scontrol` when `sacct` is missing or non-terminal and retaining `slurm_job_*_outcome.txt`. Launchers stage available evidence before returning failure. llm-d workers exit using the coordinator’s atomically published status-bearing completion marker; normal completion no longer cancels the allocation. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index cc5c9e302..edb2bb3a3 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -96,6 +96,10 @@ shape: array of benchmark row objects InferenceX-app 将路由字段作为列或配置维度，并把数值测量存入 `benchmark_results.metrics` JSONB。映射器支持共享拓扑的 v1、拆分 prefill/decode 拓扑的 v2，以及嵌套 AgentX 指标的 v3。未知数值指标会被保留并产生警告，因此架构可以扩展，同时不会无提示地丢失数值数据。 +### Slurm 完成状态文件 + +共享 Slurm 等待逻辑检查分配的最终状态和退出码；当 `sacct` 记录缺失或尚未进入最终状态时查询 `scontrol`，并保留 `slurm_job_*_outcome.txt`。启动器先保存已有证据再返回失败。llm-d 工作进程根据协调进程原子发布的完成状态退出；正常结束不再取消 Slurm 分配。 + ## 评测工件 ### 单配置身份和收集 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 88c163461..db1cd7627 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7399,3 +7399,15 @@ - "Disable adaptive verification in the EVAL_ONLY DSpark config as well. It trims verification requests on device, which the ROCm DeepseekV4IndexerBackend does not support, so the eval-only engine refused to start (run 34651830283, c32). Evals keep real block rejection; throughput settings are unchanged." - "EVAL_ONLY 的 DSpark 配置同样关闭自适应验证：它会在设备端裁剪验证请求，而 ROCm 的 DeepseekV4IndexerBackend 不支持该操作，导致仅评测引擎拒绝启动（运行 34651830283，c32）。评测仍保留真实块拒绝采样；吞吐设置不变。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + - dsv4-fp4-gb200-llmd-vllm + scenario-type: + - fixed-seq-len + description: + - "Verify terminal Slurm receipts and exit llm-d workers normally after preserving available evidence." + - "验证 Slurm 最终状态，并在保留已有证据后正常退出 llm-d 工作进程。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3052 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 960e33e9d..dd634bb53 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -449,10 +449,12 @@ EOF tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID + SRT_JOB_RC=0 + verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -504,6 +506,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else SQUASH_FILE="/data/home/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index b5e0ef627..62cd88d27 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -373,11 +373,12 @@ LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" # Waits for the log file to appear, fails fast if the job dies first, then # streams until the job leaves the queue. -stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -448,3 +449,5 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index c0c959d25..e68951f73 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -338,10 +338,12 @@ echo "Tailing LOG_FILE: $LOG_FILE" tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID +SRT_JOB_RC=0 +verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -393,6 +395,8 @@ for i in 1 2 3 4 5; do done find . -name '.nfs*' -delete 2>/dev/null || true +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else # HF_HUB_CACHE is set to help with dataset download inside the container # for eval jobs. diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index b6090491b..4a695aa9f 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -158,7 +158,8 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then trap 'bundle_server_logs "$BENCHMARK_LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true' EXIT INT TERM HUP LOG_FILE="${BENCHMARK_LOGS_DIR}/slurm_job-${JOB_ID}.out" - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + SRT_JOB_RC=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? while IFS= read -r -d '' result_file; do copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 @@ -173,7 +174,7 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then fi scancel "$JOB_ID" 2>/dev/null || true - exit 0 + exit "$SRT_JOB_RC" fi # MODEL_PATH: Override with pre-downloaded paths on GB200 runner @@ -822,11 +823,12 @@ trap 'exit 143' TERM HUP LOGS_DIR="outputs/$JOB_ID/logs" LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" -stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ -d "$LOGS_DIR" ]; then @@ -908,3 +910,5 @@ fi if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then copy_eval_artifacts "$LOGS_DIR/eval_results" "$GITHUB_WORKSPACE" || exit 1 fi + +exit "$SRT_JOB_RC" diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 9249e91f2..fede5b1f6 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -669,10 +669,12 @@ echo "Tailing LOG_FILE: $LOG_FILE" tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID +SRT_JOB_RC=0 +verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ -d "$LOGS_DIR" ]; then @@ -745,3 +747,5 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index af43c740c..de50907f6 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -195,10 +195,12 @@ EOF tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID + SRT_JOB_RC=0 + verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -244,6 +246,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else HF_HUB_CACHE_MOUNT="/mnt/nfs/sa-shared/gharunners/hf-hub-cache/" diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 95f519fd3..dbc2d5f58 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -348,11 +348,12 @@ EOF LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" trap 'rc=$?; bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT INT TERM HUP - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + SRT_JOB_RC=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -422,6 +423,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index accd85021..6e07dad91 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -123,6 +123,10 @@ if [[ "$IS_MULTINODE" == "true" ]]; then wait $POLL_PID + source "$GITHUB_WORKSPACE/runners/slurm_utils.sh" + slurm_outcome_rc=0 + verify_slurm_job_completion "$JOB_ID" || slurm_outcome_rc=$? + set -x # FIXME: The below is bad and is a result of the indirection of the ways in which @@ -249,6 +253,7 @@ PY sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true # Log preservation and cleanup handled by EXIT trap (cleanup_and_save_logs) + exit "$slurm_outcome_rc" else diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 8a6462f12..a8fe5b0d7 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -45,6 +45,32 @@ slurm_job_is_active() { squeue -j "$job_id" --noheader 2>/dev/null | grep -q "$job_id" } +verify_slurm_job_completion() { + local job_id="$1" records record state="" exit_code="" + # Disappearance from squeue only means the job is no longer active. Read + # the allocation's terminal record, not a successful batch/extern step. + records=$(sacct -j "$job_id" --noheader --parsable2 --format=JobIDRaw,State,ExitCode 2>/dev/null) || records="" + record=$(printf '%s\n' "$records" | awk -F'|' -v job="$job_id" '$1 == job {print; exit}') + if [[ -n "$record" ]]; then + IFS='|' read -r _ state exit_code <<< "$record" + fi + case "$state" in + COMPLETED|FAILED|CANCELLED*|TIMEOUT|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED|DEADLINE|BOOT_FAIL) ;; + *) + # Accounting may be missing or still report RUNNING after squeue + # empties. Ask the controller before judging the final outcome. + record=$(scontrol show job "$job_id" --oneliner 2>/dev/null) || record="" + state=$(printf '%s\n' "$record" | sed -n 's/.*JobState=\([^ ]*\).*/\1/p') + exit_code=$(printf '%s\n' "$record" | sed -n 's/.*ExitCode=\([^ ]*\).*/\1/p') + ;; + esac + printf '%s\n' "$record" > "${GITHUB_WORKSPACE:-.}/slurm_job_${job_id}_outcome.txt" + if [[ "$state" != "COMPLETED" || "$exit_code" != "0:0" ]]; then + echo "ERROR: Slurm job $job_id ended with state=${state:-unknown} exit=${exit_code:-unknown}" >&2 + return 1 + fi +} + stream_slurm_job_log() { local job_id="$1" local log_file="$2" @@ -52,7 +78,7 @@ stream_slurm_job_log() { while [[ ! -f "$log_file" ]]; do if ! slurm_job_is_active "$job_id"; then echo "ERROR: job $job_id failed before creating $log_file" >&2 - scontrol show job "$job_id" || true + verify_slurm_job_completion "$job_id" || true return 1 fi sleep 5 @@ -68,6 +94,7 @@ stream_slurm_job_log() { echo "Tailing $log_file" tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null wait "$poll_pid" + verify_slurm_job_completion "$job_id" } copy_to_workspace() { diff --git a/runners/test_llmd_lifecycle.py b/runners/test_llmd_lifecycle.py new file mode 100644 index 000000000..0b2145249 --- /dev/null +++ b/runners/test_llmd_lifecycle.py @@ -0,0 +1,90 @@ +"""Exercise llm-d shutdown with local stand-ins for the Slurm/container boundary.""" +import os +from pathlib import Path +import re +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' +SERVER = ROOT / 'benchmarks/multi_node/llm-d/server.sh' + + +@pytest.mark.parametrize('main_rc', [0, 7]) +def test_llmd_completed_coordinator_does_not_cancel_allocation(tmp_path, main_rc): + cwd = tmp_path / 'repo/benchmarks/multi_node/llm-d' + cwd.mkdir(parents=True) + model, logs, bindir = tmp_path / 'model', tmp_path / 'logs', tmp_path / 'bin' + for path in (model, logs, bindir): + path.mkdir() + squash = tmp_path / 'image.sqsh' + squash.write_text('fixture image') + commands = { + 'scontrol': '#!/bin/sh\nprintf "node-a\\nnode-b\\n"\n', + 'scancel': '#!/bin/sh\necho unexpected-cancel >> "$CANCEL_RECEIPT"\n', + 'sleep': '#!/bin/sh\n/bin/sleep 0.01\n', + 'srun': '#!' + sys.executable + '\n' + r''' +import os, pathlib, subprocess, sys, time +args = sys.argv[1:] +if any(a.startswith('--container-image=') for a in args): + marker = pathlib.Path(os.environ['BENCHMARK_LOGS_DIR']) / ('.bench_done.' + os.environ['SLURM_JOB_ID']) + marker.write_text(os.environ['MAIN_RC'] + '\n') + time.sleep(0.2) + sys.exit(int(os.environ['MAIN_RC'])) +if 'ip route' in ' '.join(args): + print('127.0.0.1') + sys.exit(0) +args = [arg for arg in args if not arg.startswith('--')] +sys.exit(subprocess.run(args).returncode) +''', + } + for name, script in commands.items(): + path = bindir / name + path.write_text(script) + path.chmod(0o755) + defaults = dict.fromkeys('PREFILL_WORKERS DECODE_WORKERS PREFILL_DP_SIZE DECODE_DP_SIZE ' + 'BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_MAX_CONCURRENCY ' + 'BENCH_REQUEST_RATE BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER ' + 'RUN_EVAL EVAL_ONLY EVAL_CONC EVAL_FRAMEWORK EVAL_LIMIT EVAL_SUITE ' + 'SWEBENCH_GEN_MODE SWEBENCH_USE_MODAL MODAL_TOKEN_ID MODAL_TOKEN_SECRET ' + 'IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX ' + 'RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE'.split(), '1') + env = {**os.environ, **defaults, 'PATH': str(bindir) + os.pathsep + os.environ['PATH'], + 'SLURM_JOB_ID': 'local-fixture', 'SLURM_JOB_NODELIST': 'node-[a-b]', + 'NUM_NODES': '2', 'PREFILL_NODES': '1', 'DECODE_NODES': '1', 'GPUS_PER_NODE': '2', + 'MODEL_DIR': str(model), 'MODEL_NAME': 'fixture', 'BENCHMARK_LOGS_DIR': str(logs), + 'LLMD_CONTAINER_ENGINE': 'pyxis', 'LLMD_SQUASH_FILE': str(squash), + 'MAIN_RC': str(main_rc), 'CANCEL_RECEIPT': str(tmp_path / 'cancelled')} + result = subprocess.run(['bash', str(JOB)], cwd=cwd, env=env, + capture_output=True, text=True, timeout=10) + assert result.returncode == main_rc, result.stderr + result.stdout + assert not (tmp_path / 'cancelled').exists() + + +@pytest.mark.parametrize('node_rc', [0, 7]) +def test_llmd_node_records_status_and_stops_owned_server(tmp_path, node_rc): + function = re.search(r'^finish_llmd_node\(\) \{\n.*?^\}', + SERVER.read_text(), flags=re.MULTILINE | re.DOTALL).group() + command = function + r''' +NODE_RANK=1 PREFILL_NODES=1 +BENCH_DONE_MARKER="$1/done" +# Observe the publication boundary after shell redirection opens its target, +# but before printf can write the status seen by workers. +printf() { + if [[ -e "$BENCH_DONE_MARKER" ]]; then + echo 'worker can observe an incomplete status' >&2 + return 1 + fi + builtin printf "$@" +} +sleep 60 & +VLLM_PID=$! +trap finish_llmd_node EXIT +exit "$2" +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(tmp_path), str(node_rc)], + capture_output=True, text=True, timeout=5) + assert result.returncode == node_rc, result.stderr + assert (tmp_path / 'done').read_text().strip() == str(node_rc) diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 31fa6f218..64a89b8cb 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -631,3 +631,60 @@ def test_mi355x_agentic_model_mount_and_routing( script = f"benchmarks/single_node/agentic/{prefix}_fp4_mi355x_vllm_mtp.sh" assert args[-2] == script assert (REPO_ROOT / script).is_file() + + +@pytest.mark.parametrize('state,exit_code,expected', [ + ('COMPLETED', '0:0', 0), ('FAILED', '1:0', 1), ('TIMEOUT', '0:15', 1), + ('COMPLETED', '1:0', 1), ('CANCELLED', '0:15', 1), +]) +def test_slurm_terminal_allocation_status_is_required(tmp_path, state, exit_code, expected): + result = run_bash( + f'source "$1"; export GITHUB_WORKSPACE="$2"; ' + f'sacct() {{ printf "42|{state}|{exit_code}\\n42.batch|COMPLETED|0:0\\n"; }}; ' + 'scontrol() { echo "JobId=42 JobState=COMPLETED ExitCode=0:0"; }; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == f'42|{state}|{exit_code}' + + +@pytest.mark.parametrize('accounting_state,controller_state,exit_code,expected', [ + ('', 'COMPLETED', '0:0', 0), + ('RUNNING', 'COMPLETED', '0:0', 0), + ('COMPLETING', 'COMPLETED', '0:0', 0), + ('RUNNING', 'FAILED', '1:0', 1), + ('RUNNING', 'RUNNING', '0:0', 1), +]) +def test_slurm_recent_completion_falls_back_to_controller( + tmp_path, accounting_state, controller_state, exit_code, expected, +): + record = f'42|{accounting_state}|0:0' if accounting_state else '' + controller = f'JobId=42 JobState={controller_state} ExitCode={exit_code}' + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; ' + f'sacct() {{ echo "{record}"; }}; ' + f'scontrol() {{ echo "{controller}"; }}; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == controller + + +def test_slurm_unknown_terminal_state_is_not_success(tmp_path): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; sacct() { return 1; }; ' + 'scontrol() { return 1; }; verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == 1 + assert 'state=unknown' in result.stderr + + +def test_slurm_exit_before_log_retains_terminal_receipt(tmp_path): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; ' + 'slurm_job_is_active() { return 1; }; ' + 'sacct() { printf "42|FAILED|1:0\\n"; }; ' + 'stream_slurm_job_log 42 "$2/missing.log"', SLURM_UTILS, tmp_path, + ) + assert result.returncode == 1 + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == '42|FAILED|1:0' From ba0275c4769dc42e606d6fec2e89a3c910b9eb5a Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:24:13 -0700 Subject: [PATCH 06/21] feat: separate native multinode power collection contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 拆分原生多节点功耗采集契约、UTC 上下文和生命周期屏障，不启用新路径。 --- .../workflows/benchmark-multinode-tmpl.yml | 1 + .github/workflows/benchmark-tmpl.yml | 4 + benchmarks/benchmark_lib.sh | 7 +- benchmarks/native_power_collect.sh | 73 +++++ benchmarks/native_power_lifecycle.sh | 77 +++++ docs/results-and-ingestion.md | 6 + docs/results-and-ingestion_zh.md | 6 + infx/results/fixed_sequence.py | 15 + infx/results/power/native_multinode.py | 286 ++++++++++++++++++ infx/results/power/single_node.py | 59 +++- perf-changelog.yaml | 11 + runners/test_native_collector_barriers.py | 58 ++++ runners/test_native_collector_receipts.py | 16 + utils/test_aggregate_power.py | 255 +++++++++++++++- utils/test_native_multinode_power.py | 201 ++++++++++++ 15 files changed, 1057 insertions(+), 18 deletions(-) create mode 100644 benchmarks/native_power_collect.sh create mode 100644 benchmarks/native_power_lifecycle.sh create mode 100644 infx/results/power/native_multinode.py create mode 100644 runners/test_native_collector_barriers.py create mode 100644 runners/test_native_collector_receipts.py create mode 100644 utils/test_native_multinode_power.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 80b0842f8..e9f3a3c90 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -499,6 +499,7 @@ jobs: agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json LOGS/power/** + LOGS/native_power/** result_processing_${{ env.RESULT_FILENAME }}.json LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 4d1b817ea..1da933f63 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -445,11 +445,13 @@ jobs: name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }} path: | gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json if-no-files-found: ignore @@ -462,12 +464,14 @@ jobs: ${{ env.RESULT_FILENAME }}.json agg_${{ env.RESULT_FILENAME }}.json gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json results/agentic_power_window.json results/agentic_power_timezone_offset.txt diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 305a118ef..9d99a4cc9 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -177,6 +177,7 @@ start_gpu_monitor() { GPU_METRICS_CSV="$output" GPU_MONITOR_INTERVAL="$interval" export GPU_METRICS_CSV + printf '{"timestamp_timezone":"UTC"}\n' > "${output%.csv}_context.json" if command -v nvidia-smi &>/dev/null; then GPU_MONITOR_VENDOR="nvidia" @@ -185,7 +186,7 @@ start_gpu_monitor() { rm -f "${output%.csv}_identity.csv" echo "[GPU Monitor] Warning: NVIDIA identity sidecar failed" >&2 fi - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv -l "$interval" > "$output" 2>/dev/null & GPU_MONITOR_PID=$! echo "[GPU Monitor] Started NVIDIA (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)" @@ -196,7 +197,7 @@ start_gpu_monitor() { # Python; measured on MI355X: trailing ticks were lost at kill without it). # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated # headers, and flush every row so killing the pipe cannot discard buffered samples. - PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ + TZ=UTC PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ | awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" & GPU_MONITOR_PID=$! # Hardware energy-accumulator + identity snapshots; the end-side twin in @@ -232,7 +233,7 @@ stop_gpu_monitor() { case "$GPU_MONITOR_VENDOR" in nvidia) if _repair_truncated_gpu_metrics_tail; then - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv,noheader >> "$GPU_METRICS_CSV" 2>/dev/null || echo "[GPU Monitor] Warning: final NVIDIA sample failed" >&2 fi diff --git a/benchmarks/native_power_collect.sh b/benchmarks/native_power_collect.sh new file mode 100644 index 000000000..69df07951 --- /dev/null +++ b/benchmarks/native_power_collect.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# One native SMI collector per serving node. Raw files stay on node-local scratch; +# the launcher stages them as its host user after containers stop. +set -uo pipefail +power_dir=$1 +control_dir=$2 +vendor=$3 +rank=$4 +role=$5 +gpu_indices=$6 +num_nodes=$7 +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" +source "$repo_root/benchmarks/benchmark_lib.sh" +mkdir -p "$power_dir" +collector_rc=0 +finished=0 + +write_control() { + local path="$control_dir/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + # The control directory is created by the host user before Docker starts. + # Containers must not strand root-owned files in the shared runner tree. + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +finish() { + local incoming_rc=$? + [[ "$finished" == 0 ]] || return + if [[ "$incoming_rc" != 0 ]]; then collector_rc=$incoming_rc; fi + if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi + stop_gpu_monitor + if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json + else + nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \ + > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1 + fi + python3 -m infx.results.power.native_multinode end --directory "$power_dir" \ + --collector-exit-code "$collector_rc" || collector_rc=1 + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown -R "$POWERX_HOST_UID:$POWERX_HOST_GID" "$power_dir" || collector_rc=1 + fi + write_control "done-$rank" "$collector_rc" + finished=1 +} +trap finish EXIT +trap 'collector_rc=130; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 130' INT +trap 'collector_rc=143; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 143' TERM HUP + +case "${POWERX_CLOCK_SYNCHRONIZED:-false}" in + yes|true) clock_synchronized=true ;; + *) clock_synchronized=false ;; +esac + +python3 -m infx.results.power.native_multinode begin --directory "$power_dir" \ + --vendor "$vendor" --rank "$rank" --role "$role" --gpu-indices "$gpu_indices" \ + --num-nodes "$num_nodes" --clock-synchronized "$clock_synchronized" || exit 1 +start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 +[[ "$GPU_MONITOR_VENDOR" == "$vendor" ]] || exit 1 +if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json +fi +write_control "ready-$rank" ready +while [[ ! -f "$control_dir/stop" ]]; do + _background_process_is_running "$GPU_MONITOR_PID" || exit 1 + sleep 1 & + wait $! || true +done diff --git a/benchmarks/native_power_lifecycle.sh b/benchmarks/native_power_lifecycle.sh new file mode 100644 index 000000000..b24d1dada --- /dev/null +++ b/benchmarks/native_power_lifecycle.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Shared barriers for native collectors. Launchers own host scratch and mounts. + +powerx_start_collector() { + local power_dir="$1" control_dir="$2" vendor="$3" rank="$4" role="$5" gpus="$6" nodes="$7" + local indices + indices=$(seq -s, 0 "$((gpus - 1))") || return 1 + POWERX_CONTROL_DIR="$control_dir" + POWERX_NUM_NODES="$nodes" + bash "$(dirname "${BASH_SOURCE[0]}")/native_power_collect.sh" \ + "$power_dir" "$control_dir" "$vendor" "$rank" "$role" "$indices" "$nodes" & + POWERX_COLLECTOR_PID=$! +} + +powerx_write_control() { + local path="$POWERX_CONTROL_DIR/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +powerx_wait_collectors() { + local phase="$1" deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rank pending failed + while :; do + pending=0 + failed=0 + for ((rank=0; rank&2 + return 1 + fi + if [[ ! -f "$POWERX_CONTROL_DIR/$phase-$rank" ]]; then + pending=1 + elif [[ "$phase" == done && "$(cat "$POWERX_CONTROL_DIR/done-$rank")" != 0 ]]; then + failed=1 + fi + done + if [[ "$pending" == 0 ]]; then + [[ "$failed" == 0 ]] || echo "One or more PowerX collectors failed" >&2 + return "$failed" + fi + if (( SECONDS >= deadline )); then + echo "Timed out waiting for PowerX $phase receipts" >&2 + return 1 + fi + sleep 1 + done +} + +powerx_stop_collectors() { + local rc=0 + powerx_write_control stop stop || rc=$? + powerx_wait_collectors done || rc=$? + powerx_reap_collector || rc=$? + return "$rc" +} + +powerx_reap_collector() { + [[ -n "${POWERX_COLLECTOR_PID:-}" ]] || return 0 + local deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rc=0 + while kill -0 "$POWERX_COLLECTOR_PID" 2>/dev/null; do + if (( SECONDS >= deadline )); then + kill -TERM "$POWERX_COLLECTOR_PID" 2>/dev/null || true + sleep 2 + kill -KILL "$POWERX_COLLECTOR_PID" 2>/dev/null || true + rc=1 + break + fi + sleep 1 + done + wait "$POWERX_COLLECTOR_PID" || rc=$? + POWERX_COLLECTOR_PID="" + return "$rc" +} diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index a1e357b30..6da11624d 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -106,6 +106,12 @@ For multinode fixed-sequence jobs, `utils/process_result.py --all` processes eve Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. +### Native multinode telemetry + +`native_power_collect.sh` and `native_power_lifecycle.sh` provide per-node collection and bounded ready/stop receipts. Launchers opt into the native package under `LOGS/native_power`; this prerequisite enables no new recipe. The adapter validates serving GPU identity, synchronized clocks, collector completion, and complete formal-window coverage. It preserves per-node failures, sample counts, and collector revision in the audit. + +SMI collection records UTC context beside each CSV for portable replay. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index 6a054d5f0..fd3fa2a2d 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -106,6 +106,12 @@ InferenceX-app 将路由字段作为列或配置维度，并把数值测量存 启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 +### 原生多节点遥测 + +`native_power_collect.sh` 和 `native_power_lifecycle.sh` 提供每节点采集及有时限的就绪/停止状态文件。启动器可使用 `LOGS/native_power` 下的原生产物；此前置改动不会启用新 recipe。适配器验证服务 GPU 身份、时钟同步、采集完成及正式窗口完整覆盖，并在审计中保留节点故障、样本数和采集器版本。 + +SMI 采集在 CSV 旁记录 UTC 上下文以支持跨环境回放。正式窗口外的无效样本不能构成覆盖；`boundary_degenerate_rows` 保留其逐 GPU 计数。 + ## 评测工件 ### 单配置身份和收集 diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index f6fa76102..5ae8a65ee 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -248,6 +248,21 @@ def aggregate_power_result( expected_num_gpus = int(env['TP']) * int(env.get('PP_SIZE', '1')) * int(env.get('PCP_SIZE', '1')) try: if is_multinode: + native_dir = Path(env.get('POWERX_NATIVE_DIR', 'LOGS/native_power')) + if env.get('POWERX_NATIVE_DIR') or native_dir.is_dir(): + if source.is_dir() and source != native_dir: + raise ValueError('Both native and SRT power packages are present') + source = native_dir + from .power.native_multinode import run + + return run( + native_dir, bench_path, agg_path, + expected_prefill_gpus=prefill_gpus, + expected_decode_gpus=decode_gpus, + expected_aggregate_gpus=aggregate_gpus, + validation_result=validation_path, + require_power=require_power, + ) from .power.multinode import run return run( diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py new file mode 100644 index 000000000..1ffae5bd9 --- /dev/null +++ b/infx/results/power/native_multinode.py @@ -0,0 +1,286 @@ +"""Validate native SMI traces from each node of one fixed-sequence deployment. + +This format is owned by InferenceX. It does not claim the srt-slurm/DCGM wire +contract. The launcher records real serving-device membership and synchronized +host clocks; the normal result processor binds every trace to the client window. +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import os +import socket +import time +from datetime import timezone +from pathlib import Path + +from . import ALL_POWER_METRIC_KEYS +from .common import ( + _load_benchmark_data, _write_json_atomic, benchmark_window_payload, + patch_power_metrics, +) +from .single_node import _derived_metrics, _detect_columns, _parse_timestamp, integrate_power + + +def _identity(path: Path, vendor: str) -> dict[str, str]: + """Map the vendor enumeration index to a physical UUID; never use PCI alone.""" + if vendor == "nvidia": + with path.open(newline="") as stream: + rows = [] + for row in csv.DictReader(stream, skipinitialspace=True): + if any(not isinstance(k, str) or not isinstance(v, str) for k, v in row.items()): + raise ValueError("invalid_device_identity") + rows.append({k.strip().lower(): v.strip() for k, v in row.items()}) + elif vendor == "amd": + payload = json.loads(path.read_text()) + rows = [] + + def visit(value: object) -> None: + if isinstance(value, list): + for item in value: + visit(item) + elif isinstance(value, dict): + row = {str(k).lower(): v for k, v in value.items()} + if "gpu" in row and "uuid" in row: + rows.append(row) + else: + for item in value.values(): + visit(item) + + visit(payload) + else: + raise ValueError("unsupported_native_vendor") + identities = {} + for row in rows: + index = str(row.get("index", row.get("gpu", ""))).strip() + uuid = str(row.get("uuid", "")).strip() + if not index.isdigit() or not uuid or uuid.lower() in {"n/a", "none", "null"}: + raise ValueError("invalid_device_identity") + if index in identities or uuid in identities.values(): + raise ValueError("duplicate_device_identity") + identities[index] = uuid + if not identities: + raise ValueError("device_identity_missing") + return identities + + +def record_begin(directory: Path, *, vendor: str, node: str, rank: int, + role: str, gpu_indices: list[int], num_nodes: int, job_id: str, + clock_synchronized: bool, revision: str) -> None: + if (role not in {"prefill", "decode", "aggregate"} or rank < 0 or + num_nodes <= rank or not gpu_indices or min(gpu_indices) < 0 or + len(set(gpu_indices)) != len(gpu_indices)): + raise ValueError("invalid_native_topology") + directory.mkdir(parents=True, exist_ok=True) + _write_json_atomic(directory / "manifest.json", { + "schema_version": 1, "collector": "inferencex-native-smi", + "vendor": vendor, "node": node, "rank": rank, "role": role, + "selected_gpu_indices": gpu_indices, "expected_num_nodes": num_nodes, + "job_id": job_id, "collector_revision": revision, + "clock_source": "utc_ntp", "clock_synchronized": clock_synchronized, + "clock_observation": "timedatectl NTPSynchronized on serving host; no measured clock offset", + "collection_start_unix": time.time(), "lifecycle": "collecting", + }) + + +def record_end(directory: Path, *, collector_exit_code: int) -> None: + manifest = json.loads((directory / "manifest.json").read_text()) + manifest.update(collection_end_unix=time.time(), + collector_exit_code=collector_exit_code, + lifecycle="complete" if collector_exit_code == 0 else "failed") + _write_json_atomic(directory / "manifest.json", manifest) + + +def run(power_dir: Path, bench_result: Path, agg_result: Path, *, + expected_prefill_gpus: int, expected_decode_gpus: int, expected_aggregate_gpus: int = 0, + validation_result: Path | None = None, require_power: bool = False) -> int: + validation_result = validation_result or bench_result.with_name( + f"power_validation_{bench_result.stem}.json") + benchmark, reasons = _load_benchmark_data(bench_result) + expected_gpus = expected_prefill_gpus + expected_decode_gpus + expected_aggregate_gpus + if expected_aggregate_gpus and (expected_prefill_gpus or expected_decode_gpus): + reasons.append("native_mixed_aggregate_role_topology") + roles: dict[str, list[str]] = {"prefill": [], "decode": [], "aggregate": []} + receipts = [] + node_errors = [] + samples = [] + ranks = [] + nodes = [] + jobs = set() + revisions = set() + expected_nodes = set() + paths = sorted(power_dir.glob("node-*/manifest.json")) + if not paths: + reasons.append("native_manifests_missing") + if expected_gpus <= 0 or min(expected_prefill_gpus, expected_decode_gpus, expected_aggregate_gpus) < 0: + reasons.append("invalid_expected_gpu_count") + for path in paths: + try: + manifest = json.loads(path.read_text()) + rank = manifest["rank"] + role = manifest["role"] + if (manifest.get("schema_version") != 1 or + manifest.get("collector") != "inferencex-native-smi" or + type(rank) is not int or rank < 0 or role not in roles or + path.parent.name != f"node-{rank}"): + raise ValueError("invalid_native_manifest") + if (not isinstance(manifest.get("node"), str) or not manifest["node"] or + not isinstance(manifest.get("job_id"), str) or + not isinstance(manifest.get("collector_revision"), str) or + type(manifest.get("expected_num_nodes")) is not int or + manifest["expected_num_nodes"] <= 0): + raise ValueError("invalid_native_run_identity") + ranks.append(rank) + nodes.append(manifest["node"]) + expected_nodes.add(manifest["expected_num_nodes"]) + jobs.add(manifest["job_id"]) + revisions.add(manifest["collector_revision"]) + if (manifest.get("lifecycle") != "complete" or + manifest.get("collector_exit_code") != 0): + reasons.append("native_collector_incomplete") + if (manifest.get("clock_source") != "utc_ntp" or + manifest.get("clock_synchronized") is not True): + reasons.append("native_clock_not_synchronized") + start, end = manifest["collection_start_unix"], manifest["collection_end_unix"] + if (not all(type(x) in (int, float) and math.isfinite(x) for x in (start, end)) + or end <= start or (benchmark is not None and + (start > benchmark.start_unix or end < benchmark.end_unix))): + reasons.append("native_collection_window_mismatch") + selected = manifest["selected_gpu_indices"] + if (not isinstance(selected, list) or not selected or + any(type(i) is not int or i < 0 for i in selected) or + len(set(selected)) != len(selected)): + raise ValueError("invalid_native_gpu_selection") + suffix = "csv" if manifest["vendor"] == "nvidia" else "json" + stem = "gpu_metrics_identity" if suffix == "csv" else "gpu_metrics_devices" + first_path = path.parent / f"{stem}.{suffix}" + last_path = path.parent / f"{stem}_end.{suffix}" + first = _identity(first_path, manifest["vendor"]) + last = _identity(last_path, manifest["vendor"]) + selected_ids = {str(i): first[str(i)] for i in selected} + if any(last.get(i) != uuid for i, uuid in selected_ids.items()): + reasons.append("native_device_identity_changed") + previous = {uuid for members in roles.values() for uuid in members} + if previous.intersection(selected_ids.values()): + reasons.append("native_duplicate_physical_gpu") + roles[role].extend(selected_ids.values()) + csv_path = path.parent / "gpu_metrics.csv" + with csv_path.open(newline="") as stream: + reader = csv.DictReader(stream, skipinitialspace=True) + reader.fieldnames = [c.strip() for c in (reader.fieldnames or [])] + t_col, p_col, g_col = _detect_columns(reader.fieldnames) + if not all((t_col, p_col, g_col)): + raise ValueError("native_telemetry_columns_missing") + for row in reader: + gpu = (row.get(g_col) or "").strip() + if gpu not in first: + reasons.append("native_unknown_device_index") + continue + if gpu not in selected_ids: + continue + timestamp = _parse_timestamp((row.get(t_col) or ""), naive_timezone=timezone.utc) + # Invalid rows are retained for the common validator to reject. + samples.append((timestamp, selected_ids[gpu], row.get(p_col) or "")) + receipts.append({**manifest, "physical_gpu_ids": selected_ids, + "manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "telemetry_sha256": hashlib.sha256(csv_path.read_bytes()).hexdigest(), + "identity_sha256": hashlib.sha256(first_path.read_bytes()).hexdigest(), + "identity_end_sha256": hashlib.sha256(last_path.read_bytes()).hexdigest()}) + except (OSError, ValueError, KeyError, TypeError, csv.Error) as exc: + reasons.append("native_node_invalid") + node_errors.append({"node": path.parent.name, "type": type(exc).__name__, "detail": str(exc)}) + if (len(expected_nodes) != 1 or not expected_nodes or + type(next(iter(expected_nodes), None)) is not int or + set(ranks) != set(range(next(iter(expected_nodes), 0))) or + len(set(nodes)) != len(nodes) or len(ranks) != len(set(ranks))): + reasons.append("native_node_topology_mismatch") + if len(jobs) != 1 or "" in jobs or len(revisions) != 1 or "" in revisions: + reasons.append("native_run_identity_mismatch") + if expected_aggregate_gpus: + if roles["prefill"] or roles["decode"] or len(roles["aggregate"]) != expected_aggregate_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif expected_decode_gpus: + if roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or len(roles["decode"]) != expected_decode_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif roles["decode"] or len(roles["aggregate"] + roles["prefill"]) != expected_prefill_gpus: + reasons.append("native_role_gpu_count_mismatch") + combined_path = validation_result.with_name(f"{validation_result.stem}_native.csv") + combined_path.parent.mkdir(parents=True, exist_ok=True) + with combined_path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["timestamp", "gpu", "power"]) + writer.writerows(samples) + integration = None + if benchmark is not None: + integration = integrate_power(combined_path, start_unix=benchmark.start_unix, + end_unix=benchmark.end_unix, expected_num_gpus=expected_gpus) + reasons.extend(integration.invalid_reasons) + metrics = {} + if not reasons and integration is not None and benchmark is not None: + metrics = _derived_metrics(integration, benchmark) + if expected_decode_gpus: + prefill = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["prefill"]) + decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) + metrics.update(prefill_gpu_energy_j=prefill, decode_gpu_energy_j=decode, + prefill_avg_power_w=prefill / benchmark.integration_duration_s / expected_prefill_gpus, + decode_avg_power_w=decode / benchmark.integration_duration_s / expected_decode_gpus, + prefill_joules_per_input_token=prefill / benchmark.total_input_tokens, + decode_joules_per_output_token=decode / benchmark.total_output_tokens) + valid = not reasons + try: + patch_power_metrics(agg_result, metric_keys=ALL_POWER_METRIC_KEYS, power_valid=valid, metrics=metrics) + except (OSError, ValueError): + reasons.append("aggregate_result_unwritable") + valid, metrics = False, {} + audit = {"schema_version": 1, "telemetry_kind": "native_multinode_smi", + "power_valid": valid, "reasons": list(dict.fromkeys(reasons)), + "benchmark_result": str(bench_result), + "benchmark_result_sha256": hashlib.sha256(bench_result.read_bytes()).hexdigest() if bench_result.is_file() else None, + "benchmark_window": benchmark_window_payload(benchmark), + "expected_gpu_count": expected_gpus, "nodes": receipts, + "node_errors": node_errors, + "observed_gpu_count": integration.observed_num_gpus if integration else 0, + "per_gpu_role": {uuid: role for role, uuids in roles.items() for uuid in uuids}, + "per_gpu_sample_counts": integration.per_gpu_sample_counts if integration else {}, + "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s if integration else {}, + "producer": {"name": "inferencex-native-smi", "revisions": sorted(revisions), + "producer_git_commit": next(iter(revisions)) if len(revisions) == 1 else None}, + "integration_method": "per_device_trapezoidal_with_linear_boundary_interpolation", + "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear", + "metrics": metrics} + _write_json_atomic(validation_result, audit) + return int(require_power and not valid) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + begin = sub.add_parser("begin") + begin.add_argument("--directory", type=Path, required=True) + begin.add_argument("--vendor", choices=("amd", "nvidia"), required=True) + begin.add_argument("--node", default=os.environ.get("POWERX_NODE_NAME", socket.gethostname())) + begin.add_argument("--rank", type=int, required=True) + begin.add_argument("--role", choices=("prefill", "decode", "aggregate"), required=True) + begin.add_argument("--gpu-indices", required=True) + begin.add_argument("--num-nodes", type=int, required=True) + begin.add_argument("--job-id", default=os.environ.get("SLURM_JOB_ID", "")) + begin.add_argument("--revision", default=os.environ.get("POWERX_COLLECTOR_REVISION", "")) + begin.add_argument("--clock-synchronized", choices=("true", "false"), default="false") + end = sub.add_parser("end") + end.add_argument("--directory", type=Path, required=True) + end.add_argument("--collector-exit-code", type=int, required=True) + args = vars(parser.parse_args()) + action = args.pop("action") + if action == "begin": + args["gpu_indices"] = [int(i) for i in args["gpu_indices"].split(",")] + args["clock_synchronized"] = args["clock_synchronized"] == "true" + record_begin(**args) + else: + record_end(**args) + + +if __name__ == "__main__": + main() diff --git a/infx/results/power/single_node.py b/infx/results/power/single_node.py index 54669a1a8..4ce1bcaaf 100644 --- a/infx/results/power/single_node.py +++ b/infx/results/power/single_node.py @@ -9,7 +9,11 @@ aggregate and a validation sidecar, but does not fail the benchmark. Power studies can set ``REQUIRE_POWER=1`` to fail after those audit artifacts exist. The aggregate carries numeric ``power_valid`` (1/0) for metric ingestion; the -sidecar is the canonical source for boolean validity and reason codes. +sidecar is the canonical source for boolean validity and reason codes. Rows in +the ingest band but outside the formal window whose power is missing, +non-finite, or <= 0 are teardown noise: they are skipped and counted in the +sidecar's ``boundary_degenerate_rows`` instead of poisoning validity or faking +window bracketing. """ from __future__ import annotations @@ -22,7 +26,7 @@ import os import re import sys -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from statistics import mean @@ -65,6 +69,10 @@ class PowerIntegration: per_gpu_max_sample_gap_s: dict[str, float] per_gpu_energy_j: dict[str, float] device_issues: dict[str, list[str]] + # Rows in the ingest band but outside the formal window whose power was + # missing/N-A/non-finite/<=0, skipped and counted per GPU; "unknown" + # buckets rows without a GPU identity. + boundary_degenerate_rows: dict[str, int] = field(default_factory=dict) avg_power_w: float | None = None p75_power_w: float | None = None p75_total_gpu_power_w: float | None = None @@ -78,7 +86,7 @@ def observed_num_gpus(self) -> int: return len(self.observed_gpu_ids) -def _parse_timestamp(value: str) -> float | None: +def _parse_timestamp(value: str, *, naive_timezone: timezone | None = None) -> float | None: """Best-effort timestamp parse to Unix epoch seconds (local wall clock). Handles the formats observed in practice: @@ -96,7 +104,7 @@ def _parse_timestamp(value: str) -> float | None: # nvidia-smi: "YYYY/MM/DD HH:MM:SS.ffffff" for fmt in ("%Y/%m/%d %H:%M:%S.%f", "%Y/%m/%d %H:%M:%S"): try: - return datetime.strptime(value, fmt).timestamp() + return datetime.strptime(value, fmt).replace(tzinfo=naive_timezone).timestamp() except ValueError: pass # ISO 8601 (amd-smi variants). fromisoformat tolerates 'T' or space separator @@ -108,7 +116,7 @@ def _parse_timestamp(value: str) -> float | None: return None if dt.tzinfo is None: # Treat naive timestamps as local time (matches nvidia-smi convention). - return dt.timestamp() + return dt.replace(tzinfo=naive_timezone).timestamp() return dt.astimezone(timezone.utc).timestamp() @@ -146,6 +154,16 @@ def _detect_columns(header: list[str]) -> tuple[str | None, str | None, str | No return timestamp_col, power_col, gpu_col +def _telemetry_timezone(csv_path: Path) -> timezone | None: + context = csv_path.with_name(f"{csv_path.stem}_context.json") + if not context.is_file(): + return None + payload = json.loads(context.read_text()) + if not isinstance(payload, dict) or payload.get("timestamp_timezone") != "UTC": + raise ValueError("unsupported_telemetry_timezone") + return timezone.utc + + def aggregate_power( csv_path: Path, start_unix: float, @@ -162,6 +180,7 @@ def aggregate_power( return None try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [c.strip() for c in (reader.fieldnames or [])] @@ -189,7 +208,7 @@ def aggregate_power( for row in reader: ts_raw = (row.get(timestamp_col) or "").strip() pw_raw = (row.get(power_col) or "").strip() - ts = _parse_timestamp(ts_raw) + ts = _parse_timestamp(ts_raw, naive_timezone=timestamp_timezone) pw = _parse_power(pw_raw) if ts is None or pw is None: continue @@ -204,7 +223,7 @@ def aggregate_power( if gpu_id: per_sample_gpus.setdefault(bucket, set()).add(gpu_id) gpu_keys.add(gpu_id) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): return None if not per_sample_total: @@ -237,6 +256,7 @@ def _empty_integration( *, expected_num_gpus: int | None, reasons: list[str], + boundary_degenerate_rows: dict[str, int] | None = None, ) -> PowerIntegration: """Build an invalid integration result when no device data is available.""" return PowerIntegration( @@ -248,6 +268,7 @@ def _empty_integration( per_gpu_max_sample_gap_s={}, per_gpu_energy_j={}, device_issues={}, + boundary_degenerate_rows=boundary_degenerate_rows or {}, ) @@ -308,8 +329,10 @@ def integrate_power( # expose timestamps at lower resolution than their sampling cadence, so # duplicate-timestamp readings are averaged rather than treated as corrupt. raw_samples: dict[str, dict[float, list[float]]] = {} + boundary_degenerate: dict[str, int] = {} saw_missing_gpu_identity = False try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -332,7 +355,7 @@ def integrate_power( ) for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) if timestamp is None or not math.isfinite(timestamp): _append_reason(reasons, "invalid_timestamp_sample") continue @@ -348,6 +371,15 @@ def integrate_power( power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() + if (power is None or not math.isfinite(power) or power <= 0.0) and ( + timestamp < start_unix or timestamp > end_unix + ): + # SMI teardown rows can carry N/A or 0 W cells: outside the + # formal window they are counted, never used to satisfy + # bracketing or to poison in-window validity. + key = gpu_id or "unknown" + boundary_degenerate[key] = boundary_degenerate.get(key, 0) + 1 + continue if power is None: _append_reason(reasons, "invalid_power_sample") continue @@ -359,11 +391,12 @@ def integrate_power( continue values = raw_samples.setdefault(gpu_id, {}).setdefault(timestamp, []) values.append(power) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): _append_reason(reasons, "telemetry_file_unreadable") return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) if saw_missing_gpu_identity: @@ -373,6 +406,7 @@ def integrate_power( return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) observed_gpu_ids = tuple(sorted(raw_samples, key=_gpu_sort_key)) @@ -450,6 +484,7 @@ def integrate_power( per_gpu_max_sample_gap_s=per_gpu_max_sample_gap_s, per_gpu_energy_j=per_gpu_energy_j, device_issues=device_issues, + boundary_degenerate_rows=boundary_degenerate, avg_power_w=avg_power_w, p75_power_w=p75_total / len(observed_gpu_ids) if p75_total is not None else None, p75_total_gpu_power_w=p75_total, @@ -485,6 +520,7 @@ def _read_energy_snapshot(path: Path) -> dict[str, float] | None: def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float]]]: """Group every parseable (timestamp, watt) sample per GPU, sorted in time.""" samples: dict[str, list[tuple[float, float]]] = {} + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -493,7 +529,7 @@ def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float] if not timestamp_col or not power_col or not gpu_col: return {} for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() if timestamp is None or power is None or not gpu_id: @@ -679,6 +715,7 @@ def _validation_payload( "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s, "per_gpu_energy_j": integration.per_gpu_energy_j, "device_issues": integration.device_issues, + "boundary_degenerate_rows": integration.boundary_degenerate_rows, "accumulator_check": accumulator_check, "metrics": audit_metrics(metrics), } @@ -754,7 +791,7 @@ def run( try: accumulator_check = cross_check_accumulator(csv_path) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): accumulator_check = {"available": False, "reason": "cross_check_error"} try: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f3b85de24..6d321aec8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7410,3 +7410,14 @@ - "Preserve request failure outcomes, retain all available fixed-sequence diagnostics, and reject incomplete concurrency batches before publishing benchmark artifacts." - "保留请求失败状态及全部可用的固定序列诊断信息；发布基准产物前拒绝缺失并发点的批次。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Retain UTC SMI telemetry context and validate native multinode collector artifacts without enabling new recipes." + - "保留 UTC SMI 遥测上下文并验证原生多节点采集产物，不启用新 recipe。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3051 diff --git a/runners/test_native_collector_barriers.py b/runners/test_native_collector_barriers.py new file mode 100644 index 000000000..14c8fe2ce --- /dev/null +++ b/runners/test_native_collector_barriers.py @@ -0,0 +1,58 @@ +"""CPU-only lifecycle checks with controlled external collector/Slurm processes.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +LIFECYCLE = ROOT / 'benchmarks/native_power_lifecycle.sh' +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' + + +@pytest.mark.parametrize('failed_rank', [None, 1]) +def test_stop_waits_for_every_collector_and_retains_failure(tmp_path, failed_rank): + command = ''' +source "$1" +export POWERX_CONTROL_DIR="$2" POWERX_NUM_NODES=2 POWERX_BARRIER_TIMEOUT_S=5 +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.1; echo 0 > "$2/done-0") & +POWERX_COLLECTOR_PID=$! +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.2; echo "$3" > "$2/done-1") & +remote_pid=$! +powerx_stop_collectors +rc=$? +wait "$remote_pid" +exit "$rc" +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(LIFECYCLE), + str(tmp_path), '1' if failed_rank else '0'], + capture_output=True, text=True, timeout=10) + assert result.returncode == (1 if failed_rank else 0), result.stderr + assert (tmp_path / 'done-0').read_text().strip() == '0' + assert (tmp_path / 'done-1').read_text().strip() == ('1' if failed_rank else '0') + + +def test_ready_barrier_rejects_collector_that_already_stopped(tmp_path): + (tmp_path / 'ready-0').write_text('ready') + (tmp_path / 'done-0').write_text('1') + result = subprocess.run(['bash', '-c', 'source "$1"; POWERX_CONTROL_DIR="$2"; ' + 'POWERX_NUM_NODES=1; powerx_wait_collectors ready', + 'bash', str(LIFECYCLE), str(tmp_path)], capture_output=True, text=True) + assert result.returncode == 1 + assert 'stopped before benchmark readiness' in result.stderr + + +def test_control_receipt_is_owned_before_publication(tmp_path): + command = '''source "$1" +POWERX_CONTROL_DIR=$2 +POWERX_HOST_UID=1000 POWERX_HOST_GID=1000 +chown() { [[ ! -e "$POWERX_CONTROL_DIR/stop" ]]; } +powerx_write_control stop stop +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(LIFECYCLE), str(tmp_path)], + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert (tmp_path / 'stop').read_text() == 'stop\n' + assert not list(tmp_path.glob('*.tmp')) diff --git a/runners/test_native_collector_receipts.py b/runners/test_native_collector_receipts.py new file mode 100644 index 000000000..b981f57ce --- /dev/null +++ b/runners/test_native_collector_receipts.py @@ -0,0 +1,16 @@ +import subprocess +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] + +def test_native_control_receipt_is_owned_before_publication(tmp_path): + source = (ROOT / 'benchmarks/native_power_collect.sh').read_text() + function = source[source.index('write_control() {'):source.index('\nfinish() {')] + result = subprocess.run(['bash', '-c', function + ''' +control_dir=$1 +POWERX_HOST_UID=1000 POWERX_HOST_GID=1000 +chown() { [[ ! -e "$control_dir/done-0" ]]; } +write_control done-0 7 +''', 'bash', str(tmp_path)], capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert (tmp_path / 'done-0').read_text() == '7\n' + assert not list(tmp_path.glob('*.tmp')) diff --git a/utils/test_aggregate_power.py b/utils/test_aggregate_power.py index aa3bf23f9..5b8b76121 100644 --- a/utils/test_aggregate_power.py +++ b/utils/test_aggregate_power.py @@ -574,6 +574,201 @@ def test_run_rejects_malformed_telemetry_inside_window( assert audit["reasons"] == [expected_reason] + +# AMDSMI 26.2.0 `metric -p -c -t -u -w 1 --csv` header (order-faithful subset, +# measured on MI355X; see test_detect_columns_amd_watch_mode_real_header). +_MI355X_WATCH_HEADER = ( + "timestamp,gpu,gfx_activity,umc_activity,mm_activity,vcn_activity," + "jpeg_activity,gfx_busy_inst_xcp_0,jpeg_busy_xcp_0,vcn_busy_xcp_0," + "socket_power,gfx_voltage,soc_voltage,mem_voltage,throttle_status," + "power_management,gfx_0_clk,mem_0_clk,edge,hotspot,mem" +) + + +def _mi355x_watch_row(timestamp: int, gpu: int, socket_power: str) -> str: + """One data row in the shape captured from run 32433563482 (conc1): + integer-second epoch, quoted list cells with embedded commas, N/A cells, + and a trailing carriage return.""" + return ( + f"{timestamp},{gpu},0,0,N/A,\"['N/A', 'N/A', 'N/A', 'N/A']\"," + "\"['N/A', 'N/A']\",\"[0, 0, 0, 0, 0, 0, 0, 0]\",\"[0, 0]\",\"[0, 0]\"," + f"{socket_power},N/A,N/A,N/A,N/A,ENABLED,1404,2000,N/A,40,25\r" + ) + + +def test_integrate_power_skips_na_power_rows_outside_window(tmp_path: Path): + """N/A-power teardown rows past the window end are counted, not poisonous.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-2, 13): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},N/A,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + assert result.total_gpu_energy_j == pytest.approx(10_000.0) + + +def test_integrate_power_does_not_bracket_with_zero_power_tail(tmp_path: Path): + """A 0 W teardown row past the window end must not fake end bracketing. + + Legacy behavior accepted the 0 W row as a valid boundary sample, corrupting + the end interpolation with a bogus value; this intentionally flips that + case to an explicit benchmark_window_not_bracketed failure.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + # Good rows stop at end-4; the only post-end sample per GPU has power 0. + for offset in range(-1, 7): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},0,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "benchmark_window_not_bracketed" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + + +def test_integrate_power_preserves_boundary_counts_without_usable_samples(tmp_path: Path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text( + "timestamp,gpu,socket_power\n" + "1699999999,0,N/A\n" + "1700000011,0,0\n" + "1700000011,1,N/A\n", + encoding="utf-8", + ) + + result = integrate_power( + csv, + start_unix=1_700_000_000, + end_unix=1_700_000_010, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "no_usable_power_samples" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 2, "1": 1} + + +def test_integrate_power_keeps_zero_power_semantics_inside_window(tmp_path: Path): + """Frozen legacy behavior: an in-window 0 W sample still integrates.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-1, 12): + watts = "0.0" if offset == 5 else "500.0" + lines.append(f"{base + offset},0,{watts},65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=1, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {} + # Trapezoids dip to 0 at t=5: 8 x 500 + 2 x 250 = 4500 J. + assert result.total_gpu_energy_j == pytest.approx(4_500.0) + + +def test_run_writes_boundary_degenerate_rows_to_sidecar(tmp_path: Path): + base = 1_700_000_000.0 + csv = tmp_path / "gpu_metrics.csv" + _write_constant_window_samples( + csv, + start=base, + end=base + 10, + watts_per_gpu=500.0, + num_gpus=2, + ) + bench = tmp_path / "bench.json" + agg = tmp_path / "agg.json" + validation = tmp_path / "power_validation.json" + _write_bench_result( + bench, + start=base, + end=base + 10, + duration=10.0, + total_output=2_000, + total_input=10_000, + ) + agg.write_text(json.dumps({"hw": "mi355x"}), encoding="utf-8") + + exit_code = run(csv, bench, agg, expected_num_gpus=2, validation_result=validation) + + assert exit_code == 0 + audit = json.loads(validation.read_text()) + # Present-and-empty for clean streams: readers can rely on the key. + assert audit["boundary_degenerate_rows"] == {} + + +def test_integrate_power_regression_mi355x_integer_ticks_end_gap(tmp_path: Path): + """Run-32433563482 conc1 regression: amd-smi integer-second ticks stop 4 s + before the fractional aiperf window end (last tick 1787277605 vs end + ...609.157497), so bracketing fails and the producer-side telemetry loss is + attributed as benchmark_window_not_bracketed on every GPU. + + The retrieved artifact's trailing rows all carry valid socket_power + (254-264 W) with N/A activity/voltage cells; the N/A- and 0-power teardown + rows appended past the window end are the documented synthetic degenerate + shapes, asserting they are counted rather than used for bracketing.""" + csv = tmp_path / "gpu_metrics.csv" + start = 1_787_277_560.155891 + end = 1_787_277_609.157497 + last_tick = 1_787_277_605 + powers = [259, 255, 263, 264, 256, 254, 259, 259] + lines = [_MI355X_WATCH_HEADER] + for tick in range(1_787_277_555, last_tick + 1): + for gpu in range(8): + lines.append(_mi355x_watch_row(tick, gpu, str(powers[gpu]))) + # amd-smi watch mode emits a blank line between tick groups. + lines.append("") + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_610, gpu, "N/A")) + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_611, gpu, "0")) + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=start, + end_unix=end, + expected_num_gpus=8, + ) + + assert result.power_valid is False + assert result.invalid_reasons == ("benchmark_window_not_bracketed",) + assert result.device_issues == { + str(gpu): ["benchmark_window_not_bracketed"] for gpu in range(8) + } + assert result.boundary_degenerate_rows == {str(gpu): 2 for gpu in range(8)} + + def test_run_patches_agg_with_power_and_joules(tmp_path: Path): base = 1_700_000_000.0 csv = tmp_path / "gpu_metrics.csv" @@ -708,6 +903,7 @@ def test_power_replacement_removes_stale_metrics(tmp_path, patch_validated_power path.write_text(json.dumps({ "hw": "fixture", "avg_power_w": 99, "total_gpu_energy_j": 50, "power_invalid_reasons": ["stale"], + "power_audit": {"source": "previous-run.json"}, })) patch_validated_power(path, power_valid=valid, metrics={ "avg_power_w": 12.34567, "joules_per_output_token": 0.12345678, @@ -1433,12 +1629,12 @@ def test_power_percentiles_uses_synchronized_total_not_device_percentiles(tmp_pa def test_power_percentiles_weights_time_and_clips_the_validated_window(tmp_path): csv_path = tmp_path / "power.csv" - # Dense readings near the high end must not bias a uniform linear ramp. - _write_amd_csv(csv_path, [(0, 0, 0), (1, 0, 100), (1.9, 0, 190), (2, 0, 200)]) + # Dense readings must not bias the uniform 150-250 W ramp inside the window. + _write_amd_csv(csv_path, [(0, 0, 100), (1, 0, 200), (1.9, 0, 290), (2, 0, 300)]) result = integrate_power(csv_path, start_unix=0.5, end_unix=1.5, expected_num_gpus=1) assert result.power_valid - assert result.p75_power_w == pytest.approx(125) - assert result.p90_power_w == pytest.approx(140) + assert result.p75_power_w == pytest.approx(225) + assert result.p90_power_w == pytest.approx(240) def test_power_percentiles_is_withheld_for_invalid_telemetry(tmp_path): @@ -1462,3 +1658,54 @@ def test_power_percentiles_aligns_asynchronous_gpu_samples(tmp_path): assert result.p75_power_w == pytest.approx(300) assert result.p90_total_gpu_power_w == pytest.approx(600) assert result.p90_power_w == pytest.approx(300) + + +@pytest.mark.parametrize('step_name', ['Upload GPU metrics', 'Upload power audit bundle']) +@pytest.mark.parametrize('directory', ['', 'results']) +def test_uploaded_telemetry_replays_in_a_different_timezone(tmp_path, step_name, directory): + import os + import shutil + import yaml + + repo = Path(__file__).resolve().parents[1] + source = tmp_path / 'source' + telemetry = source / directory + telemetry.mkdir(parents=True) + (telemetry / 'gpu_metrics.csv').write_text( + 'timestamp,index,power.draw [W]\n' + '2024/01/01 00:00:00.000,0,100 W\n' + '2024/01/01 00:00:01.000,0,100 W\n' + '2024/01/01 00:00:02.000,0,100 W\n') + (telemetry / 'gpu_metrics_context.json').write_text('{"timestamp_timezone":"UTC"}') + workflow = yaml.safe_load((repo / '.github/workflows/benchmark-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job['steps'] + if step.get('name') == step_name) + archive = tmp_path / 'downloaded' + for pattern in step['with']['path'].splitlines(): + for file in source.glob(pattern): + destination = archive / file.relative_to(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(file, destination) + result = subprocess.run( + [sys.executable, '-c', """ +import sys, time +from pathlib import Path +from infx.results.power.single_node import integrate_power +time.tzset() +result = integrate_power(Path(sys.argv[1]), start_unix=1704067200, end_unix=1704067202, + expected_num_gpus=1) +assert result.power_valid, result.invalid_reasons +assert result.total_gpu_energy_j == 200 +""", str(archive / directory / 'gpu_metrics.csv')], + cwd=tmp_path, env={**os.environ, 'TZ': 'Etc/GMT+8', 'PYTHONPATH': str(repo)}, + capture_output=True, text=True, timeout=10, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize('context', ['{"timestamp_timezone":"PST"}', '{}', '{invalid']) +def test_legacy_average_rejects_invalid_telemetry_context(tmp_path, context): + csv = tmp_path / 'gpu_metrics.csv' + csv.write_text('timestamp,index,power.draw [W]\n1,0,100\n2,0,100\n') + csv.with_name('gpu_metrics_context.json').write_text(context) + assert aggregate_power(csv, 1, 2) is None diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py new file mode 100644 index 000000000..0aeee2511 --- /dev/null +++ b/utils/test_native_multinode_power.py @@ -0,0 +1,201 @@ +"""Native collector acceptance uses independent, hand-computed two-node traces.""" +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from infx.results.power.native_multinode import record_begin, record_end, run +from infx.results.power.single_node import integrate_power + +REPO = Path(__file__).resolve().parents[1] + + +def _package(tmp_path, vendor="amd"): + root = tmp_path / "native_power" + for rank, role, watts in ((0, "prefill", 100), (1, "decode", 300)): + node = root / f"node-{rank}" + record_begin(node, vendor=vendor, node=f"host-{rank}", rank=rank, role=role, + gpu_indices=[0], num_nodes=2, job_id="job-123", revision="revision-abc", + clock_synchronized=True) + record_end(node, collector_exit_code=0) + manifest = json.loads((node / "manifest.json").read_text()) + manifest.update(collection_start_unix=0, collection_end_unix=5) + (node / "manifest.json").write_text(json.dumps(manifest)) + for ending in ("", "_end"): + if vendor == "amd": + (node / f"gpu_metrics_devices{ending}.json").write_text(json.dumps([ + {"gpu": 0, "uuid": f"uuid-{rank}"}, {"gpu": 1, "uuid": f"unused-{rank}"}])) + else: + (node / f"gpu_metrics_identity{ending}.csv").write_text( + f"index, uuid, pci.bus_id\n0, uuid-{rank}, 0000:01:00.0\n1, unused-{rank}, 0000:02:00.0\n") + # The spare physical GPU is visible but does not belong to the server. + (node / "gpu_metrics.csv").write_text("timestamp,gpu,power\n" + "".join( + f"{tick},0,{watts}\n{tick},1,900\n" for tick in range(5))) + bench = tmp_path / "result.json" + bench.write_text(json.dumps({"benchmark_start_time_unix": 1, "benchmark_end_time_unix": 3, + "duration": 2, "completed": 2, "total_input_tokens": 20, + "total_output_tokens": 10})) + agg = tmp_path / "agg.json" + agg.write_text(json.dumps({"avg_power_w": 999, "prefill_gpu_energy_j": 999})) + return root, bench, agg + + +@pytest.mark.parametrize("vendor", ["amd", "nvidia"]) +def test_native_whole_fleet_and_role_energy_use_only_serving_devices(tmp_path, vendor): + root, bench, agg = _package(tmp_path, vendor) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 1 + assert actual["total_gpu_energy_j"] == 800 + assert actual["avg_power_w"] == 200 + assert actual["p90_power_w"] == 200 + assert actual["p75_power_w"] == 200 + assert actual["joules_per_successful_query"] == 400 + assert actual["prefill_gpu_energy_j"] == 200 + assert actual["decode_gpu_energy_j"] == 600 + assert actual["prefill_joules_per_input_token"] == 10 + assert actual["decode_joules_per_output_token"] == 60 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert audit["observed_gpu_count"] == 2 + assert audit["per_gpu_role"] == {"uuid-0": "prefill", "uuid-1": "decode"} + assert len(audit["nodes"][0]["telemetry_sha256"]) == 64 + from infx.results.power.audit import audit_summary + summary = audit_summary(audit, "power_validation_result.json")["power_audit"] + assert summary["sample_count"] == 10 # Five samples on each of two participating GPUs. + assert summary["producer_sha"] == "revision-abc" + + +def test_native_parse_failure_has_a_public_reason_and_retained_detail(tmp_path): + from infx.results.power.audit import audit_summary + root, bench, agg = _package(tmp_path) + (root / "node-1/manifest.json").write_text("not JSON") + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + summary = audit_summary(audit, "power_validation_result.json") + assert "native_node_invalid" in summary["power_invalid_reasons"] + assert audit["node_errors"][0]["node"] == "node-1" + assert audit["node_errors"][0]["detail"] + + +@pytest.mark.parametrize(("field", "value", "reason"), [ + ("clock_synchronized", False, "native_clock_not_synchronized"), + ("lifecycle", "collecting", "native_collector_incomplete"), + ("collection_end_unix", 2, "native_collection_window_mismatch"), + ("job_id", "other-job", "native_run_identity_mismatch"), + ("role", "prefill", "native_role_gpu_count_mismatch"), + ("expected_num_nodes", 3, "native_node_topology_mismatch"), +]) +def test_native_invalid_evidence_clears_stale_metrics_and_writes_audit(tmp_path, field, value, reason): + root, bench, agg = _package(tmp_path) + path = root / "node-1/manifest.json" + manifest = json.loads(path.read_text()); manifest[field] = value + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 0 + assert "avg_power_w" not in actual + assert "prefill_gpu_energy_j" not in actual + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert reason in audit["reasons"] + + +def test_native_device_replacement_cannot_preserve_validity(tmp_path): + root, bench, agg = _package(tmp_path) + (root / "node-1/gpu_metrics_devices_end.json").write_text('[{"gpu":0,"uuid":"replacement"}]') + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + assert "native_device_identity_changed" in json.loads((tmp_path / "power_validation_result.json").read_text())["reasons"] + + +def test_native_aggregate_nodes_do_not_invent_prefill_decode_metrics(tmp_path): + root, bench, agg = _package(tmp_path) + for path in root.glob("*/manifest.json"): + manifest = json.loads(path.read_text()); manifest["role"] = "aggregate" + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=0, expected_decode_gpus=0, + expected_aggregate_gpus=2, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["avg_power_w"] == 200 + assert "prefill_gpu_energy_j" not in actual + + +def test_utc_context_replays_in_a_different_timezone(tmp_path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text("timestamp,index,power.draw [W]\n2026/01/01 00:00:00,0,100\n2026/01/01 00:00:02,0,100\n") + (tmp_path / "gpu_metrics_context.json").write_text('{"timestamp_timezone":"UTC"}') + script = "from pathlib import Path; from infx.results.power.single_node import integrate_power; " + \ + f"r=integrate_power(Path({str(csv)!r}),start_unix=1767225600,end_unix=1767225602,expected_num_gpus=1); assert r.power_valid; assert r.total_gpu_energy_j == 200" + subprocess.run([sys.executable, "-c", script], cwd=REPO, + env={**os.environ, "TZ": "America/Los_Angeles"}, check=True, timeout=10) + + +@pytest.mark.parametrize("clock_value, synchronized", [ + ("yes", True), ("true", True), ("no", False), ("false", False), ("", False), +]) +def test_native_supervisor_reaps_monitor_and_writes_completion(tmp_path, clock_value, synchronized): + binary = tmp_path / "bin"; binary.mkdir() + (binary / "python3").symlink_to(sys.executable) + fake = binary / "nvidia-smi" + fake.write_text(f'''#!{sys.executable} +import datetime, os, sys, time +if any("index,uuid" in arg for arg in sys.argv): + print("index, uuid, pci.bus_id"); print("0, gpu-0, 0000:01:00.0") +else: + with open({str(tmp_path / 'monitor.pid')!r}, "w") as stream: stream.write(str(os.getpid())) + if "-l" in sys.argv: print("timestamp,index,power.draw [W]", flush=True) + while True: + print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y/%m/%d %H:%M:%S.%f") + ",0,100", flush=True) + if "-l" not in sys.argv: break + time.sleep(0.1) +''') + fake.chmod(0o755) + control = tmp_path / "control"; control.mkdir() + node = tmp_path / "node-0" + process = subprocess.Popen(["bash", str(REPO / "benchmarks/native_power_collect.sh"), + str(node), str(control), "nvidia", "0", "aggregate", "0", "1"], + env={**os.environ, "PATH": f"{binary}:{os.environ['PATH']}", + "SLURM_JOB_ID": "test-job", "POWERX_COLLECTOR_REVISION": "revision", + "POWERX_CLOCK_SYNCHRONIZED": clock_value}, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + try: + deadline = time.monotonic() + 10 + while not (control / "ready-0").exists(): + assert process.poll() is None + assert time.monotonic() < deadline + time.sleep(0.02) + (control / "stop").write_text("stop") + stdout, stderr = process.communicate(timeout=10) + assert process.returncode == 0, (stdout, stderr) + assert (control / "done-0").read_text().strip() == "0" + manifest = json.loads((node / "manifest.json").read_text()) + assert manifest["lifecycle"] == "complete" + assert manifest["clock_synchronized"] is synchronized + assert (node / "gpu_metrics_identity_end.csv").exists() + finally: + if process.poll() is None: + process.terminate(); process.communicate(timeout=10) + + +def test_result_processor_discovers_staged_native_package(tmp_path, monkeypatch): + from infx.results.fixed_sequence import aggregate_power_result + + root, bench, agg = _package(tmp_path) + logs = tmp_path / "LOGS" + logs.mkdir() + root.rename(logs / "native_power") + monkeypatch.chdir(tmp_path) + env = {"IS_MULTINODE": "true", "PREFILL_GPUS": "1", "DECODE_GPUS": "1", + "RESULT_FILENAME": "result", "REQUIRE_POWER": "1"} + assert aggregate_power_result(env, bench, agg) == 0 + assert json.loads(agg.read_text())["total_gpu_energy_j"] == 800 + assert (tmp_path / "power_validation_result.json").is_file() + + # Two competing formats must not silently select one package. + (logs / "power").mkdir() + assert aggregate_power_result(env, bench, agg) == 1 + invalid = json.loads(agg.read_text()) + assert invalid["power_valid"] == 0 + assert "total_gpu_energy_j" not in invalid From d70d6caf64d27810dbc4ad014da6cba4e93577e3 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:28:35 -0700 Subject: [PATCH 07/21] feat: collect native power for llm-d 8k1k MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为 llm-d 8k1k 收集原生功耗，并协调节点 readiness、排空与结果证据保存。 --- benchmarks/multi_node/llm-d/job.slurm | 45 ++++++++++++++++-- benchmarks/multi_node/llm-d/server.sh | 31 ++++++++++-- benchmarks/multi_node/llm-d/submit.sh | 4 ++ configs/nvidia-master.yaml | 1 + docs/configuration-procedures.md | 4 ++ docs/configuration-procedures_zh.md | 4 ++ perf-changelog.yaml | 10 ++++ runners/launch_gb200-nv.sh | 2 + runners/test_llmd_power_lifecycle.py | 68 +++++++++++++++++++++++++++ 9 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 runners/test_llmd_power_lifecycle.py diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm index e3338c31d..19598d868 100644 --- a/benchmarks/multi_node/llm-d/job.slurm +++ b/benchmarks/multi_node/llm-d/job.slurm @@ -63,11 +63,43 @@ export DOCKER_CONT_NAME : "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}" DOCKER_MOUNT_PATH="/workspace" -# Workers exit through server.sh after reading the coordinator's final status. -# Normal completion must not cancel an otherwise successful allocation. +# Normal completion is coordinated inside server.sh. Cancelling a successful +# allocation would discard its terminal success state and interrupt collectors. BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" rm -f "$BENCH_DONE_MARKER" +POWERX_NATIVE_ENABLED="${POWERX_NATIVE_ENABLED:-0}" +export POWERX_HOST_UID="$(id -u)" POWERX_HOST_GID="$(id -g)" +export POWERX_COLLECTOR_REVISION="$(git -C "$DI_REPO_DIR" rev-parse HEAD)" +export POWERX_RAW_ROOT="${POWERX_RAW_ROOT:-/tmp/inferencex-native-$SLURM_JOB_ID}" +srun --nodes="$NUM_NODES" --ntasks-per-node=1 mkdir -p "$POWERX_RAW_ROOT" +if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + mkdir -p "$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID" "$DI_REPO_DIR/LOGS/native_power" + chmod 777 "$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID" + srun --nodes="$NUM_NODES" --ntasks-per-node=1 chmod 777 "$POWERX_RAW_ROOT" + srun --nodes="$NUM_NODES" --ntasks-per-node=1 bash -c ' + { hostname; timedatectl show -p NTPSynchronized --value 2>/dev/null || echo false; } \ + > "$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID/host-$SLURM_PROCID" + ' +fi main_rc=0 +finish_llmd_job() { + local rc=$? + trap - EXIT + [[ "$rc" == 0 ]] || main_rc=$rc + + if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + # Copy as the host user only after collectors and serving containers drain. + srun --nodes="$NUM_NODES" --ntasks-per-node=1 bash -c ' + source_dir="$POWERX_RAW_ROOT/node-$SLURM_PROCID" + [[ -d "$source_dir" ]] || exit 1 + cp -R "$source_dir" "$DI_REPO_DIR/LOGS/native_power/" + ' || main_rc=$? + fi + exit "$main_rc" +} +trap finish_llmd_job EXIT +trap 'exit 143' TERM HUP +trap 'exit 130' INT # Container engine: 'docker' (default) for clusters where the SLURM # user can talk to /var/run/docker.sock (e.g. h200-dgxc-slurm); 'pyxis' @@ -105,10 +137,15 @@ exec docker run --rm \ --privileged \ -v ${MODEL_DIR}:/models:ro \ -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ + -v ${POWERX_RAW_ROOT}:/powerx_native \ -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ -v ${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro \ -v ${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro \ -v ${DI_REPO_DIR}/benchmarks/llm-d/envoy.yaml:/etc/envoy/envoy.yaml:ro \ + -e POWERX_NATIVE_ENABLED=$POWERX_NATIVE_ENABLED \ + -e POWERX_HOST_UID=$POWERX_HOST_UID \ + -e POWERX_HOST_GID=$POWERX_HOST_GID \ + -e POWERX_COLLECTOR_REVISION=$POWERX_COLLECTOR_REVISION \ -e SLURM_JOB_ID=\$SLURM_JOB_ID \ -e NODE_RANK=\$SLURM_PROCID \ -e NUM_NODES=$NUM_NODES \ @@ -186,7 +223,7 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then export IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX export RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE - PYXIS_ENV_LIST="NUM_NODES,PREFILL_NODES,DECODE_NODES,ALL_IPS,PREFILL_LEADER_IP,DECODE_LEADER_IP" + PYXIS_ENV_LIST="POWERX_NATIVE_ENABLED,POWERX_HOST_UID,POWERX_HOST_GID,POWERX_COLLECTOR_REVISION,NUM_NODES,PREFILL_NODES,DECODE_NODES,ALL_IPS,PREFILL_LEADER_IP,DECODE_LEADER_IP" PYXIS_ENV_LIST+=",PREFILL_WORKERS,DECODE_WORKERS" PYXIS_ENV_LIST+=",PREFILL_DP_ADDR,DECODE_DP_ADDR,MODEL_NAME,GPUS_PER_NODE" PYXIS_ENV_LIST+=",PREFILL_DP_SIZE,DECODE_DP_SIZE" @@ -198,7 +235,7 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then PYXIS_ENV_LIST+=",RUNNER_TYPE,RESULT_FILENAME,SPEC_DECODING,IS_MULTINODE,CONFIG_FILE" PYXIS_MOUNTS="${MODEL_DIR}:/models:ro" - PYXIS_MOUNTS+=",${BENCHMARK_LOGS_DIR}:/benchmark_logs" + PYXIS_MOUNTS+=",${BENCHMARK_LOGS_DIR}:/benchmark_logs,${POWERX_RAW_ROOT}:/powerx_native" PYXIS_MOUNTS+=",${DI_REPO_DIR}:${DOCKER_MOUNT_PATH}" PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro" PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro" diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index b38940a85..d1fd8ee9e 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -114,17 +114,28 @@ else DP_ADDR="$DECODE_DP_ADDR" fi -# One coordinator publishes the benchmark status; workers leave normally when -# it finishes so Slurm can distinguish success from cancellation. +# Exactly one coordinator owns the benchmark and global shutdown, even when +# decode has several independent engine leaders. BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" BENCH_RC=0 +source /workspace/benchmarks/native_power_lifecycle.sh finish_llmd_node() { local rc=$? pid trap - EXIT + if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 && -n "${POWERX_COLLECTOR_PID:-}" ]]; then + if [[ "$NODE_RANK" == "$PREFILL_NODES" || "$rc" != 0 ]]; then + powerx_stop_collectors || rc=$? + else + powerx_reap_collector || rc=$? + fi + fi if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then - printf '%s\n' "$rc" > "$BENCH_DONE_MARKER.tmp" && - mv -f "$BENCH_DONE_MARKER.tmp" "$BENCH_DONE_MARKER" || rc=1 + printf '%s\n' "$rc" > "$BENCH_DONE_MARKER.tmp" || rc=1 + if [[ -n "${POWERX_HOST_UID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$BENCH_DONE_MARKER.tmp" || rc=1 + fi + mv -f "$BENCH_DONE_MARKER.tmp" "$BENCH_DONE_MARKER" || rc=1 fi for pid in "${ENVOY_PID:-}" "${EPP_PID:-}" "${SIDECAR_PID:-}" "${VLLM_PID:-}"; do [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true @@ -134,6 +145,13 @@ finish_llmd_node() { trap finish_llmd_node EXIT trap 'exit 143' TERM HUP trap 'exit 130' INT +if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then + _power_control="$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID" + export POWERX_NODE_NAME="$(sed -n '1p' "$_power_control/host-$NODE_RANK")" + export POWERX_CLOCK_SYNCHRONIZED="$(sed -n '2p' "$_power_control/host-$NODE_RANK")" + powerx_start_collector "/powerx_native/node-$NODE_RANK" "$_power_control" \ + nvidia "$NODE_RANK" "$ROLE" "$GPUS_PER_NODE" "$NUM_NODES" +fi DP_SIZE_LOCAL="$GPUS_PER_NODE" START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) @@ -568,6 +586,10 @@ PY done echo "All ${#_prefill_ips[@]} prefill vLLM endpoint(s) ready" + if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then + powerx_wait_collectors ready + fi + # ---- Benchmark sweep (one run per concurrency level) ---- # BENCH_MAX_CONCURRENCY is an 'x'-delimited list from submit.sh (e.g. "1024x512"). IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" @@ -646,6 +668,7 @@ PY ) fi + # EXIT drains every collector before signaling workers to stop serving. exit "$BENCH_RC" else while [[ ! -f "$BENCH_DONE_MARKER" ]]; do diff --git a/benchmarks/multi_node/llm-d/submit.sh b/benchmarks/multi_node/llm-d/submit.sh index 9d00125c1..54c11b563 100755 --- a/benchmarks/multi_node/llm-d/submit.sh +++ b/benchmarks/multi_node/llm-d/submit.sh @@ -70,6 +70,10 @@ export PREFILL_DP_SIZE=$(( PREFILL_NODES / PREFILL_WORKERS * GPUS_PER_NODE )) export DECODE_DP_SIZE=$(( DECODE_NODES / DECODE_WORKERS * GPUS_PER_NODE )) export BENCH_INPUT_LEN=$ISL export BENCH_OUTPUT_LEN=$OSL +export POWERX_NATIVE_ENABLED=0 +if [[ "${REQUIRE_POWER:-0}" =~ ^(1|true|TRUE|yes|YES)$ && "$ISL" == 8192 && "$OSL" == 1024 && "${IS_AGENTIC:-0}" != 1 && "${SCENARIO_TYPE:-}" != agentic-coding && "${EVAL_ONLY:-false}" != true ]]; then + export POWERX_NATIVE_ENABLED=1 +fi export BENCH_MAX_CONCURRENCY=$CONCURRENCIES export BENCH_REQUEST_RATE=$REQUEST_RATE export BENCH_RANDOM_RANGE_RATIO=$RANDOM_RANGE_RATIO diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index db0784ef1..ebbb7b9fe 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4438,6 +4438,7 @@ dsv4-fp4-gb200-llmd-vllm: fixed-seq-len: - isl: 8192 osl: 1024 + require-power: true search-space: # Low latency: 1 prefill DEP8 + 1 decode TP8. - spec-decoding: "none" diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 581e5d09a..9c2efceb8 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -114,6 +114,10 @@ The runner-name prefix is load-bearing: workflow routing uses `launch_${RUNNER_N 6. Verify every runner is **Idle** in [repository runner settings](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners) before adding it to sweep traffic. 7. Verify launcher mounts for `_work`, HF cache, staged weights, and squash images from a compute node. Root containers must not leave root-owned files in the shared workspace. +## Native llm-d power + +Only fixed 8192/1024 `dsv4-fp4-gb200-llmd-vllm` requires native power. Every serving node collects its selected devices; the coordinator waits for readiness and drain before normal shutdown. Results and audits are retained together. Other sequence lengths, AgentX and eval-only do not enable this collector. Hardware qualification and publication remain pending. + ## Register an srt-slurm recipe Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md). Checked-in recipes: [`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index aeff271a8..c8a8d9b46 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -114,6 +114,10 @@ runner 名称前缀是关键契约：workflow 通过 `launch_${RUNNER_NAME%%_*}. 6. 将 runner 加入 sweep 流量前，在[仓库 runner 设置页](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners)确认每个 runner 都是 **Idle**。 7. 从计算节点验证 launcher 对 `_work`、HF cache、预置权重和 squash 镜像的挂载。root 容器不得在共享 workspace 留下 root 所有的文件。 +## llm-d 原生功耗 + +仅固定 8192/1024 的 `dsv4-fp4-gb200-llmd-vllm` 要求原生功耗。每个服务节点采集所选设备；协调进程等待 ready 与 drain 后正常退出。结果与审计数据共同保留。其他序列长度、AgentX 和 eval-only 不启用此采集器。硬件资格验证与发布仍待完成。 + ## 注册 srt-slurm 配方 映射来源：[`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md)。检入的配方：[`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/)。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 834053d29..cbdab700a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7433,3 +7433,13 @@ - "Verify terminal Slurm receipts and exit llm-d workers normally after preserving available evidence." - "验证 Slurm 最终状态，并在保留已有证据后正常退出 llm-d 工作进程。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm + scenario-type: + - fixed-seq-len + description: + - Require native power only for llm-d 8k1k and preserve per-node collector readiness, + drain and completion evidence. + - 仅对 llm-d 8k1k 要求原生功耗，并保留每节点采集器 ready、drain 和完成证据。 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3027 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 4a695aa9f..c93cdba95 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -173,6 +173,8 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then copy_eval_artifacts "$EVAL_DIR" "$GITHUB_WORKSPACE" || exit 1 fi + mkdir -p "$GITHUB_WORKSPACE/LOGS" + cp -a "$BENCHMARK_LOGS_DIR/." "$GITHUB_WORKSPACE/LOGS/" || exit 1 scancel "$JOB_ID" 2>/dev/null || true exit "$SRT_JOB_RC" fi diff --git a/runners/test_llmd_power_lifecycle.py b/runners/test_llmd_power_lifecycle.py new file mode 100644 index 000000000..2daee562a --- /dev/null +++ b/runners/test_llmd_power_lifecycle.py @@ -0,0 +1,68 @@ +"""CPU-only lifecycle checks with controlled external collector/Slurm processes.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +LIFECYCLE = ROOT / 'benchmarks/native_power_lifecycle.sh' +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' + + +@pytest.mark.parametrize('main_rc', [0, 7]) +def test_llmd_job_stages_native_evidence_and_keeps_main_step_status(tmp_path, main_rc): + repo = tmp_path / 'repo' + cwd = repo / 'benchmarks/multi_node/llm-d' + cwd.mkdir(parents=True) + model, logs, bindir = tmp_path / 'model', tmp_path / 'logs', tmp_path / 'bin' + for path in (model, logs, bindir): + path.mkdir() + squash = tmp_path / 'image.sqsh' + squash.write_text('fixture image') + (bindir / 'scontrol').write_text('#!/bin/sh\nprintf "node-a\\nnode-b\\n"\n') + (bindir / 'git').write_text('#!/bin/sh\necho 0123456789012345678901234567890123456789\n') + (bindir / 'srun').write_text('#!' + sys.executable + '\n' + r''' +import json, os, pathlib, subprocess, sys +args = sys.argv[1:] +with open(os.environ['CALLS'], 'a') as f: + f.write(json.dumps(args) + '\n') +if any(a.startswith('--container-image=') for a in args): + for rank in range(2): + out = pathlib.Path(os.environ['POWERX_RAW_ROOT']) / f'node-{rank}' + out.mkdir(parents=True, exist_ok=True) + (out / 'manifest.json').write_text(json.dumps({'rank':rank,'synthetic':True})) + sys.exit(int(os.environ['MAIN_RC'])) +args = [a for a in args if not a.startswith('--')] +if 'ip route' in ' '.join(args): + print('127.0.0.1') + sys.exit(0) +for rank in range(2): + subprocess.run(args, env={**os.environ,'SLURM_PROCID':str(rank)}, check=True) +''') + for path in bindir.iterdir(): + path.chmod(0o755) + defaults = dict.fromkeys('PREFILL_WORKERS DECODE_WORKERS PREFILL_DP_SIZE DECODE_DP_SIZE ' + 'BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_MAX_CONCURRENCY ' + 'BENCH_REQUEST_RATE BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER ' + 'RUN_EVAL EVAL_ONLY EVAL_CONC EVAL_FRAMEWORK EVAL_LIMIT EVAL_SUITE ' + 'SWEBENCH_GEN_MODE SWEBENCH_USE_MODAL MODAL_TOKEN_ID MODAL_TOKEN_SECRET ' + 'IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX ' + 'RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE'.split(), '1') + env = {**os.environ, **defaults, 'PATH':str(bindir) + os.pathsep + os.environ['PATH'], + 'SLURM_JOB_ID':f'local-{os.getpid()}-{main_rc}', 'SLURM_JOB_NODELIST':'node-[a-b]', + 'NUM_NODES':'2','PREFILL_NODES':'1','DECODE_NODES':'1','GPUS_PER_NODE':'2', + 'MODEL_DIR':str(model),'MODEL_NAME':'fixture','BENCHMARK_LOGS_DIR':str(logs), + 'LLMD_CONTAINER_ENGINE':'pyxis','LLMD_SQUASH_FILE':str(squash), + 'POWERX_NATIVE_ENABLED':'1','POWERX_RAW_ROOT':str(tmp_path/'raw'), + 'MAIN_RC':str(main_rc),'CALLS':str(tmp_path/'calls.jsonl')} + result = subprocess.run(['bash', str(JOB)], cwd=cwd, env=env, + capture_output=True, text=True, timeout=20) + assert result.returncode == main_rc, result.stderr + result.stdout + for rank in range(2): + saved = repo / f'LOGS/native_power/node-{rank}/manifest.json' + assert json.loads(saved.read_text()) == {'rank':rank,'synthetic':True} + + From f691e7e7d72b49bd3db08ddbf89f414ddb8cb620 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:30:16 -0700 Subject: [PATCH 08/21] fix: retain per-window SMI context artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留每个测量窗口的 SMI 上下文工件，覆盖原 #3027 的共享上传规则。 --- .github/workflows/benchmark-tmpl.yml | 8 ++++---- utils/test_process_result.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 1da933f63..632401822 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -445,13 +445,13 @@ jobs: name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }} path: | gpu_metrics.csv - gpu_metrics_context.json + gpu_metrics*_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv results/gpu_metrics*.csv - results/gpu_metrics_context.json + results/gpu_metrics*_context.json results/gpu_metrics_identity.json if-no-files-found: ignore @@ -464,14 +464,14 @@ jobs: ${{ env.RESULT_FILENAME }}.json agg_${{ env.RESULT_FILENAME }}.json gpu_metrics.csv - gpu_metrics_context.json + gpu_metrics*_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json results/gpu_metrics*.csv - results/gpu_metrics_context.json + results/gpu_metrics*_context.json results/gpu_metrics_identity.json results/agentic_power_window.json results/agentic_power_timezone_offset.txt diff --git a/utils/test_process_result.py b/utils/test_process_result.py index d14d6a87a..7bc8dc4d9 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -1623,3 +1623,19 @@ def test_public_power_audit_bounds_text_and_device_identifiers(): assert audit['exporter_image_sha256'] == 'a' * 64 assert len(audit['observed_gpu_ids']) == 1024 assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] + + +@pytest.mark.parametrize('step_name', ['Upload GPU metrics', 'Upload power audit bundle']) +def test_workflow_retains_context_for_each_metrics_csv(tmp_path, step_name): + import yaml + + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == step_name) + patterns = step['with']['path'].splitlines() + for relative in ['gpu_metrics_concurrency_4_context.json', + 'results/gpu_metrics_concurrency_4_context.json']: + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('{"timestamp_timezone":"UTC"}') + assert any(path in tmp_path.glob(pattern.strip()) for pattern in patterns) From 3b2cb67e16def3cb7c732347e85db38952c9b0e7 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:35:39 -0700 Subject: [PATCH 09/21] fix: skip llm-d power preparation when disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 未启用功耗时跳过 llm-d 原生采集准备和挂载。 --- benchmarks/multi_node/llm-d/job.slurm | 18 ++++++++++++------ runners/test_llmd_power_lifecycle.py | 13 ++++++++++--- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm index 19598d868..eb3a42b84 100644 --- a/benchmarks/multi_node/llm-d/job.slurm +++ b/benchmarks/multi_node/llm-d/job.slurm @@ -68,11 +68,14 @@ DOCKER_MOUNT_PATH="/workspace" BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" rm -f "$BENCH_DONE_MARKER" POWERX_NATIVE_ENABLED="${POWERX_NATIVE_ENABLED:-0}" -export POWERX_HOST_UID="$(id -u)" POWERX_HOST_GID="$(id -g)" -export POWERX_COLLECTOR_REVISION="$(git -C "$DI_REPO_DIR" rev-parse HEAD)" -export POWERX_RAW_ROOT="${POWERX_RAW_ROOT:-/tmp/inferencex-native-$SLURM_JOB_ID}" -srun --nodes="$NUM_NODES" --ntasks-per-node=1 mkdir -p "$POWERX_RAW_ROOT" +export POWERX_HOST_UID="" POWERX_HOST_GID="" POWERX_COLLECTOR_REVISION="" +POWERX_DOCKER_MOUNT="" if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + export POWERX_HOST_UID="$(id -u)" POWERX_HOST_GID="$(id -g)" + export POWERX_COLLECTOR_REVISION="$(git -C "$DI_REPO_DIR" rev-parse HEAD)" + export POWERX_RAW_ROOT="${POWERX_RAW_ROOT:-/tmp/inferencex-native-$SLURM_JOB_ID}" + POWERX_DOCKER_MOUNT="-v ${POWERX_RAW_ROOT}:/powerx_native" + srun --nodes="$NUM_NODES" --ntasks-per-node=1 mkdir -p "$POWERX_RAW_ROOT" mkdir -p "$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID" "$DI_REPO_DIR/LOGS/native_power" chmod 777 "$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID" srun --nodes="$NUM_NODES" --ntasks-per-node=1 chmod 777 "$POWERX_RAW_ROOT" @@ -137,7 +140,7 @@ exec docker run --rm \ --privileged \ -v ${MODEL_DIR}:/models:ro \ -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ - -v ${POWERX_RAW_ROOT}:/powerx_native \ + $POWERX_DOCKER_MOUNT \ -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ -v ${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro \ -v ${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro \ @@ -235,7 +238,10 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then PYXIS_ENV_LIST+=",RUNNER_TYPE,RESULT_FILENAME,SPEC_DECODING,IS_MULTINODE,CONFIG_FILE" PYXIS_MOUNTS="${MODEL_DIR}:/models:ro" - PYXIS_MOUNTS+=",${BENCHMARK_LOGS_DIR}:/benchmark_logs,${POWERX_RAW_ROOT}:/powerx_native" + PYXIS_MOUNTS+=",${BENCHMARK_LOGS_DIR}:/benchmark_logs" + if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + PYXIS_MOUNTS+=",${POWERX_RAW_ROOT}:/powerx_native" + fi PYXIS_MOUNTS+=",${DI_REPO_DIR}:${DOCKER_MOUNT_PATH}" PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro" PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro" diff --git a/runners/test_llmd_power_lifecycle.py b/runners/test_llmd_power_lifecycle.py index deeb9709d..50c01ee94 100644 --- a/runners/test_llmd_power_lifecycle.py +++ b/runners/test_llmd_power_lifecycle.py @@ -8,12 +8,12 @@ import pytest ROOT = Path(__file__).resolve().parents[1] -LIFECYCLE = ROOT / 'benchmarks/native_power_lifecycle.sh' JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' @pytest.mark.parametrize('main_rc', [0, 7]) -def test_llmd_job_stages_native_evidence_and_keeps_main_step_status(tmp_path, main_rc): +@pytest.mark.parametrize('enabled', [False, True]) +def test_llmd_job_stages_native_evidence_and_keeps_main_step_status(tmp_path, main_rc, enabled): repo = tmp_path / 'repo' cwd = repo / 'benchmarks/multi_node/llm-d' cwd.mkdir(parents=True) @@ -30,6 +30,9 @@ def test_llmd_job_stages_native_evidence_and_keeps_main_step_status(tmp_path, ma with open(os.environ['CALLS'], 'a') as f: f.write(json.dumps(args) + '\n') if any(a.startswith('--container-image=') for a in args): + if os.environ['POWERX_NATIVE_ENABLED'] != '1': + assert not any('/powerx_native' in a for a in args) + sys.exit(int(os.environ['MAIN_RC'])) for rank in range(2): out = pathlib.Path(os.environ['POWERX_RAW_ROOT']) / f'node-{rank}' out.mkdir(parents=True, exist_ok=True) @@ -56,11 +59,15 @@ def test_llmd_job_stages_native_evidence_and_keeps_main_step_status(tmp_path, ma 'NUM_NODES':'2','PREFILL_NODES':'1','DECODE_NODES':'1','GPUS_PER_NODE':'2', 'MODEL_DIR':str(model),'MODEL_NAME':'fixture','BENCHMARK_LOGS_DIR':str(logs), 'LLMD_CONTAINER_ENGINE':'pyxis','LLMD_SQUASH_FILE':str(squash), - 'POWERX_NATIVE_ENABLED':'1','POWERX_RAW_ROOT':str(tmp_path/'raw'), + 'POWERX_NATIVE_ENABLED':'1' if enabled else '0','POWERX_RAW_ROOT':str(tmp_path/'raw'), 'MAIN_RC':str(main_rc),'CALLS':str(tmp_path/'calls.jsonl')} result = subprocess.run(['bash', str(JOB)], cwd=cwd, env=env, capture_output=True, text=True, timeout=20) assert result.returncode == main_rc, result.stderr + result.stdout + if not enabled: + assert not (repo / 'LOGS/native_power').exists() + assert not (tmp_path / 'raw').exists() + return for rank in range(2): saved = repo / f'LOGS/native_power/node-{rank}/manifest.json' assert json.loads(saved.read_text()) == {'rank':rank,'synthetic':True} From 86b86214e902a10293e51aa0cefe9b682aaac5f2 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:39:34 -0700 Subject: [PATCH 10/21] fix: parse Slurm allocation exit codes without derived status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按完整字段解析 Slurm 分配退出码，避免误用 DerivedExitCode。 --- runners/slurm_utils.sh | 2 +- runners/test_slurm_utils.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index a8fe5b0d7..85a2603f7 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -61,7 +61,7 @@ verify_slurm_job_completion() { # empties. Ask the controller before judging the final outcome. record=$(scontrol show job "$job_id" --oneliner 2>/dev/null) || record="" state=$(printf '%s\n' "$record" | sed -n 's/.*JobState=\([^ ]*\).*/\1/p') - exit_code=$(printf '%s\n' "$record" | sed -n 's/.*ExitCode=\([^ ]*\).*/\1/p') + exit_code=$(printf '%s\n' "$record" | tr ' ' '\n' | sed -n 's/^ExitCode=//p') ;; esac printf '%s\n' "$record" > "${GITHUB_WORKSPACE:-.}/slurm_job_${job_id}_outcome.txt" diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 64a89b8cb..7d52e1f66 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -688,3 +688,13 @@ def test_slurm_exit_before_log_retains_terminal_receipt(tmp_path): ) assert result.returncode == 1 assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == '42|FAILED|1:0' + + +@pytest.mark.parametrize('exit_code,derived,expected', [('0:0', '7:0', 0), ('1:0', '0:0', 1)]) +def test_slurm_controller_checks_allocation_not_derived_exit(tmp_path, exit_code, derived, expected): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; sacct() { return 1; }; ' + f'scontrol() {{ echo "JobId=42 JobState=COMPLETED ExitCode={exit_code} DerivedExitCode={derived}"; }}; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr From b230e7077dc24a58b4773457de911946b4a6bb2f Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:41:07 -0700 Subject: [PATCH 11/21] fix: preserve native role metrics and abort receipts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 独立生成 prefill 与 decode 角色指标，并在 AMD 采集器被回收前完成取消状态记录。 --- benchmarks/benchmark_lib.sh | 2 +- infx/results/power/native_multinode.py | 10 ++-- utils/test_native_multinode_power.py | 76 ++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 9d99a4cc9..ab6699c9f 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -225,7 +225,7 @@ stop_gpu_monitor() { # tick in the same second as the window end still fails bracketing — # the stream needs a tick at the NEXT whole second (measured on MI355X: # end=...153.325 vs last sample ...153.0). - if [[ "$GPU_MONITOR_VENDOR" == "amd" ]]; then + if [[ "$GPU_MONITOR_VENDOR" == "amd" && "${AMD_MONITOR_STOP_TIMEOUT_S:-}" != "0" ]]; then sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 )) fi kill "$GPU_MONITOR_PID" 2>/dev/null diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py index 1ffae5bd9..86b5d683b 100644 --- a/infx/results/power/native_multinode.py +++ b/infx/results/power/native_multinode.py @@ -221,13 +221,15 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, metrics = {} if not reasons and integration is not None and benchmark is not None: metrics = _derived_metrics(integration, benchmark) - if expected_decode_gpus: + if roles["prefill"]: prefill = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["prefill"]) - decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) - metrics.update(prefill_gpu_energy_j=prefill, decode_gpu_energy_j=decode, + metrics.update(prefill_gpu_energy_j=prefill, prefill_avg_power_w=prefill / benchmark.integration_duration_s / expected_prefill_gpus, + prefill_joules_per_input_token=prefill / benchmark.total_input_tokens) + if roles["decode"]: + decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) + metrics.update(decode_gpu_energy_j=decode, decode_avg_power_w=decode / benchmark.integration_duration_s / expected_decode_gpus, - prefill_joules_per_input_token=prefill / benchmark.total_input_tokens, decode_joules_per_output_token=decode / benchmark.total_output_tokens) valid = not reasons try: diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py index 0aeee2511..c2945cb75 100644 --- a/utils/test_native_multinode_power.py +++ b/utils/test_native_multinode_power.py @@ -199,3 +199,79 @@ def test_result_processor_discovers_staged_native_package(tmp_path, monkeypatch) invalid = json.loads(agg.read_text()) assert invalid["power_valid"] == 0 assert "total_gpu_energy_j" not in invalid + + +@pytest.mark.parametrize('role', ['prefill', 'decode']) +def test_native_single_role_preserves_whole_fleet_and_role_metrics(tmp_path, role): + root, bench, agg = _package(tmp_path) + for path in root.glob('*/manifest.json'): + manifest = json.loads(path.read_text()) + manifest['role'] = role + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=2 if role == 'prefill' else 0, + expected_decode_gpus=2 if role == 'decode' else 0, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual['total_gpu_energy_j'] == 800 + assert actual[f'{role}_gpu_energy_j'] == 800 + assert actual[f'{role}_avg_power_w'] == 200 + opposite = 'decode' if role == 'prefill' else 'prefill' + assert f'{opposite}_gpu_energy_j' not in actual + assert json.loads((tmp_path / 'power_validation_result.json').read_text())['power_valid'] + + +def test_native_amd_abort_skips_legacy_tail_wait(tmp_path): + result = subprocess.run(['bash', '-c', '''source "$1" +kill() { return 0; }; wait() { return 0; } +sleep() { echo unexpected-tail-wait >&2; } +_write_amd_smi_sidecar() { return 0; } +GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd AMD_MONITOR_STOP_TIMEOUT_S=0 +GPU_METRICS_CSV="$2/missing.csv" +stop_gpu_monitor +''', 'bash', str(REPO / 'benchmarks/benchmark_lib.sh'), str(tmp_path)], + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert 'unexpected-tail-wait' not in result.stderr + + +def test_native_amd_abort_publishes_receipt_before_reaper_deadline(tmp_path): + binary = tmp_path / 'bin' + binary.mkdir() + (binary / 'python3').symlink_to(sys.executable) + fake = binary / 'amd-smi' + fake.write_text(f'''#!{sys.executable} +import sys, time +if "-w" in sys.argv: + print("timestamp,gpu,socket_power", flush=True) + while True: + print(str(int(time.time())) + ",0,100", flush=True) + time.sleep(0.1) +else: + print("[]") +''') + fake.chmod(0o755) + control = tmp_path / 'control' + control.mkdir() + node = tmp_path / 'node-0' + process = subprocess.Popen(['bash', '-c', '\n'.join([ + 'source "$1"', + 'bash "$4" "$2" "$3" amd 0 aggregate 0 1 &', + 'POWERX_COLLECTOR_PID=$! POWERX_CONTROL_DIR=$3 POWERX_NUM_NODES=1', + 'POWERX_BARRIER_TIMEOUT_S=5', + 'powerx_wait_collectors ready || exit 1', + 'POWERX_BARRIER_TIMEOUT_S=0', + 'powerx_reap_collector', + ]), 'bash', str(REPO / 'benchmarks/native_power_lifecycle.sh'), str(node), str(control), str(REPO / 'benchmarks/native_power_collect.sh')], + env={**os.environ, 'PATH': f'{binary}:/usr/bin:/bin', 'SLURM_JOB_ID': 'test-job'}, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) + try: + stdout, stderr = process.communicate(timeout=8) + assert process.returncode == 143, (stdout, stderr) + assert (control / 'done-0').read_text().strip() == '143' + assert json.loads((node / 'manifest.json').read_text())['lifecycle'] == 'failed' + finally: + import signal + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() From 880f1153b489da0965261ededb090932d5a374dd Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:43:51 -0700 Subject: [PATCH 12/21] ci: run native collector contract regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在现有 CPU 工作流中执行原生采集契约回归，并跟踪对应文件变更。 --- .github/workflows/test-process-result.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 4d4cf0adf..7d5071770 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -10,6 +10,11 @@ on: - '.github/workflows/e2e-tests.yml' - '.github/workflows/test-process-result.yml' - 'benchmarks/benchmark_lib.sh' + - 'benchmarks/native_power_collect.sh' + - 'benchmarks/native_power_lifecycle.sh' + - 'runners/test_native_collector_barriers.py' + - 'runners/test_native_collector_receipts.py' + - 'utils/test_native_multinode_power.py' - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/**/*.yaml' - 'benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/**/*.yaml' - 'benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/**/*.yaml' @@ -57,4 +62,4 @@ jobs: run: | cd utils uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \ - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py -v From 20e5e71dfe1ca3b17a6ce95adbabe0f6fb93e7c1 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:43:52 -0700 Subject: [PATCH 13/21] ci: run llm-d lifecycle regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在现有 CPU 工作流中执行 llm-d 生命周期回归，并跟踪相关脚本变更。 --- .github/workflows/test-changelog-gate.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-changelog-gate.yml b/.github/workflows/test-changelog-gate.yml index 7d49c2ff1..d88effabe 100644 --- a/.github/workflows/test-changelog-gate.yml +++ b/.github/workflows/test-changelog-gate.yml @@ -31,6 +31,8 @@ on: - "runners/launch_*.sh" - "runners/slurm_utils.sh" - "runners/test_slurm_utils.py" + - "runners/test_llmd_lifecycle.py" + - "benchmarks/multi_node/llm-d/**" - "utils/ci_priority.py" - "utils/test_ci_priority.py" - ".github/workflows/reuse-sweep-comment.yml" @@ -123,4 +125,5 @@ jobs: utils/evals/test_batched_eval.py \ utils/evals/test_run_eval_dispatch.py \ runners/test_slurm_utils.py \ + runners/test_llmd_lifecycle.py \ -v -n 4 From 449d97067f24c93c6dd51f63ae9ad6bf6db28d59 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:45:28 -0700 Subject: [PATCH 14/21] ci: run fixed-sequence power regression tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将新增功耗回归测试与相关路径接入现有 CPU 工作流。 --- .github/workflows/test-process-result.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 4d4cf0adf..6ad144fb5 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -36,6 +36,8 @@ on: - 'utils/test_gb300_power_official_contract.py' - 'utils/test_inject_srt_power_concurrencies.py' - 'utils/test_process_result.py' + - 'benchmarks/multi_node/llm-d/**' + - 'runners/test_llmd_power_lifecycle.py' permissions: contents: read @@ -57,4 +59,4 @@ jobs: run: | cd utils uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \ - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py ../runners/test_llmd_power_lifecycle.py -v From 671b0c1e534787749ebc7e7c56070953a963b569 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:48:07 -0700 Subject: [PATCH 15/21] fix: honor accepted power requirement field names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 矩阵生成同时识别校验允许的字段名与别名，避免静默丢失功耗门槛。 --- infx/matrix/generate.py | 2 +- utils/matrix_logic/test_generate_sweep_configs.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/infx/matrix/generate.py b/infx/matrix/generate.py index ee6166045..abadad2ce 100644 --- a/infx/matrix/generate.py +++ b/infx/matrix/generate.py @@ -809,7 +809,7 @@ def _fixed_sequence_entries( is_multinode = config.get(Fields.MULTINODE.value, False) disagg = config.get(Fields.DISAGG.value, False) isl, osl = sequence[Fields.ISL.value], sequence[Fields.OSL.value] - require_power = sequence.get(Fields.REQUIRE_POWER.value, False) + require_power = sequence.get(Fields.REQUIRE_POWER.value, sequence.get("require_power", False)) if require_power and (isl, osl) != (8192, 1024): raise ValueError("require-power rollout supports only fixed-sequence 8192/1024") model_code = config[Fields.MODEL_PREFIX.value] diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index eeae0b83d..d88db4e9e 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -3244,7 +3244,8 @@ def test_empty_matrix_has_no_jobs(self, split_e2e_configs): @pytest.mark.parametrize("multinode", [False, True]) -def test_require_power_is_scoped_to_one_fixed_sequence(multinode, sample_single_node_config, +@pytest.mark.parametrize("power_key", ["require-power", "require_power"]) +def test_require_power_is_scoped_to_one_fixed_sequence(multinode, power_key, sample_single_node_config, sample_multinode_config, sample_runner_config): from infx.matrix.generate import expand_full_sweep, select_matrix_evals from infx.matrix.validation import MultiNodeSeqLenConfig, SingleNodeSeqLenConfig @@ -3257,7 +3258,7 @@ def test_require_power_is_scoped_to_one_fixed_sequence(multinode, sample_single_ sequences[-1]["isl"] = 8192 before = expand_full_sweep(config, sample_runner_config) assert all("require-power" not in row for row in before) - sequences[-1]["require-power"] = True + sequences[-1][power_key] = True schema = MultiNodeSeqLenConfig if multinode else SingleNodeSeqLenConfig schema.model_validate(sequences[-1]) after = expand_full_sweep(config, sample_runner_config) @@ -3267,6 +3268,6 @@ def test_require_power_is_scoped_to_one_fixed_sequence(multinode, sample_single_ evals = select_matrix_evals(copy.deepcopy(after), mode="subset") assert evals assert all("require-power" not in row for row in evals) - sequences[0]["require-power"] = True + sequences[0][power_key] = True with pytest.raises(ValueError, match="only fixed-sequence 8192/1024"): expand_full_sweep(config, sample_runner_config) From 654c33b2f71584f81a4c11a65e97a3dbd0fc41a9 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:48:30 -0700 Subject: [PATCH 16/21] test: synchronize shared replay signal readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将已验证的信号就绪屏障移至共享原生采集前置，避免重放 exec 前的竞态。 --- utils/agentic/aggregation/test_power_lifecycle.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/utils/agentic/aggregation/test_power_lifecycle.py b/utils/agentic/aggregation/test_power_lifecycle.py index 20710e68e..6d955af12 100644 --- a/utils/agentic/aggregation/test_power_lifecycle.py +++ b/utils/agentic/aggregation/test_power_lifecycle.py @@ -280,8 +280,13 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( }} stop_gpu_monitor() {{ printf 'monitor-stop\n' >> {str(event_log)!r}; }} fake_replay() {{ - printf 'replay-ready\n' >> {str(event_log)!r} - exec sleep 30 + exec {sys.executable!r} -c ' +import signal, sys, time +signal.signal(signal.SIGINT, signal.SIG_DFL) +signal.signal(signal.SIGTERM, signal.SIG_DFL) +print("replay-ready", file=open(sys.argv[1], "a"), flush=True) +time.sleep(30) +' {str(event_log)!r} }} trap 'printf "parent-exit\\n" >> {str(event_log)!r}' EXIT trap 'printf "parent-int\\n" >> {str(event_log)!r}; exit 130' INT @@ -301,7 +306,8 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( ) try: # The monitor starts before the production signal traps are installed. - # Wait for replay so the signal actually exercises those traps. + # Publish readiness from the execed process after restoring signal handling; + # a shell marker before exec races with the group SIGINT. deadline = time.monotonic() + 5 while time.monotonic() < deadline: if event_log.exists() and "replay-ready" in event_log.read_text().splitlines(): From fe3a0d49fb07d44015a21be39039078dd01e6bfd Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:48:30 -0700 Subject: [PATCH 17/21] test: synchronize shared replay signal readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将已验证的信号就绪屏障移至共享原生采集前置，避免重放 exec 前的竞态。 --- utils/agentic/aggregation/test_power_lifecycle.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/utils/agentic/aggregation/test_power_lifecycle.py b/utils/agentic/aggregation/test_power_lifecycle.py index 20710e68e..6d955af12 100644 --- a/utils/agentic/aggregation/test_power_lifecycle.py +++ b/utils/agentic/aggregation/test_power_lifecycle.py @@ -280,8 +280,13 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( }} stop_gpu_monitor() {{ printf 'monitor-stop\n' >> {str(event_log)!r}; }} fake_replay() {{ - printf 'replay-ready\n' >> {str(event_log)!r} - exec sleep 30 + exec {sys.executable!r} -c ' +import signal, sys, time +signal.signal(signal.SIGINT, signal.SIG_DFL) +signal.signal(signal.SIGTERM, signal.SIG_DFL) +print("replay-ready", file=open(sys.argv[1], "a"), flush=True) +time.sleep(30) +' {str(event_log)!r} }} trap 'printf "parent-exit\\n" >> {str(event_log)!r}' EXIT trap 'printf "parent-int\\n" >> {str(event_log)!r}; exit 130' INT @@ -301,7 +306,8 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( ) try: # The monitor starts before the production signal traps are installed. - # Wait for replay so the signal actually exercises those traps. + # Publish readiness from the execed process after restoring signal handling; + # a shell marker before exec races with the group SIGINT. deadline = time.monotonic() + 5 while time.monotonic() < deadline: if event_log.exists() and "replay-ready" in event_log.read_text().splitlines(): From d141e71c8be2bb6c57c37a2a623bfb6c274e5921 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:02:16 -0700 Subject: [PATCH 18/21] fix: preserve diagnostic sidecars and legacy result processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 只忽略已定义的诊断 sidecar，保留损坏与缺失点门禁；旧处理器逐点处理并累计失败，新版导入错误不回退。 --- .../workflows/benchmark-multinode-tmpl.yml | 42 ++++++++- infx/results/fixed_sequence.py | 9 +- utils/test_process_result.py | 87 +++++++++++++++++++ 3 files changed, 136 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 80b0842f8..d26d4e0bb 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -480,7 +480,47 @@ jobs: export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA" fi - python3 utils/process_result.py --all + # Older measured refs predate the batch CLI. Import failures in a + # present processor are errors, not a reason to downgrade validation. + has_batch=$(python3 - <<'PY' + import importlib + from pathlib import Path + if Path('infx/results/fixed_sequence.py').is_file(): + module = importlib.import_module('infx.results.fixed_sequence') + print(int(hasattr(module, 'process_multinode_results'))) + else: + print(0) + PY + ) + if [ "$has_batch" = 1 ]; then + python3 utils/process_result.py --all + else + batch_rc=0 + observed=() + for result_file in "${RESULT_FILENAME}"_*.json; do + [ -f "$result_file" ] || continue + case "$result_file" in + *.pytorch.json|"${RESULT_FILENAME}_gpu_metrics_context.json"|"${RESULT_FILENAME}_gpu_metrics_identity.json") continue ;; + esac + if [[ "$result_file" =~ _(c|conc|concurrency_)([0-9]+)(_req_rate_[^_]+)?_gpus_([0-9]+)(_ctx_([0-9]+)_gen_([0-9]+))?\.json$ ]]; then + observed+=("${BASH_REMATCH[2]}") + gpus=${BASH_REMATCH[4]} + prefill_gpus=${BASH_REMATCH[6]:-$gpus} + decode_gpus=${BASH_REMATCH[7]:-0} + RESULT_FILENAME=${result_file%.json} IS_MULTINODE=true PREFILL_GPUS="$prefill_gpus" DECODE_GPUS="$decode_gpus" \ + python3 utils/process_result.py || batch_rc=1 + else + echo "Unrecognized result filename: $result_file" >&2 + batch_rc=1 + fi + done + read -r -a expected <<< "$CONC_LIST" + if [[ "$(printf '%s\n' "${observed[@]}" | sort -n)" != "$(printf '%s\n' "${expected[@]}" | sort -n)" ]]; then + echo 'Legacy result batch has missing, duplicate or unexpected concurrencies' >&2 + batch_rc=1 + fi + exit "$batch_rc" + fi - name: Upload result if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index f6fa76102..3db53a3b7 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -318,9 +318,16 @@ def process_multinode_results(env: Mapping[str, str]) -> int: if not expected or min(expected) <= 0: raise ValueError('CONC_LIST must contain positive concurrencies') points: list[dict[str, Any]] = [] + ignored_sidecars: list[str] = [] observed: set[int] = set() status = 0 for path in sorted(Path('.').glob(f"{env['RESULT_FILENAME']}_*.json")): + if path.name.endswith('.pytorch.json') or path.name in { + f"{env['RESULT_FILENAME']}_gpu_metrics_context.json", + f"{env['RESULT_FILENAME']}_gpu_metrics_identity.json", + }: + ignored_sidecars.append(path.name) + continue point: dict[str, Any] = {'source': path.name} try: match = re.search( @@ -364,7 +371,7 @@ def process_multinode_results(env: Mapping[str, str]) -> int: status = max(status, int(bool(missing or unexpected))) summary = {'expected_concurrencies': sorted(expected), 'missing_concurrencies': missing, 'unexpected_concurrencies': unexpected, - 'points': points, 'exit_code': status} + 'points': points, 'ignored_sidecars': ignored_sidecars, 'exit_code': status} Path(f"result_processing_{env['RESULT_FILENAME']}.json").write_text(json.dumps(summary, indent=2)) if missing or unexpected: print(f'[process_result] incomplete sweep: missing={missing}, unexpected={unexpected}', file=sys.stderr) diff --git a/utils/test_process_result.py b/utils/test_process_result.py index d14d6a87a..2b394e3bb 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -1623,3 +1623,90 @@ def test_public_power_audit_bounds_text_and_device_identifiers(): assert audit['exporter_image_sha256'] == 'a' * 64 assert len(audit['observed_gpu_ids']) == 1024 assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] + + +@pytest.mark.parametrize('sidecar', ['run_recipe_conc4_gpus_4_ctx_2_gen_2.pytorch.json', + 'run_gpu_metrics_context.json', 'run_gpu_metrics_identity.json']) +@pytest.mark.parametrize('point_state', ['valid', 'missing', 'malformed']) +def test_multinode_batch_retains_sidecars_without_counting_them_as_points( + tmp_path, multinode_env_vars, sample_benchmark_result, sidecar, point_state, +): + (tmp_path / sidecar).write_text('{"diagnostic": true}') + if point_state != 'missing': + (tmp_path / 'run_recipe_conc4_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': 4}) + if point_state == 'valid' else '{broken') + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4', + 'PYTHONPATH': str(REPO_ROOT)} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, env=env, + capture_output=True, text=True) + assert result.returncode == int(point_state != 'valid'), result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['ignored_sidecars'] == [sidecar] + assert receipt['missing_concurrencies'] == ([] if point_state == 'valid' else [4]) + assert (tmp_path / sidecar).read_text() == '{"diagnostic": true}' + + +def test_multinode_batch_rejects_unknown_point_filename( + tmp_path, multinode_env_vars, sample_benchmark_result, +): + for name in ['run_conc4_gpus_4_ctx_2_gen_2.json', 'run_conc16_gpus_bad.json']: + (tmp_path / name).write_text(json.dumps({**sample_benchmark_result, 'max_concurrency': 4})) + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4 16', 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1 + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [16] + assert any('filename lacks' in point.get('error', '') for point in receipt['points']) + + +@pytest.mark.parametrize('first_rc', [0, 7]) +@pytest.mark.parametrize('expected_concs', ['4 16', '4 8 16', '4']) +def test_multinode_workflow_processes_every_point_with_legacy_processor(tmp_path, first_rc, expected_concs): + (tmp_path / 'utils').mkdir() + (tmp_path / 'utils/process_result.py').write_text('''import json, os, sys +from pathlib import Path +stem = os.environ['RESULT_FILENAME'] +data = json.loads(Path(stem + '.json').read_text()) +Path('agg_' + stem + '.json').write_text(json.dumps(data)) +sys.exit(data['exit_code']) +''') + for conc, rc in [(4, first_rc), (16, 0)]: + (tmp_path / f'run_conc{conc}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({'exit_code': rc})) + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == 'Process result') + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step['run']], cwd=tmp_path, + env={**os.environ, 'RESULT_FILENAME': 'run', 'POWER_PRODUCER_SHA': '', + 'CONC_LIST': expected_concs, 'PYTHONPATH': '', + 'PATH': f"{Path(sys.executable).parent}:{os.environ['PATH']}"}, + capture_output=True, text=True, timeout=10) + assert result.returncode == int(bool(first_rc) or expected_concs != '4 16'), result.stderr + for conc in [4, 16]: + assert (tmp_path / f'agg_run_conc{conc}_gpus_4_ctx_2_gen_2.json').is_file() + + +def test_multinode_workflow_does_not_downgrade_processor_import_errors(tmp_path): + (tmp_path / 'infx/results').mkdir(parents=True) + (tmp_path / 'infx/__init__.py').touch() + (tmp_path / 'infx/results/__init__.py').touch() + (tmp_path / 'infx/results/fixed_sequence.py').write_text( + 'raise RuntimeError("broken processor import")\n') + (tmp_path / 'utils').mkdir() + (tmp_path / 'utils/process_result.py').write_text( + 'from pathlib import Path\nPath("legacy-called").touch()\n') + (tmp_path / 'run_conc4_gpus_4.json').write_text('{}') + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == 'Process result') + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step['run']], cwd=tmp_path, + env={**os.environ, 'RESULT_FILENAME': 'run', 'POWER_PRODUCER_SHA': '', + 'CONC_LIST': '4', 'PYTHONPATH': '', + 'PATH': f"{Path(sys.executable).parent}:{os.environ['PATH']}"}, + capture_output=True, text=True, timeout=10) + assert result.returncode != 0 + assert 'broken processor import' in result.stderr + assert not (tmp_path / 'legacy-called').exists() From 8408779c22817e681238ac323f7b4c4ad34977b0 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:02:55 -0700 Subject: [PATCH 19/21] test: preserve existing result test section formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 恢复合并时意外改动的测试注释分隔线与空行。 --- utils/test_process_result.py | 42 ++++++++++++------------------------ 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 350856ea7..9d3ba8ddf 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -155,11 +155,9 @@ def test_fixed_topology_rejects_empty_parallelism( build_result(sample_benchmark_result, {**single_node_env_vars, name: ""}) -# ====================================================================== - +# ============================================================================= # Test Fixtures - Based on real benchmark output structure -# ====================================================================== - +# ============================================================================= @pytest.fixture def sample_benchmark_result(): @@ -298,11 +296,9 @@ def failing_import(name, *args, **kwargs): ) -# ====================================================================== - +# ============================================================================= # Test script execution via subprocess -# ====================================================================== - +# ============================================================================= class TestProcessResultScript: """Tests for process_result.py script execution.""" @@ -532,11 +528,9 @@ def test_missing_result_file(self, tmp_path, single_node_env_vars): assert result.returncode != 0 -# ====================================================================== - +# ============================================================================= # Test latency and throughput calculations -# ====================================================================== - +# ============================================================================= class TestCalculations: """Tests for throughput and latency calculations.""" @@ -665,11 +659,9 @@ def test_multinode_zero_total_gpus_fails(self, tmp_path, sample_benchmark_result assert "Multinode results require at least one GPU" in result.stderr -# ====================================================================== - +# ============================================================================= # Test output file generation -# ====================================================================== - +# ============================================================================= class TestOutputFile: """Tests for output file generation.""" @@ -698,11 +690,9 @@ def test_output_file_has_correct_prefix(self, tmp_path, sample_benchmark_result, assert output_file.exists() -# ====================================================================== - +# ============================================================================= # Test edge cases -# ====================================================================== - +# ============================================================================= class TestEdgeCases: """Tests for edge cases and special scenarios.""" @@ -751,11 +741,9 @@ def test_integer_conversion(self, tmp_path, single_node_env_vars): assert isinstance(output_data["isl"], int) assert isinstance(output_data["osl"], int) -# ====================================================================== - +# ============================================================================= # Integration: power aggregation patches the agg JSON -# ====================================================================== - +# ============================================================================= class TestPowerAggregationIntegration: """End-to-end wiring: process_result.py invokes aggregate_power.py and @@ -1335,11 +1323,9 @@ def test_start_stop_gpu_monitor_amd_lifecycle(self, tmp_path): assert identity == {"gpu_data": []} -# ====================================================================== - +# ============================================================================= # Integration: multinode power aggregation patches the agg JSON -# ====================================================================== - +# ============================================================================= class TestMultinodePower: From 6ed8f8b5ed69e71fdd6972ceaeaa8d97a160f347 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:05:34 -0700 Subject: [PATCH 20/21] fix: validate native role counts and retain boundary sample audits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 独立核对每种角色的 GPU 数量，并记录窗口外异常样本，窗口内异常仍阻止功耗有效性。 --- infx/results/power/native_multinode.py | 7 +++--- utils/test_native_multinode_power.py | 30 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py index 86b5d683b..78ff89889 100644 --- a/infx/results/power/native_multinode.py +++ b/infx/results/power/native_multinode.py @@ -202,10 +202,8 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, if expected_aggregate_gpus: if roles["prefill"] or roles["decode"] or len(roles["aggregate"]) != expected_aggregate_gpus: reasons.append("native_role_gpu_count_mismatch") - elif expected_decode_gpus: - if roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or len(roles["decode"]) != expected_decode_gpus: - reasons.append("native_role_gpu_count_mismatch") - elif roles["decode"] or len(roles["aggregate"] + roles["prefill"]) != expected_prefill_gpus: + elif (roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or + len(roles["decode"]) != expected_decode_gpus): reasons.append("native_role_gpu_count_mismatch") combined_path = validation_result.with_name(f"{validation_result.stem}_native.csv") combined_path.parent.mkdir(parents=True, exist_ok=True) @@ -247,6 +245,7 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, "observed_gpu_count": integration.observed_num_gpus if integration else 0, "per_gpu_role": {uuid: role for role, uuids in roles.items() for uuid in uuids}, "per_gpu_sample_counts": integration.per_gpu_sample_counts if integration else {}, + "boundary_degenerate_rows": integration.boundary_degenerate_rows if integration else {}, "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s if integration else {}, "producer": {"name": "inferencex-native-smi", "revisions": sorted(revisions), "producer_git_commit": next(iter(revisions)) if len(revisions) == 1 else None}, diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py index c2945cb75..e4234facc 100644 --- a/utils/test_native_multinode_power.py +++ b/utils/test_native_multinode_power.py @@ -275,3 +275,33 @@ def test_native_amd_abort_publishes_receipt_before_reaper_deadline(tmp_path): except ProcessLookupError: pass process.communicate() + + +def test_native_prefill_only_rejects_aggregate_role_devices(tmp_path): + root, bench, agg = _package(tmp_path) + manifest_path = root / 'node-1/manifest.json' + manifest = json.loads(manifest_path.read_text()) + manifest['role'] = 'aggregate' + manifest_path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=2, expected_decode_gpus=0, + require_power=True) == 1 + audit = json.loads((tmp_path / 'power_validation_result.json').read_text()) + assert 'native_role_gpu_count_mismatch' in audit['reasons'] + assert json.loads(agg.read_text())['power_valid'] == 0 + assert 'prefill_avg_power_w' not in json.loads(agg.read_text()) + + +@pytest.mark.parametrize('tick', [2, 4]) +def test_native_audit_retains_boundary_noise_without_relaxing_in_window_errors(tmp_path, tick): + root, bench, agg = _package(tmp_path) + for rank in [0, 1]: + with (root / f'node-{rank}/gpu_metrics.csv').open('a') as stream: + stream.write(f'{tick},0,N/A\n') + if rank == 0: + stream.write(f'{tick},0,0\n') + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, + require_power=True) == int(tick == 2) + audit = json.loads((tmp_path / 'power_validation_result.json').read_text()) + assert audit['boundary_degenerate_rows'] == ({'uuid-0': 2, 'uuid-1': 1} if tick == 4 else {}) + if tick == 4: + assert json.loads(agg.read_text())['total_gpu_energy_j'] == 800 From 85384dd3ba00223513ec5ef65a2f27b422b8c7ae Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:32:14 -0700 Subject: [PATCH 21/21] fix: validate native collector readiness and identity completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 发布就绪收据前检查采集进程仍存活，并将 AMD 结束身份采集失败写入失败收据。补充真实进程与清单回归测试。 --- benchmarks/native_power_collect.sh | 3 +- runners/test_native_collector_receipts.py | 77 +++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/benchmarks/native_power_collect.sh b/benchmarks/native_power_collect.sh index 69df07951..445d00ac3 100644 --- a/benchmarks/native_power_collect.sh +++ b/benchmarks/native_power_collect.sh @@ -35,7 +35,7 @@ finish() { if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi stop_gpu_monitor if [[ "$vendor" == amd ]]; then - _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json + amd-smi list --json > "$power_dir/gpu_metrics_devices_end.json" || collector_rc=1 else nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \ > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1 @@ -65,6 +65,7 @@ start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 if [[ "$vendor" == amd ]]; then _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json fi +_background_process_is_running "$GPU_MONITOR_PID" || exit 1 write_control "ready-$rank" ready while [[ ! -f "$control_dir/stop" ]]; do _background_process_is_running "$GPU_MONITOR_PID" || exit 1 diff --git a/runners/test_native_collector_receipts.py b/runners/test_native_collector_receipts.py index b981f57ce..63d9de677 100644 --- a/runners/test_native_collector_receipts.py +++ b/runners/test_native_collector_receipts.py @@ -14,3 +14,80 @@ def test_native_control_receipt_is_owned_before_publication(tmp_path): assert result.returncode == 0, result.stderr assert (tmp_path / 'done-0').read_text() == '7\n' assert not list(tmp_path.glob('*.tmp')) + + +def _collector_with_monitor(tmp_path, *, alive, end_identity_rc=0): + import shutil + import sys + + scripts = tmp_path / 'repo/benchmarks' + scripts.mkdir(parents=True) + shutil.copyfile(ROOT / 'benchmarks/native_power_collect.sh', scripts / 'native_power_collect.sh') + library = (ROOT / 'benchmarks/benchmark_lib.sh').read_text() + functions = [] + for name in ['_background_process_is_running', '_write_amd_smi_sidecar']: + start = library.index(name + '() {') + functions.append(library[start:library.index('\n}', start) + 2]) + (scripts / 'benchmark_lib.sh').write_text('\n'.join(functions) + ''' +GPU_MONITOR_PID="" GPU_MONITOR_VENDOR="" +start_gpu_monitor() { + GPU_MONITOR_VENDOR=amd + if [[ "$MONITOR_ALIVE" == 1 ]]; then + sleep 30 & + GPU_MONITOR_PID=$! + else + false & + GPU_MONITOR_PID=$! + wait "$GPU_MONITOR_PID" || true + fi +} +stop_gpu_monitor() { + kill "$GPU_MONITOR_PID" 2>/dev/null || true + wait "$GPU_MONITOR_PID" 2>/dev/null || true +} +amd-smi() { + if [[ -f "$IDENTITY_CALLED" && "$END_IDENTITY_RC" != 0 ]]; then return "$END_IDENTITY_RC"; fi + touch "$IDENTITY_CALLED" + printf '[{"gpu":0,"uuid":"test-gpu"}]\\n' +} +''') + control = tmp_path / 'control' + control.mkdir() + (control / 'stop').touch() + power = tmp_path / 'power' + result = subprocess.run(['bash', str(scripts / 'native_power_collect.sh'), str(power), + str(control), 'amd', '0', 'prefill', '0', '1'], + env={'PATH': f'{Path(sys.executable).parent}:/usr/bin:/bin', + 'PYTHONPATH': str(ROOT), 'MONITOR_ALIVE': str(int(alive)), + 'END_IDENTITY_RC': str(end_identity_rc), + 'IDENTITY_CALLED': str(tmp_path / 'identity-called')}, + capture_output=True, text=True, timeout=5) + return result, control, power + + +def test_dead_monitor_never_publishes_ready(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=False) + assert not (control / 'ready-0').exists(), result.stderr + assert (control / 'done-0').read_text().strip() != '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'failed' + + +def test_amd_end_identity_failure_is_not_a_successful_done_receipt(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=True, end_identity_rc=7) + assert (control / 'ready-0').exists(), result.stderr + assert (control / 'done-0').read_text().strip() != '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'failed' + + +def test_live_monitor_with_successful_identity_completes(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=True) + assert result.returncode == 0, result.stderr + assert (control / 'ready-0').exists() + assert (control / 'done-0').read_text().strip() == '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'complete'