From c3040728c629feac18cdd26aca7cdbc246b03b73 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:23:08 -0700 Subject: [PATCH 01/17] fix: preserve complete request outcomes and diagnostic batches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留完整请求结果状态与诊断批次，拆分 Slurm 生命周期和原生功耗采集。 --- .../workflows/benchmark-multinode-tmpl.yml | 26 +-- .github/workflows/benchmark-tmpl.yml | 4 +- docs/results-and-ingestion.md | 12 +- docs/results-and-ingestion_zh.md | 12 +- infx/bench_serving/benchmark_outcome.py | 31 +++ infx/bench_serving/benchmark_serving.py | 17 +- infx/results/fixed_sequence.py | 109 ++++++++- infx/results/power/__init__.py | 1 + infx/results/power/audit.py | 50 ++++ perf-changelog.yaml | 11 + utils/test_process_result.py | 217 +++++++++++++++++- 11 files changed, 444 insertions(+), 46 deletions(-) create mode 100644 infx/bench_serving/benchmark_outcome.py create mode 100644 infx/results/power/audit.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index bae33211b..80b0842f8 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -470,7 +470,7 @@ jobs: fi - name: Process result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ always() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} env: RUNNER_TYPE: ${{ inputs.runner }} run: | @@ -480,29 +480,10 @@ jobs: export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA" fi - # Process each result file - for result_file in ${RESULT_FILENAME}_*.json; do - if [ -f "$result_file" ]; then - echo "Processing $result_file" - # Extract GPU count, prefill_gpus and decode_gpus from filename for tp_size calculation - gpus=$(echo "$result_file" | sed -n "s/.*_gpus_\([0-9][0-9]*\).*\.json/\1/p") - prefill_gpus=$(echo "$result_file" | sed -n "s/.*_ctx_\([0-9]*\).*\.json/\1/p") - decode_gpus=$(echo "$result_file" | sed -n "s/.*_gen_\([0-9]*\).*\.json/\1/p") - - if [ -n "$gpus" ]; then - # Non-disagg results lack _ctx_/_gen_ fields; all GPUs belong to the single worker - if [ -z "$prefill_gpus" ] || [ -z "$decode_gpus" ]; then - prefill_gpus=$gpus - decode_gpus=0 - fi - echo "Extracted: gpus=$gpus, prefill_gpus=$prefill_gpus, decode_gpus=$decode_gpus" - RESULT_FILENAME=${result_file%.json} IS_MULTINODE=true PREFILL_GPUS="$prefill_gpus" DECODE_GPUS="$decode_gpus" python3 utils/process_result.py - fi - fi - done + python3 utils/process_result.py --all - name: Upload result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bmk_${{ env.RESULT_FILENAME }} @@ -518,6 +499,7 @@ jobs: agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json LOGS/power/** + result_processing_${{ env.RESULT_FILENAME }}.json LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json LOGS/agentic/**/agentic_power_timezone_offset.txt diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index b08ef0775..4d1b817ea 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -395,14 +395,14 @@ jobs: fi - name: Process result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ always() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} env: RUNNER_TYPE: ${{ inputs.runner }} run: | python3 utils/process_result.py - name: Upload result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bmk_${{ env.RESULT_FILENAME }} diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index daba2715b..a1e357b30 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -92,10 +92,20 @@ The fixed-sequence transformer requires runner, framework, precision, speculativ | Latency and interactivity | Each benchmark input key ending in `ms` is converted from milliseconds to seconds with `_ms` removed. Keys containing `tpot` also produce an `intvty` reciprocal. | | Optional runtime metadata | `router` as exactly `{name, version}`, `kv_p2p_transfer`, and measured power patched from `gpu_metrics.csv` when available | -Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physical GPU count. Multinode per-GPU denominators use the declared prefill and decode GPU counts. Invalid or missing required metadata fails transformation. Power aggregation is explicitly best effort and cannot fail the benchmark aggregate. +Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physical GPU count. Multinode per-GPU denominators use the declared prefill and decode GPU counts. Invalid or missing required metadata fails transformation. Power aggregation is best effort by default; `REQUIRE_POWER=1` fails the job after preserving available results and audits when power validation fails. InferenceX-app treats routing fields as columns or config dimensions and stores numeric measurements in `benchmark_results.metrics` JSONB. The mapper supports v1 shared topology, v2 split prefill/decode topology, and nested v3 AgentX metrics. Unknown numeric metrics are retained and warned about, which permits schema growth without silently losing numeric data. +### Fixed-sequence outcomes and PowerX audits + +The serving client records `benchmark_outcome` before saving its raw result. It retains the existing maximum request-failure rate of 5%, including the requested/completed/failed counts. The processor verifies this record, copies it to the aggregate, and returns failure even when telemetry is valid. Zero successful requests retain a diagnostic aggregate without fabricated reciprocal latency. Legacy results without outcome metadata remain distinguishable; power validity alone never establishes benchmark success or answer quality. + +`power_invalid_reasons` and `power_audit` carry a bounded summary alongside numeric metrics. The summary includes the available measurement window, expected/observed GPU counts, sampling diagnostics, observed device identifiers and producer pin. Its `source` names the retained `power_validation_*.json` sidecar. Device identifiers retain the collector's semantics; local SMI indices are not physical UUID proof. + +For multinode fixed-sequence jobs, `utils/process_result.py --all` processes every available result before returning failure. It accepts `_c_gpus_...`, `_conc_gpus_...`, and AMD `_concurrency__req_rate__gpus_...` filenames, including `inf` request rates. It compares result concurrencies with `CONC_LIST`, rejects duplicate or contradictory point identities, and records omissions/errors in `result_processing_.json`. Aggregate workers pass `AGGREGATE_GPUS` with zero role GPU counts to telemetry validation; separate prefill/decode energy remains absent. Existing layout fields remain compatible, and `num_aggregate_gpu` marks the shared allocation. + +Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index cc5c9e302..6a054d5f0 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -92,10 +92,20 @@ shape: array of benchmark row objects | 延迟和交互性 | 基准输入中每个以 `ms` 结尾的键都会从毫秒换算为秒，并移除 `_ms`。包含 `tpot` 的键还会产生其倒数 `intvty`。 | | 可选运行时元数据 | 形式必须精确为 `{name, version}` 的 `router`、`kv_p2p_transfer`，以及在可用时由 `gpu_metrics.csv` 补入的实测功耗 | -单节点 GPU 数为 `tp * pp * pcp_size`。DCP 不会增加物理 GPU 数。多节点每 GPU 指标的分母使用声明的 prefill 和 decode GPU 数。无效或缺失的必需元数据会使转换失败。功耗聚合明确采用尽力而为模式，不能导致基准聚合失败。 +单节点 GPU 数为 `tp * pp * pcp_size`。DCP 不会增加物理 GPU 数。多节点每 GPU 指标的分母使用声明的 prefill 和 decode GPU 数。无效或缺失的必需元数据会使转换失败。功耗聚合默认尽力而为；当设置 `REQUIRE_POWER=1` 时，功耗验证失败会在保留已有结果和审计后使任务失败。 InferenceX-app 将路由字段作为列或配置维度，并把数值测量存入 `benchmark_results.metrics` JSONB。映射器支持共享拓扑的 v1、拆分 prefill/decode 拓扑的 v2，以及嵌套 AgentX 指标的 v3。未知数值指标会被保留并产生警告，因此架构可以扩展，同时不会无提示地丢失数值数据。 +### 固定序列基准结果状态与 PowerX 审计 + +服务客户端在保存原始结果前写入 `benchmark_outcome`，保留现有的 5% 最大请求失败率，以及请求总数、完成数和失败数。处理器检查该记录并复制到聚合结果中；即使遥测有效，请求失败率超限仍返回失败。零成功请求会保留诊断聚合结果，但不会生成不存在的延迟倒数。没有状态元数据的历史结果仍可区分；功耗有效不能证明基准成功或答案质量。 + +`power_invalid_reasons` 和 `power_audit` 在数值指标旁携带有界摘要，包括可用的测量窗口、预期与观测 GPU 数、采样诊断、观测设备标识和生产者版本。`source` 指向保留的 `power_validation_*.json` 工件名称。设备标识保留采集器原有语义，本地 SMI 序号不是物理 UUID 的证明。 + +对于多节点固定序列任务，`utils/process_result.py --all` 先处理所有已有结果，再返回失败。它接受 `_c_gpus_...`、`_conc_gpus_...` 和 AMD 的 `_concurrency__req_rate__gpus_...` 文件名，也支持 `inf` 请求速率。它将结果并发度与 `CONC_LIST` 比较，拒绝重复或矛盾的点身份，并将遗漏和错误记录到 `result_processing_.json`。共享工作池通过 `AGGREGATE_GPUS` 及零值角色 GPU 数进行遥测验证；独立的 prefill/decode 能耗保持缺失。现有布局字段保持兼容，`num_aggregate_gpu` 明确标记共享分配。 + +启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 + ## 评测工件 ### 单配置身份和收集 diff --git a/infx/bench_serving/benchmark_outcome.py b/infx/bench_serving/benchmark_outcome.py new file mode 100644 index 000000000..e14dca470 --- /dev/null +++ b/infx/bench_serving/benchmark_outcome.py @@ -0,0 +1,31 @@ +"""The serving client's existing request-failure gate, preserved with results.""" + +from __future__ import annotations + +from typing import Literal, TypedDict + + +MAX_FAILURE_RATE = 0.05 + + +class BenchmarkOutcome(TypedDict): + status: Literal["passed", "failed"] + requested: int + completed: int + failed: int + max_failure_rate: float + + +def benchmark_outcome(requested: int, completed: int) -> BenchmarkOutcome: + """Record the same five-percent gate before cleanup or artifact upload.""" + if (type(requested) is not int or type(completed) is not int + or requested <= 0 or not 0 <= completed <= requested): + raise ValueError("Benchmark request counts must satisfy 0 <= completed <= requested > 0") + failed = requested - completed + return { + "status": "failed" if failed / requested > MAX_FAILURE_RATE else "passed", + "requested": requested, + "completed": completed, + "failed": failed, + "max_failure_rate": MAX_FAILURE_RATE, + } diff --git a/infx/bench_serving/benchmark_serving.py b/infx/bench_serving/benchmark_serving.py index 82dfec973..1619d28ae 100644 --- a/infx/bench_serving/benchmark_serving.py +++ b/infx/bench_serving/benchmark_serving.py @@ -62,6 +62,7 @@ from argparse import ArgumentParser as FlexibleArgumentParser from .benchmark_utils import convert_to_pytorch_benchmark_format +from .benchmark_outcome import benchmark_outcome from .encoding_dsv4 import encode_messages as dsv4_encode_messages MILLISECONDS_TO_SECONDS_CONVERSION = 1000 @@ -898,6 +899,11 @@ def main(args: argparse.Namespace): lora_modules=args.lora_modules, )) + # Preserve the request gate before writing results: cleanup or a surrounding + # launcher must not turn an error-containing benchmark into a passed point. + outcome = benchmark_outcome(args.num_prompts, benchmark_result["completed"]) + benchmark_result["benchmark_outcome"] = outcome + # Save config and results to json if args.save_result: result_json: Dict[str, Any] = {} @@ -957,14 +963,11 @@ def main(args: argparse.Namespace): json.dump(result_json, outfile) save_to_pytorch_benchmark_format(args, result_json, file_name) - max_failure_rate = 0.05 - completed = benchmark_result["completed"] - failure_rate = 1 - completed / args.num_prompts - if failure_rate > max_failure_rate: + if outcome["status"] == "failed": raise SystemExit( - f"FAIL: request failure rate {failure_rate:.1%} exceeds " - f"{max_failure_rate:.0%} threshold " - f"({completed}/{args.num_prompts} completed)" + f"FAIL: request failure rate {outcome['failed'] / outcome['requested']:.1%} exceeds " + f"{outcome['max_failure_rate']:.0%} threshold " + f"({outcome['completed']}/{outcome['requested']} completed)" ) diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index ef6ad2b8e..f6fa76102 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -3,7 +3,9 @@ from __future__ import annotations import json +import math import os +import re import sys from collections.abc import Iterable, Mapping from pathlib import Path @@ -12,6 +14,7 @@ from .metadata import parse_component_metadata from .power import ALL_POWER_METRIC_KEYS, POWER_METRIC_SCHEMA_VERSION, with_power_metrics from .topology import Parallelism, validate_parallelism +from infx.bench_serving.benchmark_outcome import benchmark_outcome _BASE_ENV_VARS = ( @@ -49,6 +52,14 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s 'isl': int(env['ISL']), 'osl': int(env['OSL']), } + if 'benchmark_outcome' in benchmark: + outcome = benchmark['benchmark_outcome'] + expected = benchmark_outcome(outcome['requested'], outcome['completed']) + if (outcome != expected + or benchmark.get('completed', expected['completed']) != expected['completed'] + or benchmark.get('num_prompts', expected['requested']) != expected['requested']): + raise ValueError("Benchmark outcome does not match the recorded request counts and gate") + data['benchmark_outcome'] = expected router = parse_component_metadata(env.get('ROUTER_METADATA'), 'ROUTER_METADATA') if router is not None: @@ -61,9 +72,6 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s is_multinode = env.get('IS_MULTINODE', 'false').lower() == 'true' if is_multinode: - # TODO: Eventually will have to have a separate condition in here for multinode disagg and - # multinode agg. For now, just assume that multinode implies disagg. - multinode_vars = ['PREFILL_GPUS', 'DECODE_GPUS', 'PREFILL_NUM_WORKERS', 'PREFILL_TP', 'PREFILL_EP', 'PREFILL_DP_ATTN', 'DECODE_NUM_WORKERS', 'DECODE_TP', 'DECODE_EP', 'DECODE_DP_ATTN'] @@ -76,6 +84,14 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s ) prefill_gpus = int(env['PREFILL_GPUS']) decode_gpus = int(env['DECODE_GPUS']) + aggregate_gpus = int(env.get('AGGREGATE_GPUS', '0')) + if aggregate_gpus: + if aggregate_gpus < 0 or prefill_gpus or decode_gpus or disagg: + raise ValueError("Aggregate GPUs require non-disaggregated topology without role GPUs") + # Preserve the existing multinode layout fields for consumers, but + # expose the actual shared role and validate telemetry as `agg`. + prefill_gpus = aggregate_gpus + data['num_aggregate_gpu'] = aggregate_gpus prefill_num_workers = int(env['PREFILL_NUM_WORKERS']) prefill = Parallelism( tp=int(env['PREFILL_TP']), @@ -154,9 +170,9 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s data = data | single_node_data for key, value in benchmark.items(): - if key.endswith('ms'): + if key.endswith('ms') and math.isfinite(float(value)): data[key.replace('_ms', '')] = float(value) / 1000.0 - if 'tpot' in key: + if 'tpot' in key and math.isfinite(float(value)) and float(value) > 0: data[key.replace('_ms', '').replace( 'tpot', 'intvty')] = 1000.0 / float(value) return data @@ -221,7 +237,8 @@ def aggregate_power_result( source = Path(env.get('POWER_ARTIFACT_DIR', 'LOGS/power')) prefill_gpus = int(env['PREFILL_GPUS']) decode_gpus = int(env['DECODE_GPUS']) - expected_num_gpus = prefill_gpus + decode_gpus + aggregate_gpus = int(env.get('AGGREGATE_GPUS', '0')) + expected_num_gpus = prefill_gpus + decode_gpus + aggregate_gpus else: candidates = [env.get('GPU_METRICS_CSV'), 'gpu_metrics.csv', '/workspace/gpu_metrics.csv'] source = next( @@ -237,6 +254,7 @@ def aggregate_power_result( source, bench_path, agg_path, prefill_gpus=prefill_gpus, decode_gpus=decode_gpus, + aggregate_gpus=aggregate_gpus, expected_producer_sha=env.get('POWER_PRODUCER_SHA') or None, logs_root=Path(env.get('POWER_RESULT_ROOT', 'LOGS')), validation_result=validation_path, @@ -265,8 +283,7 @@ def aggregate_power_result( return int(require_power) -def main() -> int: - env = os.environ +def process_result(env: Mapping[str, str]) -> int: require_environment(env, _BASE_ENV_VARS) result_filename = env['RESULT_FILENAME'] bench_path = Path(f'{result_filename}.json') @@ -277,10 +294,86 @@ def main() -> int: with open(agg_path, 'w') as f: json.dump(data, f, indent=2) status = aggregate_power_result(env, bench_path, agg_path) + validation_path = Path(f'power_validation_{result_filename}.json') + from .power.audit import audit_summary + + result = json.loads(agg_path.read_text()) + try: + validation = json.loads(validation_path.read_text()) + result.update(audit_summary(validation, validation_path.name)) + except (OSError, ValueError, TypeError) as exc: + print(f'[process_result] audit summary unavailable: {exc}', file=sys.stderr) + result['power_invalid_reasons'] = ['validation_artifact_unavailable'] + # A required run must preserve its audit as well as numeric metrics. + status = max(status, int(env.get('REQUIRE_POWER', '').lower() in {'1', 'true', 'yes'})) + agg_path.write_text(json.dumps(result, indent=2)) with open(agg_path) as f: print(json.dumps(json.load(f), indent=2)) + return max(status, int(data.get('benchmark_outcome', {}).get('status') == 'failed')) + + +def process_multinode_results(env: Mapping[str, str]) -> int: + """Process every available point and preserve sweep omissions before failing.""" + expected = {int(value) for value in env['CONC_LIST'].split()} + if not expected or min(expected) <= 0: + raise ValueError('CONC_LIST must contain positive concurrencies') + points: list[dict[str, Any]] = [] + observed: set[int] = set() + status = 0 + for path in sorted(Path('.').glob(f"{env['RESULT_FILENAME']}_*.json")): + point: dict[str, Any] = {'source': path.name} + try: + match = re.search( + r'_(?:c|conc|concurrency_)(\d+)(?:_req_rate_[^_]+)?_gpus_(\d+)' + r'(?:_ctx_(\d+)_gen_(\d+))?$', path.stem) + if match is None: + raise ValueError('Result filename lacks concurrency and physical GPU counts') + concurrency, total = int(match[1]), int(match[2]) + raw = json.loads(path.read_text()) + if raw['max_concurrency'] != concurrency: + raise ValueError('Result concurrency does not match its filename') + if concurrency in observed: + raise ValueError('Duplicate concurrency result within one recipe') + observed.add(concurrency) + point['concurrency'] = concurrency + point_env = {**env, 'RESULT_FILENAME': path.stem, 'IS_MULTINODE': 'true'} + disagg = env['DISAGG'].lower() == 'true' + # Some disaggregated recipe groups include a shared-worker point. + # Its zero decode workers and filename must both describe aggregate + # execution; the group-level flag cannot manufacture role energy. + aggregate = not disagg or int(env['DECODE_NUM_WORKERS']) == 0 + if aggregate: + if match[3] is not None and (int(match[3]) != total or int(match[4]) != 0): + raise ValueError('Aggregate result contains separate role GPU counts') + point_env.update(DISAGG='false', PREFILL_GPUS='0', DECODE_GPUS='0', + AGGREGATE_GPUS=str(total)) + else: + if match[3] is None: + raise ValueError('Disaggregated result lacks prefill/decode GPU counts') + prefill, decode = int(match[3]), int(match[4]) + if prefill + decode != total: + raise ValueError('Role GPU counts do not equal total GPU count') + point_env.update(PREFILL_GPUS=str(prefill), DECODE_GPUS=str(decode), AGGREGATE_GPUS='0') + point['exit_code'] = process_result(point_env) + except (OSError, ValueError, KeyError, TypeError) as exc: + point.update(exit_code=1, error=str(exc)) + print(f'[process_result] {path}: {exc}', file=sys.stderr) + status = max(status, point['exit_code']) + points.append(point) + missing, unexpected = sorted(expected - observed), sorted(observed - expected) + status = max(status, int(bool(missing or unexpected))) + summary = {'expected_concurrencies': sorted(expected), + 'missing_concurrencies': missing, 'unexpected_concurrencies': unexpected, + 'points': points, 'exit_code': status} + Path(f"result_processing_{env['RESULT_FILENAME']}.json").write_text(json.dumps(summary, indent=2)) + if missing or unexpected: + print(f'[process_result] incomplete sweep: missing={missing}, unexpected={unexpected}', file=sys.stderr) return status +def main() -> int: + return process_multinode_results(os.environ) if sys.argv[1:] == ['--all'] else process_result(os.environ) + + if __name__ == '__main__': sys.exit(main()) diff --git a/infx/results/power/__init__.py b/infx/results/power/__init__.py index f0e1da95c..8007f5a91 100644 --- a/infx/results/power/__init__.py +++ b/infx/results/power/__init__.py @@ -59,6 +59,7 @@ def with_power_metrics( data["power_metric_schema_version"] = schema_version data["power_valid"] = int(power_valid) data.pop("power_invalid_reasons", None) + data.pop("power_audit", None) if power_valid: for key, value in metrics.items(): if value is None or not math.isfinite(value): diff --git a/infx/results/power/audit.py b/infx/results/power/audit.py new file mode 100644 index 000000000..6463976cf --- /dev/null +++ b/infx/results/power/audit.py @@ -0,0 +1,50 @@ +"""Bounded public audit metadata from retained power-validation sidecars.""" + +from __future__ import annotations + +import math +import re +from collections.abc import Mapping +from typing import Any + + +def audit_summary(validation: Mapping[str, Any], source: str) -> dict[str, Any]: + """Project the shared app audit contract without publishing raw telemetry.""" + audit: dict[str, Any] = {"source": source} + window = validation.get("benchmark_window") or {} + producer = validation.get("producer") or {} + fields = { + "window_start_unix": window.get("start_time_unix"), + "window_end_unix": window.get("end_time_unix"), + "expected_gpu_count": validation.get("expected_gpu_count"), + "observed_gpu_count": validation.get("observed_gpu_count"), + } + counts = validation.get("per_gpu_sample_counts") or {} + if counts: + fields["sample_count"] = sum(counts.values()) + gaps = validation.get("per_gpu_max_sample_gap_s") or {} + finite_gaps = [value for value in gaps.values() + if type(value) in (int, float) and math.isfinite(value)] + if finite_gaps: + fields["max_sample_gap_s"] = max(finite_gaps) + for key, value in fields.items(): + if type(value) in (int, float) and math.isfinite(value) and value >= 0: + audit[key] = value + for target, original in (("producer_sha", "producer_git_commit"), + ("exporter_image_sha256", "exporter_image_sha256")): + value = producer.get(original) + if isinstance(value, str) and 0 < len(value) <= 128: + audit[target] = value + ids = validation.get("observed_gpu_ids") + if ids is None: + ids = list((validation.get("per_gpu_role") or {}).keys()) + if ids: + audit["observed_gpu_ids"] = list(dict.fromkeys( + str(value) for value in ids if 0 < len(str(value)) <= 128 + ))[:1024] + return { + "power_invalid_reasons": [reason for reason in validation.get("reasons", []) + if isinstance(reason, str) + and re.fullmatch(r"[a-z][a-z0-9_]{0,63}", reason)][:32], + "power_audit": audit, + } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 88c163461..f3b85de24 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7399,3 +7399,14 @@ - "Disable adaptive verification in the EVAL_ONLY DSpark config as well. It trims verification requests on device, which the ROCm DeepseekV4IndexerBackend does not support, so the eval-only engine refused to start (run 34651830283, c32). Evals keep real block rejection; throughput settings are unchanged." - "EVAL_ONLY 的 DSpark 配置同样关闭自适应验证：它会在设备端裁剪验证请求，而 ROCm 的 DeepseekV4IndexerBackend 不支持该操作，导致仅评测引擎拒绝启动（运行 34651830283，c32）。评测仍保留真实块拒绝采样；吞吐设置不变。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Preserve request failure outcomes, retain all available fixed-sequence diagnostics, and reject incomplete concurrency batches before publishing benchmark artifacts." + - "保留请求失败状态及全部可用的固定序列诊断信息；发布基准产物前拒绝缺失并发点的批次。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 485021b36..d14d6a87a 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -28,12 +28,13 @@ def test_workflow_processes_results_through_compatibility_entrypoint( step = next(step for job in workflow["jobs"].values() for step in job.get("steps", []) if step.get("name", "").startswith("Process result")) multinode = workflow_name == "benchmark-multinode-tmpl.yml" - stem = "fixture_gpus_28_ctx_20_gen_8" if multinode else "fixture" + stem = "fixture_conc64_gpus_28_ctx_20_gen_8" if multinode else "fixture" (tmp_path / f"{stem}.json").write_text(json.dumps({ **sample_benchmark_result, "total_token_throughput": 56, "output_throughput": 28, })) env = {**(multinode_env_vars if multinode else single_node_env_vars), "RESULT_FILENAME": "fixture" if multinode else stem, "POWER_PRODUCER_SHA": "", + "CONC_LIST": "64", "PATH": f"{Path(sys.executable).parent}:{os.environ['PATH']}"} result = subprocess.run(["bash", "-euo", "pipefail", "-c", step["run"]], cwd=tmp_path, env=env, capture_output=True, text=True, timeout=10) @@ -845,7 +846,7 @@ def test_missing_csv_does_not_break_process_result(self, tmp_path, single_node_e assert "avg_power_w" not in patched assert "joules_per_output_token" not in patched assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] validation = json.loads( (tmp_path / "power_validation_benchmark_result.json").read_text() @@ -876,7 +877,7 @@ def test_missing_bench_timestamps_does_not_patch(self, tmp_path, single_node_env assert "avg_power_w" not in patched assert "joules_per_output_token" not in patched assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] def test_expected_gpu_count_mismatch_is_invalid(self, tmp_path, single_node_env_vars): """TP/PP/PCP topology is checked against the observed device IDs.""" @@ -902,7 +903,7 @@ def test_expected_gpu_count_mismatch_is_invalid(self, tmp_path, single_node_env_ assert result.returncode == 0, f"Script failed: {result.stderr}" patched = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] assert "total_gpu_energy_j" not in patched def test_require_power_propagates_validation_failure( @@ -1009,7 +1010,7 @@ def test_internal_aggregation_error_is_always_auditable( agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 0 - assert "power_invalid_reasons" not in agg + assert agg["power_invalid_reasons"] == ["aggregation_internal_error"] validation = json.loads( (tmp_path / "power_validation_benchmark_result.json").read_text() ) @@ -1416,3 +1417,209 @@ def test_strict_mode_passes_on_valid_package(self, tmp_path, power_env): assert result.returncode == 0, f"Script failed: {result.stderr}" agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert agg["power_valid"] == 1 + + +@pytest.mark.parametrize('completed,status', [(100, 'passed'), (95, 'passed'), (94, 'failed')]) +def test_request_outcome_preserves_existing_failure_threshold(completed, status): + from infx.bench_serving.benchmark_outcome import benchmark_outcome + + outcome = benchmark_outcome(100, completed) + assert outcome['status'] == status + assert outcome['failed'] == 100 - completed + assert outcome['max_failure_rate'] == 0.05 + + +def test_failed_client_is_preserved_with_valid_power(tmp_path, single_node_env_vars): + start, end = 1_700_000_100.0, 1_700_000_110.0 + TestPowerAggregationIntegration._write_nvidia_csv(tmp_path / 'gpu_metrics.csv', start, end, num_gpus=2) + outcome = {'status': 'failed', 'requested': 100, 'completed': 94, 'failed': 6, + 'max_failure_rate': 0.05} + raw = {'model_id': 'fixture', 'max_concurrency': 4, 'total_token_throughput': 500, + 'output_throughput': 100, 'benchmark_start_time_unix': start, + 'benchmark_end_time_unix': end, 'duration': 10, 'completed': 94, + 'total_input_tokens': 8192, 'total_output_tokens': 1024, + 'benchmark_outcome': outcome} + result = run_script(tmp_path, {**single_node_env_vars, 'TP': '2'}, raw) + assert result.returncode == 1, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['benchmark_outcome'] == outcome + assert aggregate['power_valid'] == 1 + assert aggregate['power_invalid_reasons'] == [] + assert aggregate['power_audit']['window_start_unix'] == start + assert aggregate['power_audit']['expected_gpu_count'] == 2 + assert aggregate['power_audit']['observed_gpu_count'] == 2 + assert aggregate['power_audit']['source'] == 'power_validation_benchmark_result.json' + + +def test_request_outcome_cannot_disagree_with_raw_counts(single_node_env_vars, sample_benchmark_result): + from infx.results.fixed_sequence import build_result + + raw = {**sample_benchmark_result, 'completed': 94, + 'benchmark_outcome': {'status': 'passed', 'requested': 100, 'completed': 100, + 'failed': 0, 'max_failure_rate': 0.05}} + with pytest.raises(ValueError, match='request counts and gate'): + build_result(raw, single_node_env_vars) + + +def test_multinode_aggregate_role_through_result_processor(tmp_path, multinode_env_vars): + pkg = build_package(tmp_path, bench_extra=TestMultinodePower.BENCH_EXTRA) + manifest_path = pkg.power_dir / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + for device in manifest['expected_devices']: + for assignment in device['assignments']: + assignment.update(worker_role='agg', het_group=None) + manifest_path.write_text(json.dumps(manifest)) + env = {**multinode_env_vars, 'DISAGG': 'false', 'PREFILL_GPUS': '0', + 'DECODE_GPUS': '0', 'AGGREGATE_GPUS': '4', 'POWER_PRODUCER_SHA': PRODUCER_SHA, + 'REQUIRE_POWER': '1'} + result = run_script(tmp_path, env, json.loads(pkg.original_result.read_text())) + assert result.returncode == 0, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['power_valid'] == 1 + assert aggregate['num_aggregate_gpu'] == 4 + assert aggregate['avg_power_w'] == 350 + assert aggregate['power_audit']['producer_sha'] == PRODUCER_SHA + assert set(ROLE_METRIC_KEYS).isdisjoint(aggregate) + + +@pytest.mark.parametrize('conc_token,rate_suffix', [ + ('c', ''), ('conc', ''), + ('concurrency_', '_req_rate_1'), ('concurrency_', '_req_rate_inf'), +]) +@pytest.mark.parametrize('expected_concs,missing', [('4 8 16', [8]), ('4 16', [])]) +def test_multinode_batch_preserves_points_and_checks_completeness( + tmp_path, multinode_env_vars, sample_benchmark_result, conc_token, rate_suffix, + expected_concs, missing, +): + for conc in (4, 16): + (tmp_path / f'run_recipe_{conc_token}{conc}{rate_suffix}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': conc})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': expected_concs, 'REQUIRE_POWER': '0'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == int(bool(missing)), result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == missing + assert receipt['unexpected_concurrencies'] == [] + assert len(receipt['points']) == 2 + for conc in (4, 16): + stem = f'run_recipe_{conc_token}{conc}{rate_suffix}_gpus_4_ctx_2_gen_2' + aggregate = json.loads((tmp_path / f'agg_{stem}.json').read_text()) + assert aggregate['power_valid'] == 0 + assert aggregate['power_invalid_reasons'] + assert (tmp_path / f'power_validation_{stem}.json').is_file() + + +@pytest.mark.parametrize('role_suffix', ['', '_ctx_4_gen_0']) +def test_multinode_batch_normalizes_legacy_zero_decode_aggregate( + tmp_path, multinode_env_vars, role_suffix, +): + pkg = build_package(tmp_path, bench_extra=TestMultinodePower.BENCH_EXTRA) + manifest_path = pkg.power_dir / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + for device in manifest['expected_devices']: + for assignment in device['assignments']: + assignment.update(worker_role='agg', het_group=None) + manifest_path.write_text(json.dumps(manifest)) + stem = f'run_recipe_c4_gpus_4{role_suffix}' + (tmp_path / f'{stem}.json').write_text(pkg.original_result.read_text()) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4', 'DECODE_NUM_WORKERS': '0', 'REQUIRE_POWER': '1', + 'POWER_PRODUCER_SHA': PRODUCER_SHA} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 0, result.stderr + aggregate = json.loads((tmp_path / f'agg_{stem}.json').read_text()) + assert aggregate['disagg'] is False + assert aggregate['num_aggregate_gpu'] == 4 + assert aggregate['num_decode_gpu'] == 0 + assert aggregate['decode_num_workers'] == 0 + assert aggregate['power_valid'] == 1 + assert aggregate['avg_power_w'] == 350 + assert set(ROLE_METRIC_KEYS).isdisjoint(aggregate) + + +@pytest.mark.parametrize('decode_workers,role_suffix', [ + ('0', '_ctx_2_gen_2'), ('0', '_ctx_3_gen_0'), ('1', ''), +]) +def test_multinode_batch_rejects_misdeclared_aggregate_role_counts( + tmp_path, multinode_env_vars, sample_benchmark_result, decode_workers, role_suffix, +): + stem = f'run_recipe_c4_gpus_4{role_suffix}' + (tmp_path / f'{stem}.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': 4})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4', 'DECODE_NUM_WORKERS': decode_workers} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['points'][0]['exit_code'] == 1 + assert not (tmp_path / f'agg_{stem}.json').exists() + + +def test_zero_successful_requests_preserves_diagnostic_result(tmp_path, single_node_env_vars, sample_benchmark_result): + raw = {**sample_benchmark_result, 'tpot_p50_ms': 0, 'tpot_p99_ms': float('nan'), + 'completed': 0, 'num_prompts': 4, + 'benchmark_outcome': {'status': 'failed', 'requested': 4, 'completed': 0, + 'failed': 4, 'max_failure_rate': 0.05}} + result = run_script(tmp_path, single_node_env_vars, raw) + assert result.returncode == 1, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['benchmark_outcome']['failed'] == 4 + assert aggregate['power_valid'] == 0 + assert 'intvty_p50' not in aggregate + assert 'tpot_p99' not in aggregate + + +@pytest.mark.parametrize('extra_conc,error', [(4, 'Duplicate concurrency'), (8, None)]) +def test_multinode_batch_rejects_extra_results_without_losing_other_points( + tmp_path, multinode_env_vars, sample_benchmark_result, extra_conc, error, +): + for label, conc in [('a', 4), ('b', extra_conc), ('c', 16)]: + (tmp_path / f'run_{label}_conc{conc}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': conc})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4 16'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [] + assert receipt['unexpected_concurrencies'] == ([] if error else [8]) + if error: + assert error in receipt['points'][1]['error'] + assert (tmp_path / 'agg_run_a_conc4_gpus_4_ctx_2_gen_2.json').is_file() + assert (tmp_path / 'agg_run_c_conc16_gpus_4_ctx_2_gen_2.json').is_file() + + +def test_multinode_empty_sweep_records_every_missing_point(tmp_path, multinode_env_vars): + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4 8'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [4, 8] + assert receipt['points'] == [] + + +def test_public_power_audit_bounds_text_and_device_identifiers(): + from infx.results.power.audit import audit_summary + + summary = audit_summary({ + 'producer': {'producer_git_commit': 'x' * 129, 'exporter_image_sha256': 'a' * 64}, + 'observed_gpu_ids': ['gpu0', 'gpu0', 'x' * 129] + [f'gpu{i}' for i in range(1, 1025)], + 'reasons': ['window_missing', ''] + [f'reason_{i}' for i in range(40)], + }, 'power_validation_run.json') + assert len(summary['power_invalid_reasons']) == 32 + assert '' not in summary['power_invalid_reasons'] + audit = summary['power_audit'] + assert 'producer_sha' not in audit + assert audit['exporter_image_sha256'] == 'a' * 64 + assert len(audit['observed_gpu_ids']) == 1024 + assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] From ba0275c4769dc42e606d6fec2e89a3c910b9eb5a Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:24:13 -0700 Subject: [PATCH 02/17] feat: separate native multinode power collection contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 拆分原生多节点功耗采集契约、UTC 上下文和生命周期屏障，不启用新路径。 --- .../workflows/benchmark-multinode-tmpl.yml | 1 + .github/workflows/benchmark-tmpl.yml | 4 + benchmarks/benchmark_lib.sh | 7 +- benchmarks/native_power_collect.sh | 73 +++++ benchmarks/native_power_lifecycle.sh | 77 +++++ docs/results-and-ingestion.md | 6 + docs/results-and-ingestion_zh.md | 6 + infx/results/fixed_sequence.py | 15 + infx/results/power/native_multinode.py | 286 ++++++++++++++++++ infx/results/power/single_node.py | 59 +++- perf-changelog.yaml | 11 + runners/test_native_collector_barriers.py | 58 ++++ runners/test_native_collector_receipts.py | 16 + utils/test_aggregate_power.py | 255 +++++++++++++++- utils/test_native_multinode_power.py | 201 ++++++++++++ 15 files changed, 1057 insertions(+), 18 deletions(-) create mode 100644 benchmarks/native_power_collect.sh create mode 100644 benchmarks/native_power_lifecycle.sh create mode 100644 infx/results/power/native_multinode.py create mode 100644 runners/test_native_collector_barriers.py create mode 100644 runners/test_native_collector_receipts.py create mode 100644 utils/test_native_multinode_power.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 80b0842f8..e9f3a3c90 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -499,6 +499,7 @@ jobs: agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json LOGS/power/** + LOGS/native_power/** result_processing_${{ env.RESULT_FILENAME }}.json LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 4d1b817ea..1da933f63 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -445,11 +445,13 @@ jobs: name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }} path: | gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json if-no-files-found: ignore @@ -462,12 +464,14 @@ jobs: ${{ env.RESULT_FILENAME }}.json agg_${{ env.RESULT_FILENAME }}.json gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json results/agentic_power_window.json results/agentic_power_timezone_offset.txt diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 305a118ef..9d99a4cc9 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -177,6 +177,7 @@ start_gpu_monitor() { GPU_METRICS_CSV="$output" GPU_MONITOR_INTERVAL="$interval" export GPU_METRICS_CSV + printf '{"timestamp_timezone":"UTC"}\n' > "${output%.csv}_context.json" if command -v nvidia-smi &>/dev/null; then GPU_MONITOR_VENDOR="nvidia" @@ -185,7 +186,7 @@ start_gpu_monitor() { rm -f "${output%.csv}_identity.csv" echo "[GPU Monitor] Warning: NVIDIA identity sidecar failed" >&2 fi - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv -l "$interval" > "$output" 2>/dev/null & GPU_MONITOR_PID=$! echo "[GPU Monitor] Started NVIDIA (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)" @@ -196,7 +197,7 @@ start_gpu_monitor() { # Python; measured on MI355X: trailing ticks were lost at kill without it). # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated # headers, and flush every row so killing the pipe cannot discard buffered samples. - PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ + TZ=UTC PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ | awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" & GPU_MONITOR_PID=$! # Hardware energy-accumulator + identity snapshots; the end-side twin in @@ -232,7 +233,7 @@ stop_gpu_monitor() { case "$GPU_MONITOR_VENDOR" in nvidia) if _repair_truncated_gpu_metrics_tail; then - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv,noheader >> "$GPU_METRICS_CSV" 2>/dev/null || echo "[GPU Monitor] Warning: final NVIDIA sample failed" >&2 fi diff --git a/benchmarks/native_power_collect.sh b/benchmarks/native_power_collect.sh new file mode 100644 index 000000000..69df07951 --- /dev/null +++ b/benchmarks/native_power_collect.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# One native SMI collector per serving node. Raw files stay on node-local scratch; +# the launcher stages them as its host user after containers stop. +set -uo pipefail +power_dir=$1 +control_dir=$2 +vendor=$3 +rank=$4 +role=$5 +gpu_indices=$6 +num_nodes=$7 +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" +source "$repo_root/benchmarks/benchmark_lib.sh" +mkdir -p "$power_dir" +collector_rc=0 +finished=0 + +write_control() { + local path="$control_dir/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + # The control directory is created by the host user before Docker starts. + # Containers must not strand root-owned files in the shared runner tree. + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +finish() { + local incoming_rc=$? + [[ "$finished" == 0 ]] || return + if [[ "$incoming_rc" != 0 ]]; then collector_rc=$incoming_rc; fi + if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi + stop_gpu_monitor + if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json + else + nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \ + > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1 + fi + python3 -m infx.results.power.native_multinode end --directory "$power_dir" \ + --collector-exit-code "$collector_rc" || collector_rc=1 + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown -R "$POWERX_HOST_UID:$POWERX_HOST_GID" "$power_dir" || collector_rc=1 + fi + write_control "done-$rank" "$collector_rc" + finished=1 +} +trap finish EXIT +trap 'collector_rc=130; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 130' INT +trap 'collector_rc=143; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 143' TERM HUP + +case "${POWERX_CLOCK_SYNCHRONIZED:-false}" in + yes|true) clock_synchronized=true ;; + *) clock_synchronized=false ;; +esac + +python3 -m infx.results.power.native_multinode begin --directory "$power_dir" \ + --vendor "$vendor" --rank "$rank" --role "$role" --gpu-indices "$gpu_indices" \ + --num-nodes "$num_nodes" --clock-synchronized "$clock_synchronized" || exit 1 +start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 +[[ "$GPU_MONITOR_VENDOR" == "$vendor" ]] || exit 1 +if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json +fi +write_control "ready-$rank" ready +while [[ ! -f "$control_dir/stop" ]]; do + _background_process_is_running "$GPU_MONITOR_PID" || exit 1 + sleep 1 & + wait $! || true +done diff --git a/benchmarks/native_power_lifecycle.sh b/benchmarks/native_power_lifecycle.sh new file mode 100644 index 000000000..b24d1dada --- /dev/null +++ b/benchmarks/native_power_lifecycle.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Shared barriers for native collectors. Launchers own host scratch and mounts. + +powerx_start_collector() { + local power_dir="$1" control_dir="$2" vendor="$3" rank="$4" role="$5" gpus="$6" nodes="$7" + local indices + indices=$(seq -s, 0 "$((gpus - 1))") || return 1 + POWERX_CONTROL_DIR="$control_dir" + POWERX_NUM_NODES="$nodes" + bash "$(dirname "${BASH_SOURCE[0]}")/native_power_collect.sh" \ + "$power_dir" "$control_dir" "$vendor" "$rank" "$role" "$indices" "$nodes" & + POWERX_COLLECTOR_PID=$! +} + +powerx_write_control() { + local path="$POWERX_CONTROL_DIR/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +powerx_wait_collectors() { + local phase="$1" deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rank pending failed + while :; do + pending=0 + failed=0 + for ((rank=0; rank&2 + return 1 + fi + if [[ ! -f "$POWERX_CONTROL_DIR/$phase-$rank" ]]; then + pending=1 + elif [[ "$phase" == done && "$(cat "$POWERX_CONTROL_DIR/done-$rank")" != 0 ]]; then + failed=1 + fi + done + if [[ "$pending" == 0 ]]; then + [[ "$failed" == 0 ]] || echo "One or more PowerX collectors failed" >&2 + return "$failed" + fi + if (( SECONDS >= deadline )); then + echo "Timed out waiting for PowerX $phase receipts" >&2 + return 1 + fi + sleep 1 + done +} + +powerx_stop_collectors() { + local rc=0 + powerx_write_control stop stop || rc=$? + powerx_wait_collectors done || rc=$? + powerx_reap_collector || rc=$? + return "$rc" +} + +powerx_reap_collector() { + [[ -n "${POWERX_COLLECTOR_PID:-}" ]] || return 0 + local deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rc=0 + while kill -0 "$POWERX_COLLECTOR_PID" 2>/dev/null; do + if (( SECONDS >= deadline )); then + kill -TERM "$POWERX_COLLECTOR_PID" 2>/dev/null || true + sleep 2 + kill -KILL "$POWERX_COLLECTOR_PID" 2>/dev/null || true + rc=1 + break + fi + sleep 1 + done + wait "$POWERX_COLLECTOR_PID" || rc=$? + POWERX_COLLECTOR_PID="" + return "$rc" +} diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index a1e357b30..6da11624d 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -106,6 +106,12 @@ For multinode fixed-sequence jobs, `utils/process_result.py --all` processes eve Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. +### Native multinode telemetry + +`native_power_collect.sh` and `native_power_lifecycle.sh` provide per-node collection and bounded ready/stop receipts. Launchers opt into the native package under `LOGS/native_power`; this prerequisite enables no new recipe. The adapter validates serving GPU identity, synchronized clocks, collector completion, and complete formal-window coverage. It preserves per-node failures, sample counts, and collector revision in the audit. + +SMI collection records UTC context beside each CSV for portable replay. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index 6a054d5f0..fd3fa2a2d 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -106,6 +106,12 @@ InferenceX-app 将路由字段作为列或配置维度，并把数值测量存 启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 +### 原生多节点遥测 + +`native_power_collect.sh` 和 `native_power_lifecycle.sh` 提供每节点采集及有时限的就绪/停止状态文件。启动器可使用 `LOGS/native_power` 下的原生产物；此前置改动不会启用新 recipe。适配器验证服务 GPU 身份、时钟同步、采集完成及正式窗口完整覆盖，并在审计中保留节点故障、样本数和采集器版本。 + +SMI 采集在 CSV 旁记录 UTC 上下文以支持跨环境回放。正式窗口外的无效样本不能构成覆盖；`boundary_degenerate_rows` 保留其逐 GPU 计数。 + ## 评测工件 ### 单配置身份和收集 diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index f6fa76102..5ae8a65ee 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -248,6 +248,21 @@ def aggregate_power_result( expected_num_gpus = int(env['TP']) * int(env.get('PP_SIZE', '1')) * int(env.get('PCP_SIZE', '1')) try: if is_multinode: + native_dir = Path(env.get('POWERX_NATIVE_DIR', 'LOGS/native_power')) + if env.get('POWERX_NATIVE_DIR') or native_dir.is_dir(): + if source.is_dir() and source != native_dir: + raise ValueError('Both native and SRT power packages are present') + source = native_dir + from .power.native_multinode import run + + return run( + native_dir, bench_path, agg_path, + expected_prefill_gpus=prefill_gpus, + expected_decode_gpus=decode_gpus, + expected_aggregate_gpus=aggregate_gpus, + validation_result=validation_path, + require_power=require_power, + ) from .power.multinode import run return run( diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py new file mode 100644 index 000000000..1ffae5bd9 --- /dev/null +++ b/infx/results/power/native_multinode.py @@ -0,0 +1,286 @@ +"""Validate native SMI traces from each node of one fixed-sequence deployment. + +This format is owned by InferenceX. It does not claim the srt-slurm/DCGM wire +contract. The launcher records real serving-device membership and synchronized +host clocks; the normal result processor binds every trace to the client window. +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import os +import socket +import time +from datetime import timezone +from pathlib import Path + +from . import ALL_POWER_METRIC_KEYS +from .common import ( + _load_benchmark_data, _write_json_atomic, benchmark_window_payload, + patch_power_metrics, +) +from .single_node import _derived_metrics, _detect_columns, _parse_timestamp, integrate_power + + +def _identity(path: Path, vendor: str) -> dict[str, str]: + """Map the vendor enumeration index to a physical UUID; never use PCI alone.""" + if vendor == "nvidia": + with path.open(newline="") as stream: + rows = [] + for row in csv.DictReader(stream, skipinitialspace=True): + if any(not isinstance(k, str) or not isinstance(v, str) for k, v in row.items()): + raise ValueError("invalid_device_identity") + rows.append({k.strip().lower(): v.strip() for k, v in row.items()}) + elif vendor == "amd": + payload = json.loads(path.read_text()) + rows = [] + + def visit(value: object) -> None: + if isinstance(value, list): + for item in value: + visit(item) + elif isinstance(value, dict): + row = {str(k).lower(): v for k, v in value.items()} + if "gpu" in row and "uuid" in row: + rows.append(row) + else: + for item in value.values(): + visit(item) + + visit(payload) + else: + raise ValueError("unsupported_native_vendor") + identities = {} + for row in rows: + index = str(row.get("index", row.get("gpu", ""))).strip() + uuid = str(row.get("uuid", "")).strip() + if not index.isdigit() or not uuid or uuid.lower() in {"n/a", "none", "null"}: + raise ValueError("invalid_device_identity") + if index in identities or uuid in identities.values(): + raise ValueError("duplicate_device_identity") + identities[index] = uuid + if not identities: + raise ValueError("device_identity_missing") + return identities + + +def record_begin(directory: Path, *, vendor: str, node: str, rank: int, + role: str, gpu_indices: list[int], num_nodes: int, job_id: str, + clock_synchronized: bool, revision: str) -> None: + if (role not in {"prefill", "decode", "aggregate"} or rank < 0 or + num_nodes <= rank or not gpu_indices or min(gpu_indices) < 0 or + len(set(gpu_indices)) != len(gpu_indices)): + raise ValueError("invalid_native_topology") + directory.mkdir(parents=True, exist_ok=True) + _write_json_atomic(directory / "manifest.json", { + "schema_version": 1, "collector": "inferencex-native-smi", + "vendor": vendor, "node": node, "rank": rank, "role": role, + "selected_gpu_indices": gpu_indices, "expected_num_nodes": num_nodes, + "job_id": job_id, "collector_revision": revision, + "clock_source": "utc_ntp", "clock_synchronized": clock_synchronized, + "clock_observation": "timedatectl NTPSynchronized on serving host; no measured clock offset", + "collection_start_unix": time.time(), "lifecycle": "collecting", + }) + + +def record_end(directory: Path, *, collector_exit_code: int) -> None: + manifest = json.loads((directory / "manifest.json").read_text()) + manifest.update(collection_end_unix=time.time(), + collector_exit_code=collector_exit_code, + lifecycle="complete" if collector_exit_code == 0 else "failed") + _write_json_atomic(directory / "manifest.json", manifest) + + +def run(power_dir: Path, bench_result: Path, agg_result: Path, *, + expected_prefill_gpus: int, expected_decode_gpus: int, expected_aggregate_gpus: int = 0, + validation_result: Path | None = None, require_power: bool = False) -> int: + validation_result = validation_result or bench_result.with_name( + f"power_validation_{bench_result.stem}.json") + benchmark, reasons = _load_benchmark_data(bench_result) + expected_gpus = expected_prefill_gpus + expected_decode_gpus + expected_aggregate_gpus + if expected_aggregate_gpus and (expected_prefill_gpus or expected_decode_gpus): + reasons.append("native_mixed_aggregate_role_topology") + roles: dict[str, list[str]] = {"prefill": [], "decode": [], "aggregate": []} + receipts = [] + node_errors = [] + samples = [] + ranks = [] + nodes = [] + jobs = set() + revisions = set() + expected_nodes = set() + paths = sorted(power_dir.glob("node-*/manifest.json")) + if not paths: + reasons.append("native_manifests_missing") + if expected_gpus <= 0 or min(expected_prefill_gpus, expected_decode_gpus, expected_aggregate_gpus) < 0: + reasons.append("invalid_expected_gpu_count") + for path in paths: + try: + manifest = json.loads(path.read_text()) + rank = manifest["rank"] + role = manifest["role"] + if (manifest.get("schema_version") != 1 or + manifest.get("collector") != "inferencex-native-smi" or + type(rank) is not int or rank < 0 or role not in roles or + path.parent.name != f"node-{rank}"): + raise ValueError("invalid_native_manifest") + if (not isinstance(manifest.get("node"), str) or not manifest["node"] or + not isinstance(manifest.get("job_id"), str) or + not isinstance(manifest.get("collector_revision"), str) or + type(manifest.get("expected_num_nodes")) is not int or + manifest["expected_num_nodes"] <= 0): + raise ValueError("invalid_native_run_identity") + ranks.append(rank) + nodes.append(manifest["node"]) + expected_nodes.add(manifest["expected_num_nodes"]) + jobs.add(manifest["job_id"]) + revisions.add(manifest["collector_revision"]) + if (manifest.get("lifecycle") != "complete" or + manifest.get("collector_exit_code") != 0): + reasons.append("native_collector_incomplete") + if (manifest.get("clock_source") != "utc_ntp" or + manifest.get("clock_synchronized") is not True): + reasons.append("native_clock_not_synchronized") + start, end = manifest["collection_start_unix"], manifest["collection_end_unix"] + if (not all(type(x) in (int, float) and math.isfinite(x) for x in (start, end)) + or end <= start or (benchmark is not None and + (start > benchmark.start_unix or end < benchmark.end_unix))): + reasons.append("native_collection_window_mismatch") + selected = manifest["selected_gpu_indices"] + if (not isinstance(selected, list) or not selected or + any(type(i) is not int or i < 0 for i in selected) or + len(set(selected)) != len(selected)): + raise ValueError("invalid_native_gpu_selection") + suffix = "csv" if manifest["vendor"] == "nvidia" else "json" + stem = "gpu_metrics_identity" if suffix == "csv" else "gpu_metrics_devices" + first_path = path.parent / f"{stem}.{suffix}" + last_path = path.parent / f"{stem}_end.{suffix}" + first = _identity(first_path, manifest["vendor"]) + last = _identity(last_path, manifest["vendor"]) + selected_ids = {str(i): first[str(i)] for i in selected} + if any(last.get(i) != uuid for i, uuid in selected_ids.items()): + reasons.append("native_device_identity_changed") + previous = {uuid for members in roles.values() for uuid in members} + if previous.intersection(selected_ids.values()): + reasons.append("native_duplicate_physical_gpu") + roles[role].extend(selected_ids.values()) + csv_path = path.parent / "gpu_metrics.csv" + with csv_path.open(newline="") as stream: + reader = csv.DictReader(stream, skipinitialspace=True) + reader.fieldnames = [c.strip() for c in (reader.fieldnames or [])] + t_col, p_col, g_col = _detect_columns(reader.fieldnames) + if not all((t_col, p_col, g_col)): + raise ValueError("native_telemetry_columns_missing") + for row in reader: + gpu = (row.get(g_col) or "").strip() + if gpu not in first: + reasons.append("native_unknown_device_index") + continue + if gpu not in selected_ids: + continue + timestamp = _parse_timestamp((row.get(t_col) or ""), naive_timezone=timezone.utc) + # Invalid rows are retained for the common validator to reject. + samples.append((timestamp, selected_ids[gpu], row.get(p_col) or "")) + receipts.append({**manifest, "physical_gpu_ids": selected_ids, + "manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "telemetry_sha256": hashlib.sha256(csv_path.read_bytes()).hexdigest(), + "identity_sha256": hashlib.sha256(first_path.read_bytes()).hexdigest(), + "identity_end_sha256": hashlib.sha256(last_path.read_bytes()).hexdigest()}) + except (OSError, ValueError, KeyError, TypeError, csv.Error) as exc: + reasons.append("native_node_invalid") + node_errors.append({"node": path.parent.name, "type": type(exc).__name__, "detail": str(exc)}) + if (len(expected_nodes) != 1 or not expected_nodes or + type(next(iter(expected_nodes), None)) is not int or + set(ranks) != set(range(next(iter(expected_nodes), 0))) or + len(set(nodes)) != len(nodes) or len(ranks) != len(set(ranks))): + reasons.append("native_node_topology_mismatch") + if len(jobs) != 1 or "" in jobs or len(revisions) != 1 or "" in revisions: + reasons.append("native_run_identity_mismatch") + if expected_aggregate_gpus: + if roles["prefill"] or roles["decode"] or len(roles["aggregate"]) != expected_aggregate_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif expected_decode_gpus: + if roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or len(roles["decode"]) != expected_decode_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif roles["decode"] or len(roles["aggregate"] + roles["prefill"]) != expected_prefill_gpus: + reasons.append("native_role_gpu_count_mismatch") + combined_path = validation_result.with_name(f"{validation_result.stem}_native.csv") + combined_path.parent.mkdir(parents=True, exist_ok=True) + with combined_path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["timestamp", "gpu", "power"]) + writer.writerows(samples) + integration = None + if benchmark is not None: + integration = integrate_power(combined_path, start_unix=benchmark.start_unix, + end_unix=benchmark.end_unix, expected_num_gpus=expected_gpus) + reasons.extend(integration.invalid_reasons) + metrics = {} + if not reasons and integration is not None and benchmark is not None: + metrics = _derived_metrics(integration, benchmark) + if expected_decode_gpus: + prefill = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["prefill"]) + decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) + metrics.update(prefill_gpu_energy_j=prefill, decode_gpu_energy_j=decode, + prefill_avg_power_w=prefill / benchmark.integration_duration_s / expected_prefill_gpus, + decode_avg_power_w=decode / benchmark.integration_duration_s / expected_decode_gpus, + prefill_joules_per_input_token=prefill / benchmark.total_input_tokens, + decode_joules_per_output_token=decode / benchmark.total_output_tokens) + valid = not reasons + try: + patch_power_metrics(agg_result, metric_keys=ALL_POWER_METRIC_KEYS, power_valid=valid, metrics=metrics) + except (OSError, ValueError): + reasons.append("aggregate_result_unwritable") + valid, metrics = False, {} + audit = {"schema_version": 1, "telemetry_kind": "native_multinode_smi", + "power_valid": valid, "reasons": list(dict.fromkeys(reasons)), + "benchmark_result": str(bench_result), + "benchmark_result_sha256": hashlib.sha256(bench_result.read_bytes()).hexdigest() if bench_result.is_file() else None, + "benchmark_window": benchmark_window_payload(benchmark), + "expected_gpu_count": expected_gpus, "nodes": receipts, + "node_errors": node_errors, + "observed_gpu_count": integration.observed_num_gpus if integration else 0, + "per_gpu_role": {uuid: role for role, uuids in roles.items() for uuid in uuids}, + "per_gpu_sample_counts": integration.per_gpu_sample_counts if integration else {}, + "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s if integration else {}, + "producer": {"name": "inferencex-native-smi", "revisions": sorted(revisions), + "producer_git_commit": next(iter(revisions)) if len(revisions) == 1 else None}, + "integration_method": "per_device_trapezoidal_with_linear_boundary_interpolation", + "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear", + "metrics": metrics} + _write_json_atomic(validation_result, audit) + return int(require_power and not valid) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + begin = sub.add_parser("begin") + begin.add_argument("--directory", type=Path, required=True) + begin.add_argument("--vendor", choices=("amd", "nvidia"), required=True) + begin.add_argument("--node", default=os.environ.get("POWERX_NODE_NAME", socket.gethostname())) + begin.add_argument("--rank", type=int, required=True) + begin.add_argument("--role", choices=("prefill", "decode", "aggregate"), required=True) + begin.add_argument("--gpu-indices", required=True) + begin.add_argument("--num-nodes", type=int, required=True) + begin.add_argument("--job-id", default=os.environ.get("SLURM_JOB_ID", "")) + begin.add_argument("--revision", default=os.environ.get("POWERX_COLLECTOR_REVISION", "")) + begin.add_argument("--clock-synchronized", choices=("true", "false"), default="false") + end = sub.add_parser("end") + end.add_argument("--directory", type=Path, required=True) + end.add_argument("--collector-exit-code", type=int, required=True) + args = vars(parser.parse_args()) + action = args.pop("action") + if action == "begin": + args["gpu_indices"] = [int(i) for i in args["gpu_indices"].split(",")] + args["clock_synchronized"] = args["clock_synchronized"] == "true" + record_begin(**args) + else: + record_end(**args) + + +if __name__ == "__main__": + main() diff --git a/infx/results/power/single_node.py b/infx/results/power/single_node.py index 54669a1a8..4ce1bcaaf 100644 --- a/infx/results/power/single_node.py +++ b/infx/results/power/single_node.py @@ -9,7 +9,11 @@ aggregate and a validation sidecar, but does not fail the benchmark. Power studies can set ``REQUIRE_POWER=1`` to fail after those audit artifacts exist. The aggregate carries numeric ``power_valid`` (1/0) for metric ingestion; the -sidecar is the canonical source for boolean validity and reason codes. +sidecar is the canonical source for boolean validity and reason codes. Rows in +the ingest band but outside the formal window whose power is missing, +non-finite, or <= 0 are teardown noise: they are skipped and counted in the +sidecar's ``boundary_degenerate_rows`` instead of poisoning validity or faking +window bracketing. """ from __future__ import annotations @@ -22,7 +26,7 @@ import os import re import sys -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from statistics import mean @@ -65,6 +69,10 @@ class PowerIntegration: per_gpu_max_sample_gap_s: dict[str, float] per_gpu_energy_j: dict[str, float] device_issues: dict[str, list[str]] + # Rows in the ingest band but outside the formal window whose power was + # missing/N-A/non-finite/<=0, skipped and counted per GPU; "unknown" + # buckets rows without a GPU identity. + boundary_degenerate_rows: dict[str, int] = field(default_factory=dict) avg_power_w: float | None = None p75_power_w: float | None = None p75_total_gpu_power_w: float | None = None @@ -78,7 +86,7 @@ def observed_num_gpus(self) -> int: return len(self.observed_gpu_ids) -def _parse_timestamp(value: str) -> float | None: +def _parse_timestamp(value: str, *, naive_timezone: timezone | None = None) -> float | None: """Best-effort timestamp parse to Unix epoch seconds (local wall clock). Handles the formats observed in practice: @@ -96,7 +104,7 @@ def _parse_timestamp(value: str) -> float | None: # nvidia-smi: "YYYY/MM/DD HH:MM:SS.ffffff" for fmt in ("%Y/%m/%d %H:%M:%S.%f", "%Y/%m/%d %H:%M:%S"): try: - return datetime.strptime(value, fmt).timestamp() + return datetime.strptime(value, fmt).replace(tzinfo=naive_timezone).timestamp() except ValueError: pass # ISO 8601 (amd-smi variants). fromisoformat tolerates 'T' or space separator @@ -108,7 +116,7 @@ def _parse_timestamp(value: str) -> float | None: return None if dt.tzinfo is None: # Treat naive timestamps as local time (matches nvidia-smi convention). - return dt.timestamp() + return dt.replace(tzinfo=naive_timezone).timestamp() return dt.astimezone(timezone.utc).timestamp() @@ -146,6 +154,16 @@ def _detect_columns(header: list[str]) -> tuple[str | None, str | None, str | No return timestamp_col, power_col, gpu_col +def _telemetry_timezone(csv_path: Path) -> timezone | None: + context = csv_path.with_name(f"{csv_path.stem}_context.json") + if not context.is_file(): + return None + payload = json.loads(context.read_text()) + if not isinstance(payload, dict) or payload.get("timestamp_timezone") != "UTC": + raise ValueError("unsupported_telemetry_timezone") + return timezone.utc + + def aggregate_power( csv_path: Path, start_unix: float, @@ -162,6 +180,7 @@ def aggregate_power( return None try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [c.strip() for c in (reader.fieldnames or [])] @@ -189,7 +208,7 @@ def aggregate_power( for row in reader: ts_raw = (row.get(timestamp_col) or "").strip() pw_raw = (row.get(power_col) or "").strip() - ts = _parse_timestamp(ts_raw) + ts = _parse_timestamp(ts_raw, naive_timezone=timestamp_timezone) pw = _parse_power(pw_raw) if ts is None or pw is None: continue @@ -204,7 +223,7 @@ def aggregate_power( if gpu_id: per_sample_gpus.setdefault(bucket, set()).add(gpu_id) gpu_keys.add(gpu_id) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): return None if not per_sample_total: @@ -237,6 +256,7 @@ def _empty_integration( *, expected_num_gpus: int | None, reasons: list[str], + boundary_degenerate_rows: dict[str, int] | None = None, ) -> PowerIntegration: """Build an invalid integration result when no device data is available.""" return PowerIntegration( @@ -248,6 +268,7 @@ def _empty_integration( per_gpu_max_sample_gap_s={}, per_gpu_energy_j={}, device_issues={}, + boundary_degenerate_rows=boundary_degenerate_rows or {}, ) @@ -308,8 +329,10 @@ def integrate_power( # expose timestamps at lower resolution than their sampling cadence, so # duplicate-timestamp readings are averaged rather than treated as corrupt. raw_samples: dict[str, dict[float, list[float]]] = {} + boundary_degenerate: dict[str, int] = {} saw_missing_gpu_identity = False try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -332,7 +355,7 @@ def integrate_power( ) for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) if timestamp is None or not math.isfinite(timestamp): _append_reason(reasons, "invalid_timestamp_sample") continue @@ -348,6 +371,15 @@ def integrate_power( power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() + if (power is None or not math.isfinite(power) or power <= 0.0) and ( + timestamp < start_unix or timestamp > end_unix + ): + # SMI teardown rows can carry N/A or 0 W cells: outside the + # formal window they are counted, never used to satisfy + # bracketing or to poison in-window validity. + key = gpu_id or "unknown" + boundary_degenerate[key] = boundary_degenerate.get(key, 0) + 1 + continue if power is None: _append_reason(reasons, "invalid_power_sample") continue @@ -359,11 +391,12 @@ def integrate_power( continue values = raw_samples.setdefault(gpu_id, {}).setdefault(timestamp, []) values.append(power) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): _append_reason(reasons, "telemetry_file_unreadable") return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) if saw_missing_gpu_identity: @@ -373,6 +406,7 @@ def integrate_power( return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) observed_gpu_ids = tuple(sorted(raw_samples, key=_gpu_sort_key)) @@ -450,6 +484,7 @@ def integrate_power( per_gpu_max_sample_gap_s=per_gpu_max_sample_gap_s, per_gpu_energy_j=per_gpu_energy_j, device_issues=device_issues, + boundary_degenerate_rows=boundary_degenerate, avg_power_w=avg_power_w, p75_power_w=p75_total / len(observed_gpu_ids) if p75_total is not None else None, p75_total_gpu_power_w=p75_total, @@ -485,6 +520,7 @@ def _read_energy_snapshot(path: Path) -> dict[str, float] | None: def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float]]]: """Group every parseable (timestamp, watt) sample per GPU, sorted in time.""" samples: dict[str, list[tuple[float, float]]] = {} + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -493,7 +529,7 @@ def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float] if not timestamp_col or not power_col or not gpu_col: return {} for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() if timestamp is None or power is None or not gpu_id: @@ -679,6 +715,7 @@ def _validation_payload( "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s, "per_gpu_energy_j": integration.per_gpu_energy_j, "device_issues": integration.device_issues, + "boundary_degenerate_rows": integration.boundary_degenerate_rows, "accumulator_check": accumulator_check, "metrics": audit_metrics(metrics), } @@ -754,7 +791,7 @@ def run( try: accumulator_check = cross_check_accumulator(csv_path) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): accumulator_check = {"available": False, "reason": "cross_check_error"} try: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f3b85de24..6d321aec8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7410,3 +7410,14 @@ - "Preserve request failure outcomes, retain all available fixed-sequence diagnostics, and reject incomplete concurrency batches before publishing benchmark artifacts." - "保留请求失败状态及全部可用的固定序列诊断信息；发布基准产物前拒绝缺失并发点的批次。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Retain UTC SMI telemetry context and validate native multinode collector artifacts without enabling new recipes." + - "保留 UTC SMI 遥测上下文并验证原生多节点采集产物，不启用新 recipe。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3051 diff --git a/runners/test_native_collector_barriers.py b/runners/test_native_collector_barriers.py new file mode 100644 index 000000000..14c8fe2ce --- /dev/null +++ b/runners/test_native_collector_barriers.py @@ -0,0 +1,58 @@ +"""CPU-only lifecycle checks with controlled external collector/Slurm processes.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +LIFECYCLE = ROOT / 'benchmarks/native_power_lifecycle.sh' +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' + + +@pytest.mark.parametrize('failed_rank', [None, 1]) +def test_stop_waits_for_every_collector_and_retains_failure(tmp_path, failed_rank): + command = ''' +source "$1" +export POWERX_CONTROL_DIR="$2" POWERX_NUM_NODES=2 POWERX_BARRIER_TIMEOUT_S=5 +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.1; echo 0 > "$2/done-0") & +POWERX_COLLECTOR_PID=$! +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.2; echo "$3" > "$2/done-1") & +remote_pid=$! +powerx_stop_collectors +rc=$? +wait "$remote_pid" +exit "$rc" +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(LIFECYCLE), + str(tmp_path), '1' if failed_rank else '0'], + capture_output=True, text=True, timeout=10) + assert result.returncode == (1 if failed_rank else 0), result.stderr + assert (tmp_path / 'done-0').read_text().strip() == '0' + assert (tmp_path / 'done-1').read_text().strip() == ('1' if failed_rank else '0') + + +def test_ready_barrier_rejects_collector_that_already_stopped(tmp_path): + (tmp_path / 'ready-0').write_text('ready') + (tmp_path / 'done-0').write_text('1') + result = subprocess.run(['bash', '-c', 'source "$1"; POWERX_CONTROL_DIR="$2"; ' + 'POWERX_NUM_NODES=1; powerx_wait_collectors ready', + 'bash', str(LIFECYCLE), str(tmp_path)], capture_output=True, text=True) + assert result.returncode == 1 + assert 'stopped before benchmark readiness' in result.stderr + + +def test_control_receipt_is_owned_before_publication(tmp_path): + command = '''source "$1" +POWERX_CONTROL_DIR=$2 +POWERX_HOST_UID=1000 POWERX_HOST_GID=1000 +chown() { [[ ! -e "$POWERX_CONTROL_DIR/stop" ]]; } +powerx_write_control stop stop +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(LIFECYCLE), str(tmp_path)], + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert (tmp_path / 'stop').read_text() == 'stop\n' + assert not list(tmp_path.glob('*.tmp')) diff --git a/runners/test_native_collector_receipts.py b/runners/test_native_collector_receipts.py new file mode 100644 index 000000000..b981f57ce --- /dev/null +++ b/runners/test_native_collector_receipts.py @@ -0,0 +1,16 @@ +import subprocess +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] + +def test_native_control_receipt_is_owned_before_publication(tmp_path): + source = (ROOT / 'benchmarks/native_power_collect.sh').read_text() + function = source[source.index('write_control() {'):source.index('\nfinish() {')] + result = subprocess.run(['bash', '-c', function + ''' +control_dir=$1 +POWERX_HOST_UID=1000 POWERX_HOST_GID=1000 +chown() { [[ ! -e "$control_dir/done-0" ]]; } +write_control done-0 7 +''', 'bash', str(tmp_path)], capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert (tmp_path / 'done-0').read_text() == '7\n' + assert not list(tmp_path.glob('*.tmp')) diff --git a/utils/test_aggregate_power.py b/utils/test_aggregate_power.py index aa3bf23f9..5b8b76121 100644 --- a/utils/test_aggregate_power.py +++ b/utils/test_aggregate_power.py @@ -574,6 +574,201 @@ def test_run_rejects_malformed_telemetry_inside_window( assert audit["reasons"] == [expected_reason] + +# AMDSMI 26.2.0 `metric -p -c -t -u -w 1 --csv` header (order-faithful subset, +# measured on MI355X; see test_detect_columns_amd_watch_mode_real_header). +_MI355X_WATCH_HEADER = ( + "timestamp,gpu,gfx_activity,umc_activity,mm_activity,vcn_activity," + "jpeg_activity,gfx_busy_inst_xcp_0,jpeg_busy_xcp_0,vcn_busy_xcp_0," + "socket_power,gfx_voltage,soc_voltage,mem_voltage,throttle_status," + "power_management,gfx_0_clk,mem_0_clk,edge,hotspot,mem" +) + + +def _mi355x_watch_row(timestamp: int, gpu: int, socket_power: str) -> str: + """One data row in the shape captured from run 32433563482 (conc1): + integer-second epoch, quoted list cells with embedded commas, N/A cells, + and a trailing carriage return.""" + return ( + f"{timestamp},{gpu},0,0,N/A,\"['N/A', 'N/A', 'N/A', 'N/A']\"," + "\"['N/A', 'N/A']\",\"[0, 0, 0, 0, 0, 0, 0, 0]\",\"[0, 0]\",\"[0, 0]\"," + f"{socket_power},N/A,N/A,N/A,N/A,ENABLED,1404,2000,N/A,40,25\r" + ) + + +def test_integrate_power_skips_na_power_rows_outside_window(tmp_path: Path): + """N/A-power teardown rows past the window end are counted, not poisonous.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-2, 13): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},N/A,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + assert result.total_gpu_energy_j == pytest.approx(10_000.0) + + +def test_integrate_power_does_not_bracket_with_zero_power_tail(tmp_path: Path): + """A 0 W teardown row past the window end must not fake end bracketing. + + Legacy behavior accepted the 0 W row as a valid boundary sample, corrupting + the end interpolation with a bogus value; this intentionally flips that + case to an explicit benchmark_window_not_bracketed failure.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + # Good rows stop at end-4; the only post-end sample per GPU has power 0. + for offset in range(-1, 7): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},0,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "benchmark_window_not_bracketed" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + + +def test_integrate_power_preserves_boundary_counts_without_usable_samples(tmp_path: Path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text( + "timestamp,gpu,socket_power\n" + "1699999999,0,N/A\n" + "1700000011,0,0\n" + "1700000011,1,N/A\n", + encoding="utf-8", + ) + + result = integrate_power( + csv, + start_unix=1_700_000_000, + end_unix=1_700_000_010, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "no_usable_power_samples" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 2, "1": 1} + + +def test_integrate_power_keeps_zero_power_semantics_inside_window(tmp_path: Path): + """Frozen legacy behavior: an in-window 0 W sample still integrates.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-1, 12): + watts = "0.0" if offset == 5 else "500.0" + lines.append(f"{base + offset},0,{watts},65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=1, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {} + # Trapezoids dip to 0 at t=5: 8 x 500 + 2 x 250 = 4500 J. + assert result.total_gpu_energy_j == pytest.approx(4_500.0) + + +def test_run_writes_boundary_degenerate_rows_to_sidecar(tmp_path: Path): + base = 1_700_000_000.0 + csv = tmp_path / "gpu_metrics.csv" + _write_constant_window_samples( + csv, + start=base, + end=base + 10, + watts_per_gpu=500.0, + num_gpus=2, + ) + bench = tmp_path / "bench.json" + agg = tmp_path / "agg.json" + validation = tmp_path / "power_validation.json" + _write_bench_result( + bench, + start=base, + end=base + 10, + duration=10.0, + total_output=2_000, + total_input=10_000, + ) + agg.write_text(json.dumps({"hw": "mi355x"}), encoding="utf-8") + + exit_code = run(csv, bench, agg, expected_num_gpus=2, validation_result=validation) + + assert exit_code == 0 + audit = json.loads(validation.read_text()) + # Present-and-empty for clean streams: readers can rely on the key. + assert audit["boundary_degenerate_rows"] == {} + + +def test_integrate_power_regression_mi355x_integer_ticks_end_gap(tmp_path: Path): + """Run-32433563482 conc1 regression: amd-smi integer-second ticks stop 4 s + before the fractional aiperf window end (last tick 1787277605 vs end + ...609.157497), so bracketing fails and the producer-side telemetry loss is + attributed as benchmark_window_not_bracketed on every GPU. + + The retrieved artifact's trailing rows all carry valid socket_power + (254-264 W) with N/A activity/voltage cells; the N/A- and 0-power teardown + rows appended past the window end are the documented synthetic degenerate + shapes, asserting they are counted rather than used for bracketing.""" + csv = tmp_path / "gpu_metrics.csv" + start = 1_787_277_560.155891 + end = 1_787_277_609.157497 + last_tick = 1_787_277_605 + powers = [259, 255, 263, 264, 256, 254, 259, 259] + lines = [_MI355X_WATCH_HEADER] + for tick in range(1_787_277_555, last_tick + 1): + for gpu in range(8): + lines.append(_mi355x_watch_row(tick, gpu, str(powers[gpu]))) + # amd-smi watch mode emits a blank line between tick groups. + lines.append("") + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_610, gpu, "N/A")) + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_611, gpu, "0")) + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=start, + end_unix=end, + expected_num_gpus=8, + ) + + assert result.power_valid is False + assert result.invalid_reasons == ("benchmark_window_not_bracketed",) + assert result.device_issues == { + str(gpu): ["benchmark_window_not_bracketed"] for gpu in range(8) + } + assert result.boundary_degenerate_rows == {str(gpu): 2 for gpu in range(8)} + + def test_run_patches_agg_with_power_and_joules(tmp_path: Path): base = 1_700_000_000.0 csv = tmp_path / "gpu_metrics.csv" @@ -708,6 +903,7 @@ def test_power_replacement_removes_stale_metrics(tmp_path, patch_validated_power path.write_text(json.dumps({ "hw": "fixture", "avg_power_w": 99, "total_gpu_energy_j": 50, "power_invalid_reasons": ["stale"], + "power_audit": {"source": "previous-run.json"}, })) patch_validated_power(path, power_valid=valid, metrics={ "avg_power_w": 12.34567, "joules_per_output_token": 0.12345678, @@ -1433,12 +1629,12 @@ def test_power_percentiles_uses_synchronized_total_not_device_percentiles(tmp_pa def test_power_percentiles_weights_time_and_clips_the_validated_window(tmp_path): csv_path = tmp_path / "power.csv" - # Dense readings near the high end must not bias a uniform linear ramp. - _write_amd_csv(csv_path, [(0, 0, 0), (1, 0, 100), (1.9, 0, 190), (2, 0, 200)]) + # Dense readings must not bias the uniform 150-250 W ramp inside the window. + _write_amd_csv(csv_path, [(0, 0, 100), (1, 0, 200), (1.9, 0, 290), (2, 0, 300)]) result = integrate_power(csv_path, start_unix=0.5, end_unix=1.5, expected_num_gpus=1) assert result.power_valid - assert result.p75_power_w == pytest.approx(125) - assert result.p90_power_w == pytest.approx(140) + assert result.p75_power_w == pytest.approx(225) + assert result.p90_power_w == pytest.approx(240) def test_power_percentiles_is_withheld_for_invalid_telemetry(tmp_path): @@ -1462,3 +1658,54 @@ def test_power_percentiles_aligns_asynchronous_gpu_samples(tmp_path): assert result.p75_power_w == pytest.approx(300) assert result.p90_total_gpu_power_w == pytest.approx(600) assert result.p90_power_w == pytest.approx(300) + + +@pytest.mark.parametrize('step_name', ['Upload GPU metrics', 'Upload power audit bundle']) +@pytest.mark.parametrize('directory', ['', 'results']) +def test_uploaded_telemetry_replays_in_a_different_timezone(tmp_path, step_name, directory): + import os + import shutil + import yaml + + repo = Path(__file__).resolve().parents[1] + source = tmp_path / 'source' + telemetry = source / directory + telemetry.mkdir(parents=True) + (telemetry / 'gpu_metrics.csv').write_text( + 'timestamp,index,power.draw [W]\n' + '2024/01/01 00:00:00.000,0,100 W\n' + '2024/01/01 00:00:01.000,0,100 W\n' + '2024/01/01 00:00:02.000,0,100 W\n') + (telemetry / 'gpu_metrics_context.json').write_text('{"timestamp_timezone":"UTC"}') + workflow = yaml.safe_load((repo / '.github/workflows/benchmark-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job['steps'] + if step.get('name') == step_name) + archive = tmp_path / 'downloaded' + for pattern in step['with']['path'].splitlines(): + for file in source.glob(pattern): + destination = archive / file.relative_to(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(file, destination) + result = subprocess.run( + [sys.executable, '-c', """ +import sys, time +from pathlib import Path +from infx.results.power.single_node import integrate_power +time.tzset() +result = integrate_power(Path(sys.argv[1]), start_unix=1704067200, end_unix=1704067202, + expected_num_gpus=1) +assert result.power_valid, result.invalid_reasons +assert result.total_gpu_energy_j == 200 +""", str(archive / directory / 'gpu_metrics.csv')], + cwd=tmp_path, env={**os.environ, 'TZ': 'Etc/GMT+8', 'PYTHONPATH': str(repo)}, + capture_output=True, text=True, timeout=10, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize('context', ['{"timestamp_timezone":"PST"}', '{}', '{invalid']) +def test_legacy_average_rejects_invalid_telemetry_context(tmp_path, context): + csv = tmp_path / 'gpu_metrics.csv' + csv.write_text('timestamp,index,power.draw [W]\n1,0,100\n2,0,100\n') + csv.with_name('gpu_metrics_context.json').write_text(context) + assert aggregate_power(csv, 1, 2) is None diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py new file mode 100644 index 000000000..0aeee2511 --- /dev/null +++ b/utils/test_native_multinode_power.py @@ -0,0 +1,201 @@ +"""Native collector acceptance uses independent, hand-computed two-node traces.""" +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from infx.results.power.native_multinode import record_begin, record_end, run +from infx.results.power.single_node import integrate_power + +REPO = Path(__file__).resolve().parents[1] + + +def _package(tmp_path, vendor="amd"): + root = tmp_path / "native_power" + for rank, role, watts in ((0, "prefill", 100), (1, "decode", 300)): + node = root / f"node-{rank}" + record_begin(node, vendor=vendor, node=f"host-{rank}", rank=rank, role=role, + gpu_indices=[0], num_nodes=2, job_id="job-123", revision="revision-abc", + clock_synchronized=True) + record_end(node, collector_exit_code=0) + manifest = json.loads((node / "manifest.json").read_text()) + manifest.update(collection_start_unix=0, collection_end_unix=5) + (node / "manifest.json").write_text(json.dumps(manifest)) + for ending in ("", "_end"): + if vendor == "amd": + (node / f"gpu_metrics_devices{ending}.json").write_text(json.dumps([ + {"gpu": 0, "uuid": f"uuid-{rank}"}, {"gpu": 1, "uuid": f"unused-{rank}"}])) + else: + (node / f"gpu_metrics_identity{ending}.csv").write_text( + f"index, uuid, pci.bus_id\n0, uuid-{rank}, 0000:01:00.0\n1, unused-{rank}, 0000:02:00.0\n") + # The spare physical GPU is visible but does not belong to the server. + (node / "gpu_metrics.csv").write_text("timestamp,gpu,power\n" + "".join( + f"{tick},0,{watts}\n{tick},1,900\n" for tick in range(5))) + bench = tmp_path / "result.json" + bench.write_text(json.dumps({"benchmark_start_time_unix": 1, "benchmark_end_time_unix": 3, + "duration": 2, "completed": 2, "total_input_tokens": 20, + "total_output_tokens": 10})) + agg = tmp_path / "agg.json" + agg.write_text(json.dumps({"avg_power_w": 999, "prefill_gpu_energy_j": 999})) + return root, bench, agg + + +@pytest.mark.parametrize("vendor", ["amd", "nvidia"]) +def test_native_whole_fleet_and_role_energy_use_only_serving_devices(tmp_path, vendor): + root, bench, agg = _package(tmp_path, vendor) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 1 + assert actual["total_gpu_energy_j"] == 800 + assert actual["avg_power_w"] == 200 + assert actual["p90_power_w"] == 200 + assert actual["p75_power_w"] == 200 + assert actual["joules_per_successful_query"] == 400 + assert actual["prefill_gpu_energy_j"] == 200 + assert actual["decode_gpu_energy_j"] == 600 + assert actual["prefill_joules_per_input_token"] == 10 + assert actual["decode_joules_per_output_token"] == 60 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert audit["observed_gpu_count"] == 2 + assert audit["per_gpu_role"] == {"uuid-0": "prefill", "uuid-1": "decode"} + assert len(audit["nodes"][0]["telemetry_sha256"]) == 64 + from infx.results.power.audit import audit_summary + summary = audit_summary(audit, "power_validation_result.json")["power_audit"] + assert summary["sample_count"] == 10 # Five samples on each of two participating GPUs. + assert summary["producer_sha"] == "revision-abc" + + +def test_native_parse_failure_has_a_public_reason_and_retained_detail(tmp_path): + from infx.results.power.audit import audit_summary + root, bench, agg = _package(tmp_path) + (root / "node-1/manifest.json").write_text("not JSON") + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + summary = audit_summary(audit, "power_validation_result.json") + assert "native_node_invalid" in summary["power_invalid_reasons"] + assert audit["node_errors"][0]["node"] == "node-1" + assert audit["node_errors"][0]["detail"] + + +@pytest.mark.parametrize(("field", "value", "reason"), [ + ("clock_synchronized", False, "native_clock_not_synchronized"), + ("lifecycle", "collecting", "native_collector_incomplete"), + ("collection_end_unix", 2, "native_collection_window_mismatch"), + ("job_id", "other-job", "native_run_identity_mismatch"), + ("role", "prefill", "native_role_gpu_count_mismatch"), + ("expected_num_nodes", 3, "native_node_topology_mismatch"), +]) +def test_native_invalid_evidence_clears_stale_metrics_and_writes_audit(tmp_path, field, value, reason): + root, bench, agg = _package(tmp_path) + path = root / "node-1/manifest.json" + manifest = json.loads(path.read_text()); manifest[field] = value + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 0 + assert "avg_power_w" not in actual + assert "prefill_gpu_energy_j" not in actual + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert reason in audit["reasons"] + + +def test_native_device_replacement_cannot_preserve_validity(tmp_path): + root, bench, agg = _package(tmp_path) + (root / "node-1/gpu_metrics_devices_end.json").write_text('[{"gpu":0,"uuid":"replacement"}]') + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + assert "native_device_identity_changed" in json.loads((tmp_path / "power_validation_result.json").read_text())["reasons"] + + +def test_native_aggregate_nodes_do_not_invent_prefill_decode_metrics(tmp_path): + root, bench, agg = _package(tmp_path) + for path in root.glob("*/manifest.json"): + manifest = json.loads(path.read_text()); manifest["role"] = "aggregate" + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=0, expected_decode_gpus=0, + expected_aggregate_gpus=2, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["avg_power_w"] == 200 + assert "prefill_gpu_energy_j" not in actual + + +def test_utc_context_replays_in_a_different_timezone(tmp_path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text("timestamp,index,power.draw [W]\n2026/01/01 00:00:00,0,100\n2026/01/01 00:00:02,0,100\n") + (tmp_path / "gpu_metrics_context.json").write_text('{"timestamp_timezone":"UTC"}') + script = "from pathlib import Path; from infx.results.power.single_node import integrate_power; " + \ + f"r=integrate_power(Path({str(csv)!r}),start_unix=1767225600,end_unix=1767225602,expected_num_gpus=1); assert r.power_valid; assert r.total_gpu_energy_j == 200" + subprocess.run([sys.executable, "-c", script], cwd=REPO, + env={**os.environ, "TZ": "America/Los_Angeles"}, check=True, timeout=10) + + +@pytest.mark.parametrize("clock_value, synchronized", [ + ("yes", True), ("true", True), ("no", False), ("false", False), ("", False), +]) +def test_native_supervisor_reaps_monitor_and_writes_completion(tmp_path, clock_value, synchronized): + binary = tmp_path / "bin"; binary.mkdir() + (binary / "python3").symlink_to(sys.executable) + fake = binary / "nvidia-smi" + fake.write_text(f'''#!{sys.executable} +import datetime, os, sys, time +if any("index,uuid" in arg for arg in sys.argv): + print("index, uuid, pci.bus_id"); print("0, gpu-0, 0000:01:00.0") +else: + with open({str(tmp_path / 'monitor.pid')!r}, "w") as stream: stream.write(str(os.getpid())) + if "-l" in sys.argv: print("timestamp,index,power.draw [W]", flush=True) + while True: + print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y/%m/%d %H:%M:%S.%f") + ",0,100", flush=True) + if "-l" not in sys.argv: break + time.sleep(0.1) +''') + fake.chmod(0o755) + control = tmp_path / "control"; control.mkdir() + node = tmp_path / "node-0" + process = subprocess.Popen(["bash", str(REPO / "benchmarks/native_power_collect.sh"), + str(node), str(control), "nvidia", "0", "aggregate", "0", "1"], + env={**os.environ, "PATH": f"{binary}:{os.environ['PATH']}", + "SLURM_JOB_ID": "test-job", "POWERX_COLLECTOR_REVISION": "revision", + "POWERX_CLOCK_SYNCHRONIZED": clock_value}, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + try: + deadline = time.monotonic() + 10 + while not (control / "ready-0").exists(): + assert process.poll() is None + assert time.monotonic() < deadline + time.sleep(0.02) + (control / "stop").write_text("stop") + stdout, stderr = process.communicate(timeout=10) + assert process.returncode == 0, (stdout, stderr) + assert (control / "done-0").read_text().strip() == "0" + manifest = json.loads((node / "manifest.json").read_text()) + assert manifest["lifecycle"] == "complete" + assert manifest["clock_synchronized"] is synchronized + assert (node / "gpu_metrics_identity_end.csv").exists() + finally: + if process.poll() is None: + process.terminate(); process.communicate(timeout=10) + + +def test_result_processor_discovers_staged_native_package(tmp_path, monkeypatch): + from infx.results.fixed_sequence import aggregate_power_result + + root, bench, agg = _package(tmp_path) + logs = tmp_path / "LOGS" + logs.mkdir() + root.rename(logs / "native_power") + monkeypatch.chdir(tmp_path) + env = {"IS_MULTINODE": "true", "PREFILL_GPUS": "1", "DECODE_GPUS": "1", + "RESULT_FILENAME": "result", "REQUIRE_POWER": "1"} + assert aggregate_power_result(env, bench, agg) == 0 + assert json.loads(agg.read_text())["total_gpu_energy_j"] == 800 + assert (tmp_path / "power_validation_result.json").is_file() + + # Two competing formats must not silently select one package. + (logs / "power").mkdir() + assert aggregate_power_result(env, bench, agg) == 1 + invalid = json.loads(agg.read_text()) + assert invalid["power_valid"] == 0 + assert "total_gpu_energy_j" not in invalid From f691e7e7d72b49bd3db08ddbf89f414ddb8cb620 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:30:16 -0700 Subject: [PATCH 03/17] fix: retain per-window SMI context artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留每个测量窗口的 SMI 上下文工件，覆盖原 #3027 的共享上传规则。 --- .github/workflows/benchmark-tmpl.yml | 8 ++++---- utils/test_process_result.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 1da933f63..632401822 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -445,13 +445,13 @@ jobs: name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }} path: | gpu_metrics.csv - gpu_metrics_context.json + gpu_metrics*_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv results/gpu_metrics*.csv - results/gpu_metrics_context.json + results/gpu_metrics*_context.json results/gpu_metrics_identity.json if-no-files-found: ignore @@ -464,14 +464,14 @@ jobs: ${{ env.RESULT_FILENAME }}.json agg_${{ env.RESULT_FILENAME }}.json gpu_metrics.csv - gpu_metrics_context.json + gpu_metrics*_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json results/gpu_metrics*.csv - results/gpu_metrics_context.json + results/gpu_metrics*_context.json results/gpu_metrics_identity.json results/agentic_power_window.json results/agentic_power_timezone_offset.txt diff --git a/utils/test_process_result.py b/utils/test_process_result.py index d14d6a87a..7bc8dc4d9 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -1623,3 +1623,19 @@ def test_public_power_audit_bounds_text_and_device_identifiers(): assert audit['exporter_image_sha256'] == 'a' * 64 assert len(audit['observed_gpu_ids']) == 1024 assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] + + +@pytest.mark.parametrize('step_name', ['Upload GPU metrics', 'Upload power audit bundle']) +def test_workflow_retains_context_for_each_metrics_csv(tmp_path, step_name): + import yaml + + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == step_name) + patterns = step['with']['path'].splitlines() + for relative in ['gpu_metrics_concurrency_4_context.json', + 'results/gpu_metrics_concurrency_4_context.json']: + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('{"timestamp_timezone":"UTC"}') + assert any(path in tmp_path.glob(pattern.strip()) for pattern in patterns) From ec50eb154d2f20e68c7cd7c5986d07753083ea04 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:26:05 -0700 Subject: [PATCH 04/17] fix: isolate AMD monitor windows and worker power lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 隔离 AMD 监控窗口与工作节点功耗生命周期，保留可选采集故障证据。 --- benchmarks/benchmark_lib.sh | 171 +++++++- benchmarks/multi_node/amd_utils/bench.sh | 21 +- benchmarks/multi_node/amd_utils/job.slurm | 39 +- benchmarks/multi_node/amd_utils/power.sh | 58 +++ benchmarks/multi_node/amd_utils/server.sh | 3 + .../multi_node/amd_utils/server_atom.sh | 5 +- .../multi_node/amd_utils/server_sglang.sh | 5 +- .../multi_node/amd_utils/server_vllm.sh | 5 +- docs/configuration-procedures.md | 8 + docs/configuration-procedures_zh.md | 8 + docs/results-and-ingestion.md | 4 + docs/results-and-ingestion_zh.md | 4 + perf-changelog.yaml | 85 ++++ runners/launch_mi355x-amds.sh | 11 + runners/test_amd_monitor_wiring.py | 65 +++ runners/test_amd_power_lifecycle.py | 41 ++ .../aggregation/test_power_lifecycle.py | 404 +++++++++++++++++- utils/test_process_result.py | 20 +- 18 files changed, 922 insertions(+), 35 deletions(-) create mode 100644 benchmarks/multi_node/amd_utils/power.sh create mode 100644 runners/test_amd_monitor_wiring.py create mode 100644 runners/test_amd_power_lifecycle.py diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 9d99a4cc9..88a2e744a 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -153,8 +153,12 @@ unset _benchmark_caller # -------------------------------- GPU_MONITOR_PID="" +GPU_MONITOR_SOURCE_PID="" +GPU_MONITOR_PIPE="" GPU_MONITOR_VENDOR="" GPU_MONITOR_INTERVAL=1 +# Bounded wait for AMD telemetry to cover a stop request; 0 skips the wait. +AMD_MONITOR_STOP_TIMEOUT_S="${AMD_MONITOR_STOP_TIMEOUT_S:-30}" GPU_METRICS_CSV="${GPU_METRICS_CSV:-gpu_metrics.csv}" NVIDIA_GPU_MONITOR_QUERY="timestamp,index,power.draw,temperature.gpu,clocks.current.sm,clocks.current.memory,utilization.gpu,utilization.memory" export GPU_METRICS_CSV @@ -197,8 +201,24 @@ start_gpu_monitor() { # Python; measured on MI355X: trailing ticks were lost at kill without it). # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated # headers, and flush every row so killing the pipe cannot discard buffered samples. - TZ=UTC PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ - | awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" & + # Track both processes: killing only awk can leave amd-smi alive until + # its next write. Keep the FIFO beside this run's raw CSV, never shared. + GPU_MONITOR_PIPE="${output}.pipe.$$" + if ! mkfifo "$GPU_MONITOR_PIPE"; then + echo '[GPU Monitor] Warning: AMD telemetry FIFO is unavailable' >&2 + # A colliding path may belong to another stream; teardown must not remove it. + GPU_MONITOR_PIPE="" + GPU_MONITOR_VENDOR="" + case "${REQUIRE_POWER:-0}" in + 1|true|TRUE|yes|YES) return 1 ;; + esac + return 0 + fi + PYTHONUNBUFFERED=1 TZ=UTC amd-smi metric -p -c -t -u -w "$interval" --csv \ + > "$GPU_MONITOR_PIPE" 2>/dev/null & + GPU_MONITOR_SOURCE_PID=$! + awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' \ + < "$GPU_MONITOR_PIPE" > "$output" & GPU_MONITOR_PID=$! # Hardware energy-accumulator + identity snapshots; the end-side twin in # stop_gpu_monitor lets auditors cross-check the integrated energy @@ -216,19 +236,21 @@ start_gpu_monitor() { # Stop the background GPU monitor and report file size. stop_gpu_monitor() { if [[ -n "$GPU_MONITOR_PID" ]] && kill -0 "$GPU_MONITOR_PID" 2>/dev/null; then - # benchmark_end_time_unix is recorded shortly before the benchmark - # process exits, so the stream must cover one more sample past it for - # deterministic boundary interpolation. NVIDIA appends a one-shot - # post-exit sample below; amd-smi one-shot CSV has no timestamp column, - # so the AMD path instead lets the watch stream emit final ticks before - # the kill. Two extra intervals: amd-smi stamps integer seconds, so a - # tick in the same second as the window end still fails bracketing — - # the stream needs a tick at the NEXT whole second (measured on MI355X: - # end=...153.325 vs last sample ...153.0). + # The aggregator requires, for every GPU, a usable sample stamped at or + # after the (fractional) benchmark window end, which is always <= the + # wall clock when this stop runs. NVIDIA appends a one-shot post-exit + # sample below; amd-smi one-shot CSV has no timestamp column, so the + # AMD path polls the output file until every GPU's watch stream shows + # a usable tick at the next whole second — amd-smi stamps integer + # seconds, so that tick strictly covers any fractional window end + # (measured on MI355X: end=...609.157 vs last sample ...605). Observing + # the file rather than sleeping also defeats pipe-buffer loss when the + # awk consumer is killed: covered rows are already on disk. if [[ "$GPU_MONITOR_VENDOR" == "amd" ]]; then - sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 )) + _wait_for_amd_stop_coverage fi - kill "$GPU_MONITOR_PID" 2>/dev/null + # The monitor may exit during the coverage wait; still finish cleanup. + kill "$GPU_MONITOR_PID" 2>/dev/null || true wait "$GPU_MONITOR_PID" 2>/dev/null || true case "$GPU_MONITOR_VENDOR" in nvidia) @@ -250,6 +272,13 @@ stop_gpu_monitor() { echo "[GPU Monitor] Collected $lines rows -> $GPU_METRICS_CSV" fi fi + if [[ -n "$GPU_MONITOR_SOURCE_PID" ]]; then + kill "$GPU_MONITOR_SOURCE_PID" 2>/dev/null || true + wait "$GPU_MONITOR_SOURCE_PID" 2>/dev/null || true + fi + [[ -z "$GPU_MONITOR_PIPE" ]] || rm -f "$GPU_MONITOR_PIPE" + GPU_MONITOR_SOURCE_PID="" + GPU_MONITOR_PIPE="" GPU_MONITOR_PID="" GPU_MONITOR_VENDOR="" } @@ -272,6 +301,105 @@ _repair_truncated_gpu_metrics_tail() { return 0 } +# Print the newest telemetry tick (whole epoch seconds) that EVERY observed +# GPU has covered with a usable sample (numeric epoch timestamp, numeric +# power > 0), or nothing when the stream holds no usable epoch-stamped row +# (e.g. an amd-smi build emitting ISO timestamps). Column detection mirrors +# _POWER_COL_RE/_POWER_EXCLUDE_RE/_GPU_INDEX_COL_RE in utils/aggregate_power.py. +# POSIX awk only: the ROCm container images ship mawk/busybox awk. +_amd_monitor_min_covered_tick() { + [[ -f "$GPU_METRICS_CSV" ]] || return 0 + awk -F, ' + NR == 1 { + for (i = 1; i <= NF; i++) { + name = tolower($i) + gsub(/^ +| +$/, "", name) + sub(/\r$/, "", name) + if (!power_col && name ~ /power/ && name !~ /limit|cap|max|min/) + power_col = i + if (!gpu_col && name ~ /^(index|gpu|gpu_id|gpu_index|card|device)$/) + gpu_col = i + } + next + } + !power_col || !gpu_col { next } + { + # amd-smi quotes list-valued cells that embed commas; neutralize + # them so the power cell keeps its header-relative position. + line = $0 + sub(/\r$/, "", line) + if (line ~ /"/) { + n = split(line, seg, /"/) + line = "" + for (i = 1; i <= n; i++) { + if (i % 2 == 0) gsub(/,/, ";", seg[i]) + line = line seg[i] + } + } + count = split(line, cell, /,/) + if (count < power_col || count < gpu_col) next + if (cell[1] !~ /^[0-9]+(\.[0-9]+)?$/) next + if (cell[power_col] !~ /^[0-9]+(\.[0-9]+)?$/) next + if (cell[power_col] + 0 <= 0) next + if (cell[gpu_col] == "") next + ts = cell[1] + 0 + # Mirror _parse_timestamp in utils/aggregate_power.py: normalize + # millisecond epochs so a ms-stamping amd-smi build cannot + # trivially satisfy any second-scale stop target. + if (ts > 1e12) ts /= 1000 + gpu = cell[gpu_col] + if (!(gpu in newest) || ts > newest[gpu]) + newest[gpu] = ts + } + END { + have = 0 + for (gpu in newest) + if (!have || newest[gpu] < min) { min = newest[gpu]; have = 1 } + if (have) printf "%d\n", min + } + ' "$GPU_METRICS_CSV" 2>/dev/null + return 0 +} + +# Block until every observed GPU has a usable tick at/after the first whole +# second past stop entry, so any window end preceding the stop request is +# bracketed on file. Bounded by AMD_MONITOR_STOP_TIMEOUT_S; always returns 0 — +# on timeout or early monitor death it warns and lets aggregation attribute +# the missing coverage (fail-safe, never fail-silent). +_wait_for_amd_stop_coverage() { + local target deadline covered timeout_s + # A non-integer timeout (e.g. "30s") would abort the whole stop_gpu_monitor + # call under `set -e` at the arithmetic below, leaking the monitor process + # and skipping tail repair + the energy sidecar; fall back to the default. + timeout_s="${AMD_MONITOR_STOP_TIMEOUT_S:-30}" + if [[ ! "$timeout_s" =~ ^-?[0-9]+$ ]]; then + echo "[GPU Monitor] Warning: ignoring non-integer AMD_MONITOR_STOP_TIMEOUT_S='$timeout_s', using 30" >&2 + timeout_s=30 + fi + if [[ "$timeout_s" -le 0 ]]; then + return 0 + fi + target=$(( $(date +%s) + 1 )) + deadline=$(( target + timeout_s )) + while :; do + covered=$(_amd_monitor_min_covered_tick) + # The first usable row may arrive after stop begins. Keep the same + # deadline for empty or unsupported streams instead of stopping early. + if [[ -n "$covered" && "$covered" -ge "$target" ]]; then + return 0 + fi + if ! _background_process_is_running "$GPU_MONITOR_PID"; then + echo "[GPU Monitor] Warning: AMD monitor exited before covering the stop request (covered=$covered target=$target)" >&2 + return 0 + fi + if [[ "$(date +%s)" -ge "$deadline" ]]; then + echo "[GPU Monitor] Warning: AMD telemetry never covered the stop request within ${timeout_s}s (covered=$covered target=$target)" >&2 + return 0 + fi + sleep 1 + done +} + # Write one best-effort amd-smi snapshot; remove the file rather than keep a # partial one when the invocation fails. _write_amd_smi_sidecar() { @@ -3285,9 +3413,15 @@ run_agentic_replay_and_write_outputs() ( esac _stop_agentx_power_monitor() { + local mode="${1:-}" if [ "$agentx_monitor_stopped" = "0" ]; then - agentx_monitor_stopped=1 + if [ "$mode" = "abort" ]; then + # A cancelled run's power validity is moot; skip the AMD + # coverage wait so signal teardown stays fast. + AMD_MONITOR_STOP_TIMEOUT_S=0 + fi stop_gpu_monitor + agentx_monitor_stopped=1 fi } @@ -3331,10 +3465,11 @@ run_agentic_replay_and_write_outputs() ( agentx_monitor_stopped=0 # This function runs in a subshell, so these handlers cannot replace # launcher-owned traps. The stopped flag keeps explicit and signal/EXIT - # cleanup idempotent. - trap '_stop_agentx_power_monitor' EXIT - trap '_stop_agentx_power_monitor; exit 130' INT - trap '_stop_agentx_power_monitor; exit 143' TERM + # cleanup idempotent after stopping completes. If a signal interrupts + # the normal coverage wait, abort cleanup must still kill the monitor. + trap '_stop_agentx_power_monitor abort' EXIT + trap '_stop_agentx_power_monitor abort; exit 130' INT + trap '_stop_agentx_power_monitor abort; exit 143' TERM fi echo "$REPLAY_CMD" > "$result_dir/benchmark_command.txt" diff --git a/benchmarks/multi_node/amd_utils/bench.sh b/benchmarks/multi_node/amd_utils/bench.sh index 3dde0a68a..357dfdbe8 100755 --- a/benchmarks/multi_node/amd_utils/bench.sh +++ b/benchmarks/multi_node/amd_utils/bench.sh @@ -52,6 +52,16 @@ profile_folder="${log_path}/${ENGINE}_isl_${chosen_isl}_osl_${chosen_osl}" mkdir -p "$profile_folder" source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/power.sh" +power_required=0 +case "${REQUIRE_POWER:-0}" in + 1|true|TRUE|yes|YES) power_required=1 ;; +esac +if ! wait_amd_multinode_power ready; then + [[ "$power_required" == 0 ]] || exit 1 + echo 'PowerX: continuing without ready optional telemetry' >&2 +fi +benchmark_exit_code=0 REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" @@ -101,6 +111,7 @@ for max_concurrency in "${chosen_concurrencies[@]}"; do fi fi + point_exit_code=0 run_benchmark_serving \ --bench-serving-dir "$REPO_ROOT" \ --model "$BENCH_MODEL" \ @@ -113,7 +124,8 @@ for max_concurrency in "${chosen_concurrencies[@]}"; do --max-concurrency "$max_concurrency" \ --result-filename "$export_file" \ --result-dir /workspace/ \ - $extra_flags + $extra_flags || point_exit_code=$? + if [[ "$point_exit_code" != 0 ]]; then benchmark_exit_code=$point_exit_code; break; fi echo "-----------------------------------------" @@ -123,3 +135,10 @@ for max_concurrency in "${chosen_concurrencies[@]}"; do sleep 10 fi done + +# Stop every node while all prefill/decode servers are still alive. +if ! wait_amd_multinode_power done; then + echo 'PowerX: collector completion failed' >&2 + if [[ "$power_required" == 1 && "$benchmark_exit_code" == 0 ]]; then benchmark_exit_code=1; fi +fi +exit "$benchmark_exit_code" diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index cb6f4ad4d..06bedd9e4 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -237,8 +237,9 @@ fi # Node Selection # ============================================================================= -NUM_NODES=$((xP + yD)) -echo "NUM_NODES: $NUM_NODES (xP=$xP + yD=$yD)" +# Workers can span multiple physical nodes. Preserve the submit-time count. +NUM_NODES="${NUM_NODES:-$(( ((PREFILL_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE) * xP + ((DECODE_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE) * yD ))}" +echo "NUM_NODES: $NUM_NODES (prefill workers=$xP, decode workers=$yD)" FULL_NODELIST=$(scontrol show hostnames "$SLURM_JOB_NODELIST") SELECTED_NODES=$(echo "$FULL_NODELIST" | head -n $NUM_NODES) @@ -318,6 +319,10 @@ export DRY_RUN="${DRY_RUN:-0}" export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}" export KEEP_CONTAINERS="${KEEP_CONTAINERS:-0}" export ENGINE=$ENGINE +export POWERX_HOST_UID=$(id -u) +export POWERX_HOST_GID=$(id -g) +export POWERX_COLLECTOR_REVISION=$(git -C "$DI_REPO_DIR" rev-parse HEAD) +mkdir -p "$BENCHMARK_LOGS_DIR/power-control-${SLURM_JOB_ID}" # Eval-related env vars (threaded from submit.sh) export RUN_EVAL="${RUN_EVAL:-false}" @@ -377,6 +382,16 @@ else echo "[WARN] $RDMA_CHECK_SCRIPT not found; skipping RDMA QoS/DCQCN pre-flight check" fi +stage_native_power() { + srun --overlap --nodelist="$SELECTED_NODELIST_SRUN" --ntasks="$NUM_NODES" bash -c ' + src="/tmp/slurm_job-${SLURM_JOB_ID}/native_power/node-${SLURM_PROCID}" + if [[ -d "$src" ]]; then + mkdir -p "$BENCHMARK_LOGS_DIR/native_power" + cp -r "$src" "$BENCHMARK_LOGS_DIR/native_power/" + fi + ' +} + cleanup() { echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up container + stale logs..." # Backstop: on scancel/timeout/step-hang the foreground `exec docker run` @@ -386,11 +401,13 @@ cleanup() { # other users' containers. (Ported from InferenceY 51ebfa88.) srun --nodelist="$SELECTED_NODELIST_SRUN" \ bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' 2>/dev/null || true + stage_native_power || true rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true echo "[${SLURM_JOB_ID}] cleanup done." } -trap cleanup INT TERM HUP +trap 'cleanup; exit 130' INT +trap 'cleanup; exit 143' TERM HUP # Force NFS cache refresh on all nodes echo "Refreshing NFS caches on all nodes..." @@ -412,6 +429,11 @@ DOCKER_ENV_COMMON=( -e SLURM_JOB_ID=\$SLURM_JOB_ID -e SLURM_JOB_NODELIST=\$SLURM_JOB_NODELIST -e NNODES=\$NNODES + -e POWERX_HOST_UID=\$POWERX_HOST_UID + -e POWERX_HOST_GID=\$POWERX_HOST_GID + -e POWERX_COLLECTOR_REVISION=\$POWERX_COLLECTOR_REVISION + -e POWERX_NODE_NAME=\$POWERX_NODE_NAME + -e POWERX_CLOCK_SYNCHRONIZED=\$POWERX_CLOCK_SYNCHRONIZED -e NODE_RANK=\$SLURM_PROCID -e NODE0_ADDR=\$NODE0_ADDR -e MODEL_DIR=/models @@ -601,6 +623,10 @@ set -euo pipefail echo \"Rank \$SLURM_PROCID on \$(hostname)\" +# Capture the host's synchronization state, not the container's missing D-Bus. +export POWERX_NODE_NAME=\$(hostname) +export POWERX_CLOCK_SYNCHRONIZED=\$(timedatectl show -p NTPSynchronized --value 2>/dev/null || true) + # Per-node docker privilege detection eval \"\$DOCKER_CMD_DETECT\" echo \"[docker-detect] rank \$SLURM_PROCID: DOCKER_CMD=\$DOCKER_CMD\" @@ -784,6 +810,11 @@ echo \"[rank 0] Main container exited (rc=\$DOCKER_EXIT_CODE). Stopping vllm-rou exit \$DOCKER_EXIT_CODE " +BENCHMARK_STEP_RC=$? +# Each host copies its own node-local, root-created artifacts as the runner user. +# No raw telemetry is written as root into the shared checkout. +stage_native_power || BENCHMARK_STEP_RC=1 + if [[ "${KEEP_CONTAINERS}" != "1" ]]; then srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' '"$CLIENT_CONT_NAME"' 2>/dev/null || true' @@ -794,3 +825,5 @@ if [[ "${KEEP_CONTAINERS}" != "1" ]]; then ' fi fi + +exit "$BENCHMARK_STEP_RC" diff --git a/benchmarks/multi_node/amd_utils/power.sh b/benchmarks/multi_node/amd_utils/power.sh new file mode 100644 index 000000000..18472bb72 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/power.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +start_amd_multinode_power() { + [[ "${BENCH_INPUT_LEN:-}" == 8192 && "${BENCH_OUTPUT_LEN:-}" == 1024 && + "${EVAL_ONLY:-false}" != true && "${IS_AGENTIC:-0}" != 1 && + "${IS_AGENTIC:-false}" != true && "${DRY_RUN:-0}" != 1 ]] || return 0 + local prefill_nodes_per_worker decode_nodes_per_worker prefill_nodes total_nodes role tp worker_nodes gpu_count gpu_indices + prefill_nodes_per_worker=$(( (PREFILL_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE )) + decode_nodes_per_worker=$(( (DECODE_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE )) + prefill_nodes=$(( prefill_nodes_per_worker * xP )) + total_nodes=$(( prefill_nodes + decode_nodes_per_worker * yD )) + [[ "$total_nodes" == "$NNODES" ]] || { echo 'PowerX: inconsistent AMD node topology' >&2; return 1; } + if (( NODE_RANK < prefill_nodes )); then + role=prefill; tp=$PREFILL_TP_SIZE; worker_nodes=$prefill_nodes_per_worker + else + role=decode; tp=$DECODE_TP_SIZE; worker_nodes=$decode_nodes_per_worker + fi + # Distributed tensor parallelism assigns equal local ranks to every node, + # e.g. TP12 across two nodes uses GPU0..5 on each, not 8 GPUs plus 4 GPUs. + (( tp % worker_nodes == 0 )) || { echo 'PowerX: uneven per-node TP layout' >&2; return 1; } + gpu_count=$(( tp / worker_nodes )) + gpu_indices=$(seq 0 $((gpu_count - 1)) | paste -sd, -) + export POWERX_CONTROL_DIR="${BENCHMARK_LOGS_DIR}/power-control-${SLURM_JOB_ID}" + local native_dir="/run_logs/slurm_job-${SLURM_JOB_ID}/native_power/node-${NODE_RANK}" + bash "$WS_PATH/../../native_power_collect.sh" "$native_dir" "$POWERX_CONTROL_DIR" \ + amd "$NODE_RANK" "$role" "$gpu_indices" "$total_nodes" & + POWERX_COLLECTOR_PID=$! + # EXIT remains independent of serving-engine INT/TERM handlers. + trap 'if [[ -n "${POWERX_COLLECTOR_PID:-}" ]]; then kill "$POWERX_COLLECTOR_PID" 2>/dev/null || true; wait "$POWERX_COLLECTOR_PID" 2>/dev/null || true; fi' EXIT +} + +wait_amd_multinode_power() { + local stage=$1 deadline=$((SECONDS + 60)) ready rank + [[ -n "${POWERX_CONTROL_DIR:-}" ]] || return 0 + if [[ "$stage" == done ]]; then + printf 'stop\n' > "$POWERX_CONTROL_DIR/stop" + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$POWERX_CONTROL_DIR/stop" + fi + while (( SECONDS < deadline )); do + ready=1 + for ((rank=0; rank&2; return 1 + fi + [[ -f "$POWERX_CONTROL_DIR/$stage-$rank" ]] || ready=0 + done + if [[ "$ready" == 1 ]]; then + if [[ "$stage" == done ]]; then + for ((rank=0; rank&2 + return 1 +} diff --git a/benchmarks/multi_node/amd_utils/server.sh b/benchmarks/multi_node/amd_utils/server.sh index b62ca5816..70c2d5cbb 100755 --- a/benchmarks/multi_node/amd_utils/server.sh +++ b/benchmarks/multi_node/amd_utils/server.sh @@ -11,6 +11,9 @@ ENGINE="${ENGINE:-sglang-disagg}" WS_PATH="${WS_PATH:-${SGLANG_WS_PATH:-${VLLM_WS_PATH:-${ATOM_WS_PATH:-$(dirname "${BASH_SOURCE[0]}")}}}}" export WS_PATH ENGINE +source "$WS_PATH/power.sh" +start_amd_multinode_power || exit 1 + echo "[DISPATCHER] ENGINE=$ENGINE WS_PATH=$WS_PATH" if [[ "$ENGINE" == "vllm-disagg" ]]; then diff --git a/benchmarks/multi_node/amd_utils/server_atom.sh b/benchmarks/multi_node/amd_utils/server_atom.sh index 5bad8fd39..81cdaa6e0 100755 --- a/benchmarks/multi_node/amd_utils/server_atom.sh +++ b/benchmarks/multi_node/amd_utils/server_atom.sh @@ -378,7 +378,8 @@ if [ "$NODE_RANK" -eq 0 ]; then echo "DRY RUN: $BENCH_CMD" else set -x - eval "$BENCH_CMD" + BENCHMARK_EXIT_CODE=0 + eval "$BENCH_CMD" || BENCHMARK_EXIT_CODE=$? set +x fi @@ -619,4 +620,4 @@ else fi echo "Script completed successfully" -exit 0 \ No newline at end of file +exit "${BENCHMARK_EXIT_CODE:-0}" diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 7815e5a91..729389327 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -1064,7 +1064,8 @@ print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { set +x else set -x - eval "$BENCH_CMD" + BENCHMARK_EXIT_CODE=0 + eval "$BENCH_CMD" || BENCHMARK_EXIT_CODE=$? set +x fi @@ -1371,4 +1372,4 @@ else fi echo "Script completed successfully" -exit 0 +exit "${BENCHMARK_EXIT_CODE:-0}" diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh index 5a6ce2357..6a65bbe8a 100755 --- a/benchmarks/multi_node/amd_utils/server_vllm.sh +++ b/benchmarks/multi_node/amd_utils/server_vllm.sh @@ -323,7 +323,8 @@ if [ "$NODE_RANK" -eq 0 ]; then echo "DRY RUN: $BENCH_CMD" else set -x - eval "$BENCH_CMD" + BENCHMARK_EXIT_CODE=0 + eval "$BENCH_CMD" || BENCHMARK_EXIT_CODE=$? set +x fi @@ -539,4 +540,4 @@ fi # pkill -f etcd 2>/dev/null || true echo "Script completed successfully" -exit 0 +exit "${BENCHMARK_EXIT_CODE:-0}" diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index d853f6493..734329246 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -112,6 +112,14 @@ The runner-name prefix is load-bearing: workflow routing uses `launch_${RUNNER_N 6. Verify every runner is **Idle** in [repository runner settings](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners) before adding it to sweep traffic. 7. Verify launcher mounts for `_work`, HF cache, staged weights, and squash images from a compute node. Root containers must not leave root-owned files in the shared workspace. +## Native PowerX collection for fixed-sequence multinode runs + +The AMD SGLang/ATOM/vLLM launchers enable native SMI collection for 8192-input/1024-output runs. Every serving node starts `benchmarks/native_power_collect.sh`; the client waits for all `ready-` receipts, then requests `stop` and waits for all `done-` receipts before tearing down servers. The shared collector also accepts NVIDIA SMI for launchers that do not use the srt-slurm/DCGM contract. + +Keep `native_power/node-/gpu_metrics.csv`, the start/end device identity snapshots and `manifest.json` together. Select the actual serving GPU indices, preserve physical node counts when workers span nodes, and stage node-local files as the host runner user into `LOGS/native_power`. The result processor uses each client's formal window, validates all node/role counts and UUID membership, and reuses the shared integration/percentile math. Aggregate deployments emit whole-deployment metrics; role metrics require real separate prefill/decode pools. + +Host `timedatectl NTPSynchronized` is recorded as clock context. The shared collector accepts `yes` or `true` as synchronized; other or missing values remain unsynchronized. It does not measure the offset between nodes; common-window trace coverage is still required, and runtime clock alignment remains part of fleet qualification. Missing clock context, a replaced UUID, a missing node or an incomplete collector lifecycle makes power unavailable. Local fixtures prove the format and failure behavior, not GPU runtime or dashboard publication. + ## Register an srt-slurm recipe Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md). Checked-in recipes: [`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 883f1f8e2..806669fee 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -112,6 +112,14 @@ runner 名称前缀是关键契约：workflow 通过 `launch_${RUNNER_NAME%%_*}. 6. 将 runner 加入 sweep 流量前，在[仓库 runner 设置页](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners)确认每个 runner 都是 **Idle**。 7. 从计算节点验证 launcher 对 `_work`、HF cache、预置权重和 squash 镜像的挂载。root 容器不得在共享 workspace 留下 root 所有的文件。 +## 固定序列长度多节点运行的原生 PowerX 采集 + +AMD SGLang/ATOM/vLLM launcher 为 8192 输入、1024 输出的运行启用原生 SMI 采集。每个服务节点启动 `benchmarks/native_power_collect.sh`；客户端等待全部 `ready-` 回执，然后在基准结束后请求 `stop`，等待全部 `done-` 回执，再关闭服务。共享采集器也支持 NVIDIA SMI，供未采用 srt-slurm/DCGM 契约的 launcher 使用。 + +将 `native_power/node-/gpu_metrics.csv`、开始和结束时的设备身份快照及 `manifest.json` 一起保留。选择实际服务进程使用的 GPU 索引，worker 跨节点时保留真实物理节点数，并由宿主机 runner 用户将节点本地文件暂存到 `LOGS/native_power`。结果处理器使用每个客户端的正式窗口，验证全部节点、角色数量及 UUID 归属，再复用共享积分与百分位计算。聚合部署只输出全部署指标；角色指标要求实际分离的 prefill/decode 池。 + +宿主机的 `timedatectl NTPSynchronized` 状态作为时钟上下文记录。共享采集器将 `yes` 或 `true` 视为已同步；其他值或缺失值均视为未同步。它不测量节点间时钟偏移；仍需验证共同窗口的轨迹覆盖，并在集群运行验证中检查时钟对齐。缺失时钟上下文、UUID 被替换、节点缺失或采集生命周期未完成都会使功耗不可用。本地 fixture 只证明格式和失败处理行为，不能证明 GPU 运行或 dashboard 发布完成。 + ## 注册 srt-slurm 配方 映射来源：[`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md)。检入的配方：[`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/)。 diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index 6da11624d..bf0c81d52 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -106,6 +106,10 @@ For multinode fixed-sequence jobs, `utils/process_result.py --all` processes eve Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. +### Power telemetry stop and validation + +The single-node AMD monitor in [`benchmark_lib.sh`](../benchmarks/benchmark_lib.sh) waits for every observed GPU to have a positive, numeric power sample at or beyond the first whole second after the stop request. `AMD_MONITOR_STOP_TIMEOUT_S` bounds the wait (default `30`; `0` skips it). Streams with missing or unsupported timestamps keep the same bounded wait for a usable sample. A timeout does not certify coverage: the aggregator still rejects an unbracketed benchmark window. AgentX cancellation skips the coverage wait and stops the monitor, including when cancellation arrives during a normal stop. + ### Native multinode telemetry `native_power_collect.sh` and `native_power_lifecycle.sh` provide per-node collection and bounded ready/stop receipts. Launchers opt into the native package under `LOGS/native_power`; this prerequisite enables no new recipe. The adapter validates serving GPU identity, synchronized clocks, collector completion, and complete formal-window coverage. It preserves per-node failures, sample counts, and collector revision in the audit. diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index fd3fa2a2d..31700980a 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -106,6 +106,10 @@ InferenceX-app 将路由字段作为列或配置维度，并把数值测量存 启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 +### 功耗遥测停止与校验 + +[`benchmark_lib.sh`](../benchmarks/benchmark_lib.sh) 中的单节点 AMD 监控会等待每个已观测 GPU 都记录到有效的正数功耗样本，且时间戳不早于停止请求之后的第一个整秒。`AMD_MONITOR_STOP_TIMEOUT_S` 限制等待时长（默认 `30`；设为 `0` 可跳过等待）。时间戳缺失或不支持的数据流仍在相同的截止时间内等待有效样本。超时不代表覆盖有效：聚合器仍会拒绝未被样本完整包围的基准窗口。AgentX 取消运行时会跳过覆盖等待并停止监控，即使取消发生在正常停止的等待过程中。 + ### 原生多节点遥测 `native_power_collect.sh` 和 `native_power_lifecycle.sh` 提供每节点采集及有时限的就绪/停止状态文件。启动器可使用 `LOGS/native_power` 下的原生产物；此前置改动不会启用新 recipe。适配器验证服务 GPU 身份、时钟同步、采集完成及正式窗口完整覆盖，并在审计中保留节点故障、样本数和采集器版本。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6d321aec8..9c26e8ed6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7421,3 +7421,88 @@ - "Retain UTC SMI telemetry context and validate native multinode collector artifacts without enabling new recipes." - "保留 UTC SMI 遥测上下文并验证原生多节点采集产物，不启用新 recipe。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3051 + +- config-keys: + - dsr1-fp4-mi355x-sglang + - dsr1-fp4-mi355x-sglang-mtp + - dsr1-fp4-mi355x-atom + - dsr1-fp4-mi355x-atom-mtp + - dsr1-fp8-mi300x-sglang + - dsr1-fp8-mi325x-sglang + - dsv4-fp8-mi300x-vllm + - dsv4-fp8-mi300x-vllm-mtp + - dsr1-fp8-mi355x-sglang + - dsr1-fp8-mi355x-sglang-mtp + - qwen3.5-fp8-mi325x-sglang + - qwen3.5-fp8-mi355x-sglang + - qwen3.5-fp8-mi355x-sglang-mtp + - qwen3.5-fp8-mi355x-atom + - qwen3.5-fp8-mi355x-atom-mtp + - qwen3.5-fp8-mi355x-sglang-disagg + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-atom + - qwen3.5-fp4-mi355x-sglang-mtp + - qwen3.5-fp4-mi355x-sglang-disagg + - qwen3.5-fp8-mi300x-sglang + - dsr1-fp8-mi355x-atom + - dsr1-fp8-mi355x-atom-mtp + - dsr1-fp8-mi355x-sglang-disagg + - dsr1-fp8-mi355x-sglang-disagg-mtp + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp + - dsv4-fp4-mi355x-sglang-disagg + - dsv4-fp4-mi355x-sglang-disagg-mtp + - dsv4-fp4-mi355x-sglang + - dsv4-fp4-mi355x-sglang-mtp + - dsv4-fp4-mi355x-vllm + - dsv4-fp4-mi355x-vllm-mtp + - dsv4-fp4-mi355x-atom + - dsv4-fp4-mi355x-atom-mtp + - dsr1-fp8-mi325x-sglang-mtp + - qwen3.5-fp8-mi325x-sglang-mtp + - dsr1-fp4-mi355x-sglang-disagg-mtp + - dsv4-fp4-mi355x-atom-disagg + - dsv4-fp8-mi325x-vllm + - dsv4-fp8-mi325x-vllm-mtp + scenario-type: + - fixed-seq-len + description: + - Collect native AMD power across every 8K/1K worker node, preserve device identity and stop-window coverage, + and reject incomplete role telemetry. + - 为 8K/1K 各 AMD 工作节点采集原生功耗，保留设备身份和停止窗口覆盖，并拒绝不完整的角色遥测。 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2767 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-disagg + - qwen3.5-fp4-mi355x-sglang-disagg + - dsr1-fp8-mi355x-sglang-disagg + - dsr1-fp8-mi355x-sglang-disagg-mtp + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp + - dsr1-fp4-mi355x-sglang-disagg-mtp + - dsv4-fp4-mi355x-sglang-disagg + - dsv4-fp4-mi355x-sglang-disagg-mtp + - dsv4-fp4-mi355x-atom-disagg + scenario-type: + - fixed-seq-len + description: + - "Keep optional AMD collector failures separate from serving outcomes and publish owned native collector receipts atomically." + - "将可选 AMD 采集器故障与服务运行结果分开，并原子发布归属正确的原生采集器状态文件。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - kimik3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Keep the AMD stop deadline active while waiting for the first usable telemetry row; unsupported streams no longer sleep beyond the configured timeout." + - "等待首条有效 AMD 遥测时继续执行停止截止时间；不支持的采样流不再超出配置超时。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3041 + +- config-keys: + - kimik3-fp4-mi355x-atom-agentic-mtp + - qwen3.5-fp8-mi355x-sglang + description: + - "Keep AMD FIFO setup failures best effort unless power is required, and preserve an existing path that this monitor did not create." + - "AMD FIFO 初始化失败时仅在要求功耗的任务中返回失败，并保留本监控进程未创建的已有路径。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index accd85021..0c1da6fee 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -59,6 +59,10 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ -n "${GITHUB_ACTIONS:-}" && -n "${JOB_ID:-}" ]]; then local art_dir="$GITHUB_WORKSPACE/benchmark_artifacts" mkdir -p "$art_dir" + if [[ -d "$BENCHMARK_LOGS_DIR/native_power" ]]; then + mkdir -p "$GITHUB_WORKSPACE/LOGS/native_power" + cp -r "$BENCHMARK_LOGS_DIR/native_power/". "$GITHUB_WORKSPACE/LOGS/native_power/" || true + fi cp -r "$BENCHMARK_LOGS_DIR"/slurm_job-${JOB_ID}.{out,err} "$art_dir/" 2>/dev/null || true fi # Print .err inline so failures are visible in CI output @@ -129,6 +133,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # Dynamo jobs are launched. In a follow-up PR, the location of the result file should not # depend on the runner, it should always be in the same spot in the GH workspace. + # Preserve native power evidence before cleanup, even when result processing fails. + if [[ -d "$BENCHMARK_LOGS_DIR/native_power" ]]; then + mkdir -p "$GITHUB_WORKSPACE/LOGS/native_power" + cp -r "$BENCHMARK_LOGS_DIR/native_power/". "$GITHUB_WORKSPACE/LOGS/native_power/" + export POWERX_NATIVE_DIR="$GITHUB_WORKSPACE/LOGS/native_power" + fi + # Process results from all configurations # search for "FRAMEWORK_DIFF_IF_STATEMENT #3" for this if-statement diff --git a/runners/test_amd_monitor_wiring.py b/runners/test_amd_monitor_wiring.py new file mode 100644 index 000000000..102895eb7 --- /dev/null +++ b/runners/test_amd_monitor_wiring.py @@ -0,0 +1,65 @@ +import os +import subprocess +import sys +from pathlib import Path +REPO = Path(__file__).resolve().parents[1] + +def test_amd_stop_coverage_uses_slowest_gpu_and_normalizes_milliseconds(tmp_path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text('timestamp,gpu,vcn_activity,socket_power\n' + '1700000002000,0,"[0, 0]",250\n' + '1700000001000,1,"[0, 0]",250\n' + '1700000009000,1,"[0, 0]",N/A\n') + result = subprocess.run(["bash", "-c", 'source "$1"; GPU_METRICS_CSV="$2"; _amd_monitor_min_covered_tick', + "test", str(REPO / "benchmarks/benchmark_lib.sh"), str(csv)], + capture_output=True, text=True, check=True, timeout=10) + assert result.stdout.strip() == "1700000001" + + +def test_amd_shutdown_kills_both_pipeline_processes_when_stream_dies(tmp_path): + binary = tmp_path / "bin"; binary.mkdir() + fake = binary / "amd-smi" + fake.write_text(f'''#!{sys.executable} +import json, sys, time +if "-w" in sys.argv: + print("timestamp,gpu,socket_power", flush=True) + while True: + print(str(int(time.time())) + ",0,250", flush=True); time.sleep(.1) +else: + print("[]") +''') + fake.chmod(0o755) + csv = tmp_path / "gpu_metrics.csv" + script = '''source "$1" +start_gpu_monitor --output "$2" +source_pid=$GPU_MONITOR_SOURCE_PID +sink_pid=$GPU_MONITOR_PID +kill "$sink_pid" +wait "$sink_pid" 2>/dev/null || true +AMD_MONITOR_STOP_TIMEOUT_S=0 +stop_gpu_monitor +if kill -0 "$source_pid" 2>/dev/null; then echo 'source leaked'; exit 1; fi +if kill -0 "$sink_pid" 2>/dev/null; then echo 'sink leaked'; exit 1; fi +[[ ! -p "$2.pipe.$$" ]] +''' + subprocess.run(["bash", "-c", script, "test", str(REPO / "benchmarks/benchmark_lib.sh"), str(csv)], + env={**os.environ, "PATH": f"{binary}:{os.environ['PATH']}"}, + capture_output=True, text=True, check=True, timeout=10) + + +def test_amd_multinode_selects_equal_local_tensor_ranks(tmp_path): + arguments = tmp_path / "collector.args" + # Mock only the downstream collector command; exercise actual topology routing. + script = f'''source {str(REPO / 'benchmarks/multi_node/amd_utils/power.sh')!r} +bash() {{ printf '%s\\0' "$@" > {str(arguments)!r}; }} +start_amd_multinode_power +wait "$POWERX_COLLECTOR_PID" +''' + subprocess.run(["bash", "-c", script], env={**os.environ, "BENCH_INPUT_LEN": "8192", + "BENCH_OUTPUT_LEN": "1024", "PREFILL_TP_SIZE": "12", "DECODE_TP_SIZE": "12", + "GPUS_PER_NODE": "8", "xP": "1", "yD": "1", "NNODES": "4", "NODE_RANK": "1", + "WS_PATH": str(tmp_path), "BENCHMARK_LOGS_DIR": str(tmp_path), "SLURM_JOB_ID": "123", + "IS_AGENTIC": "0", "EVAL_ONLY": "false", "DRY_RUN": "0"}, + capture_output=True, text=True, timeout=10, check=True) + args = arguments.read_bytes().decode().split("\0") + assert args[-6:-1] == ["amd", "1", "prefill", "0,1,2,3,4,5", "4"] diff --git a/runners/test_amd_power_lifecycle.py b/runners/test_amd_power_lifecycle.py new file mode 100644 index 000000000..87e3d90c2 --- /dev/null +++ b/runners/test_amd_power_lifecycle.py @@ -0,0 +1,41 @@ +"""Keep optional collector failures separate from AMD serving outcomes.""" +import os +from pathlib import Path +import shutil +import subprocess + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize('phase', ['ready', 'done']) +@pytest.mark.parametrize('required', ['', '1', 'true', 'YES']) +@pytest.mark.parametrize('serving_rc', [0, 7]) +def test_amd_collector_failure_respects_requirement(tmp_path, phase, required, serving_rc): + benchmark_root = tmp_path / 'benchmarks' + scripts = benchmark_root / 'multi_node/amd_utils' + scripts.mkdir(parents=True) + for name in ['bench.sh', 'power.sh']: + shutil.copyfile(ROOT / 'benchmarks/multi_node/amd_utils' / name, scripts / name) + (benchmark_root / 'benchmark_lib.sh').write_text('''run_benchmark_serving() { + printf 'called\\n' > "$CALL_RECEIPT" + printf '1\\n' > "$POWERX_CONTROL_DIR/done-0" + return "$SERVING_RC" + } +''') + control = tmp_path / 'control' + control.mkdir() + (control / 'ready-0').write_text('ready\n') + if phase == 'ready': + (control / 'done-0').write_text('1\n') + receipt = tmp_path / 'called' + result = subprocess.run(['bash', str(scripts / 'bench.sh'), '1', '1', '1', '1', + '/model', 'test', str(tmp_path / 'logs'), '8192', '1024', '1'], + env={**os.environ, 'POWERX_CONTROL_DIR': str(control), 'NNODES': '1', + 'REQUIRE_POWER': required, 'SERVING_RC': str(serving_rc), + 'CALL_RECEIPT': str(receipt), 'POWERX_HOST_UID': str(os.getuid()), + 'POWERX_HOST_GID': str(os.getgid())}, capture_output=True, text=True, timeout=10) + expected = 1 if required and phase == 'ready' else serving_rc or int(bool(required)) + assert result.returncode == expected, result.stderr + assert receipt.exists() == (not required or phase != 'ready') diff --git a/utils/agentic/aggregation/test_power_lifecycle.py b/utils/agentic/aggregation/test_power_lifecycle.py index 20710e68e..97688f4f9 100644 --- a/utils/agentic/aggregation/test_power_lifecycle.py +++ b/utils/agentic/aggregation/test_power_lifecycle.py @@ -2,6 +2,7 @@ from __future__ import annotations +import csv import json import os import re @@ -278,7 +279,9 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( start_gpu_monitor() {{ printf 'monitor-pid:%s\n' "${{BASHPID:-$$}}" >> {str(event_log)!r} }} -stop_gpu_monitor() {{ printf 'monitor-stop\n' >> {str(event_log)!r}; }} +stop_gpu_monitor() {{ + printf 'monitor-stop:%s\n' "${{AMD_MONITOR_STOP_TIMEOUT_S:-unset}}" >> {str(event_log)!r} +}} fake_replay() {{ printf 'replay-ready\n' >> {str(event_log)!r} exec sleep 30 @@ -321,7 +324,404 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( assert proc.returncode == expected_rc, stderr events = _events(tmp_path) - assert events.count("monitor-stop") == 1 + stop_events = [event for event in events if event.startswith("monitor-stop")] + # Signal teardown must stop exactly once, in abort mode: the coverage wait + # is skipped by setting AMD_MONITOR_STOP_TIMEOUT_S=0 before stopping. + assert stop_events == ["monitor-stop:0"] expected_parent_event = "parent-int" if sent_signal == signal.SIGINT else "parent-term" assert expected_parent_event in events assert events[-1] == "parent-exit" + + + +@pytest.mark.parametrize( + ("sent_signal", "expected_rc"), + [(signal.SIGINT, 130), (signal.SIGTERM, 143)], +) +def test_signal_during_amd_coverage_wait_stops_monitor( + tmp_path: Path, sent_signal: signal.Signals, expected_rc: int +): + result_dir = tmp_path / "results" + result_dir.mkdir() + event_log = tmp_path / "events.log" + script = f""" +source {str(BENCHMARK_LIB)!r} +start_gpu_monitor() {{ + GPU_METRICS_CSV="$2" + printf 'timestamp,gpu,socket_power\n1,0,500\n' > "$GPU_METRICS_CSV" + command sleep 60 >/dev/null 2>&1 & + GPU_MONITOR_PID=$! + GPU_MONITOR_VENDOR=amd + printf 'monitor:%s\nlifecycle:%s\n' "$GPU_MONITOR_PID" "${{BASHPID:-$(exec sh -c 'echo "$PPID"')}}" >> {str(event_log)!r} +}} +sleep() {{ + printf 'coverage-wait\n' >> {str(event_log)!r} + command sleep "$@" +}} +_write_amd_smi_sidecar() {{ :; }} +fake_replay() {{ :; }} +trap 'printf "parent-exit\\n" >> {str(event_log)!r}' EXIT +REPLAY_CMD=fake_replay +ENABLE_AGENTX_POWER=1 +IS_MULTINODE=false +AMD_MONITOR_STOP_TIMEOUT_S=30 +run_agentic_replay_and_write_outputs {str(result_dir)!r} +exit $? +""" + proc = subprocess.Popen( + ["bash", "-c", script], + env={**os.environ, "PATH": "/usr/bin:/bin"}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + events = _events(tmp_path) if event_log.exists() else [] + if "coverage-wait" in events: + break + time.sleep(0.01) + else: + pytest.fail("AMD coverage wait did not start") + + pids = dict(event.split(":") for event in events if ":" in event) + # Signal only the lifecycle shell: signalling the whole group would + # kill the monitor directly and hide a broken cleanup handler. + os.kill(int(pids["lifecycle"]), sent_signal) + _, stderr = proc.communicate(timeout=5) + assert proc.returncode == expected_rc, stderr + assert _events(tmp_path)[-1] == "parent-exit" + with pytest.raises(ProcessLookupError): + os.kill(int(pids["monitor"]), 0) + finally: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.communicate() + + +# AMDSMI 26.2.0 `metric -p -c -t -u -w 1 --csv` header (order-faithful subset, +# measured on MI355X; mirrors test_detect_columns_amd_watch_mode_real_header). +_MI355X_WATCH_HEADER = ( + "timestamp,gpu,gfx_activity,umc_activity,mm_activity,vcn_activity," + "jpeg_activity,gfx_busy_inst_xcp_0,jpeg_busy_xcp_0,vcn_busy_xcp_0," + "socket_power,gfx_voltage,soc_voltage,mem_voltage,throttle_status," + "power_management,gfx_0_clk,mem_0_clk,edge,hotspot,mem" +) + +# amd-smi quotes list-valued cells with embedded commas; the coverage helper +# must keep the power cell at its header-relative position through them. +_WATCH_ROW_FORMAT = ( + "%s,%s,0,0,N/A,\"['N/A', 'N/A']\",\"['N/A', 'N/A']\",\"[0, 0]\"," + "\"[0, 0]\",\"[0, 0]\",%s,N/A,N/A,N/A,N/A,ENABLED,1404,2000,N/A,40,25\\n" +) + + +def _bash_single_quote(text: str) -> str: + return "'" + text.replace("'", "'\\''") + "'" + + +def _run_amd_stop( + tmp_path: Path, + *, + producer_script: str, + timeout_s: int | str, + interval: int = 1, + setup_script: str = "", +) -> subprocess.CompletedProcess[str]: + """Run the real stop_gpu_monitor against a scripted AMD telemetry producer.""" + csv_path = tmp_path / "gpu_metrics.csv" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + amd_smi = bin_dir / "amd-smi" + amd_smi.write_text("#!/bin/bash\nprintf 'gpu,total_energy_consumption\\n0,100.0\\n'\n") + amd_smi.chmod(0o755) + script = f""" +set -e +source {str(BENCHMARK_LIB)!r} +GPU_METRICS_CSV={str(csv_path)!r} +printf '%s\\n' {_bash_single_quote(_MI355X_WATCH_HEADER)} > "$GPU_METRICS_CSV" +emit_row() {{ + printf {_bash_single_quote(_WATCH_ROW_FORMAT)} "$1" "$2" "$3" >> "$GPU_METRICS_CSV" +}} +{setup_script} +( {producer_script} ) & +GPU_MONITOR_PID=$! +printf '%s\\n' "$GPU_MONITOR_PID" > {str(tmp_path / "producer.pid")!r} +GPU_MONITOR_VENDOR=amd +GPU_MONITOR_INTERVAL={interval} +AMD_MONITOR_STOP_TIMEOUT_S={timeout_s} +date +%s > {str(tmp_path / "pre.txt")!r} +stop_gpu_monitor +date +%s > {str(tmp_path / "post.txt")!r} +""" + return subprocess.run( + ["bash", "-c", script], + env={ + **os.environ, + "PATH": f"{bin_dir}:/usr/bin:/bin", + "PYTHONDONTWRITEBYTECODE": "1", + }, + capture_output=True, + text=True, + check=False, + timeout=60, + ) + + +def _min_covered_tick(csv_path: Path) -> int: + """Newest usable tick (numeric ts, power > 0) covered by every GPU.""" + newest: dict[str, float] = {} + with csv_path.open(newline="", encoding="utf-8") as f: + for row in csv.DictReader(f): + try: + timestamp = float((row.get("timestamp") or "").strip()) + power = float((row.get("socket_power") or "").strip()) + except ValueError: + continue + if timestamp > 1e12: # millisecond epoch, mirror _parse_timestamp + timestamp /= 1000.0 + gpu = (row.get("gpu") or "").strip() + if not gpu or power <= 0: + continue + newest[gpu] = max(newest.get(gpu, 0.0), timestamp) + assert newest, "no usable telemetry rows" + return int(min(newest.values())) + + +def _stop_epochs(tmp_path: Path) -> tuple[int, int]: + pre = int((tmp_path / "pre.txt").read_text().strip()) + post = int((tmp_path / "post.txt").read_text().strip()) + return pre, post + + +def _assert_producer_dead(tmp_path: Path) -> None: + producer_pid = int((tmp_path / "producer.pid").read_text().strip()) + with pytest.raises(ProcessLookupError): + os.kill(producer_pid, 0) + + +def test_amd_stop_waits_until_every_gpu_covers_stop_request(tmp_path: Path): + producer = """ +while :; do + now=$(date +%s) + emit_row "$now" 0 500 + emit_row "$now" 1 505 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s=30) + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + assert "never covered the stop request" not in result.stdout + result.stderr + pre, _ = _stop_epochs(tmp_path) + # Every GPU has a usable tick at/after the first whole second past stop + # entry, so any fractional window end before the stop is bracketed. + assert _min_covered_tick(tmp_path / "gpu_metrics.csv") >= pre + 1 + _assert_producer_dead(tmp_path) + assert duration < 10 + + +def test_amd_stop_ignores_degenerate_rows_for_coverage(tmp_path: Path): + setup = """ +stale=$(( $(date +%s) - 30 )) +emit_row "$stale" 0 500 +emit_row "$stale" 1 505 +""" + producer = """ +while :; do + now=$(date +%s) + emit_row "$now" 0 N/A + emit_row "$now" 1 N/A + sleep 0.2 +done +""" + result = _run_amd_stop( + tmp_path, + producer_script=producer, + timeout_s=2, + setup_script=setup, + ) + + assert result.returncode == 0, result.stderr + assert "never covered the stop request" in result.stderr + pre, post = _stop_epochs(tmp_path) + assert post - pre >= 2 + _assert_producer_dead(tmp_path) + + +def test_amd_stop_requires_coverage_per_gpu(tmp_path: Path): + setup = """ +stale=$(( $(date +%s) - 30 )) +emit_row "$stale" 1 505 +""" + producer = """ +while :; do + emit_row "$(date +%s)" 0 500 + sleep 0.2 +done +""" + result = _run_amd_stop( + tmp_path, + producer_script=producer, + timeout_s=2, + setup_script=setup, + ) + + assert result.returncode == 0, result.stderr + # GPU 1 never covers the stop request, so min-over-GPUs coverage times out + # even though GPU 0 keeps producing fresh usable ticks. + assert "never covered the stop request" in result.stderr + pre, post = _stop_epochs(tmp_path) + assert post - pre >= 2 + _assert_producer_dead(tmp_path) + + +def test_amd_stop_preserves_outputs_when_monitor_exits_during_wait(tmp_path: Path): + setup = """ +emit_row 1 0 500 +sleep() { + # End the real producer on the first coverage poll without a timing race. + kill "$GPU_MONITOR_PID" + wait "$GPU_MONITOR_PID" 2>/dev/null || true +} +""" + result = _run_amd_stop( + tmp_path, + producer_script="exec /bin/sleep 60", + timeout_s=3, + setup_script=setup, + ) + + assert result.returncode == 0, result.stderr + assert "AMD monitor exited before covering the stop request" in result.stderr + assert (tmp_path / "gpu_metrics_energy_end.csv").read_text() == ( + "gpu,total_energy_consumption\n0,100.0\n" + ) + _stop_epochs(tmp_path) # The caller continued after stop under set -e. + _assert_producer_dead(tmp_path) + + +def test_amd_stop_survives_non_integer_timeout(tmp_path: Path): + producer = """ +while :; do + now=$(date +%s) + emit_row "$now" 0 500 + emit_row "$now" 1 505 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s="30s") + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + # A non-integer timeout must not unwind stop_gpu_monitor via a bash + # arithmetic error (which would leak the monitor and skip tail repair + # and the energy sidecar): it warns, falls back to 30, and still waits. + assert "ignoring non-integer AMD_MONITOR_STOP_TIMEOUT_S='30s'" in result.stderr + assert "never covered the stop request" not in result.stdout + result.stderr + pre, _ = _stop_epochs(tmp_path) + assert _min_covered_tick(tmp_path / "gpu_metrics.csv") >= pre + 1 + _assert_producer_dead(tmp_path) + assert duration < 10 + + +def test_amd_stop_normalizes_millisecond_epoch_timestamps(tmp_path: Path): + producer = """ +while :; do + now=$(( $(date +%s) * 1000 + 123 )) + emit_row "$now" 0 500 + emit_row "$now" 1 505 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s=30) + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + # Raw millisecond epochs (~1.8e12) dwarf any second-scale target, so + # without normalization the poll would return + # instantly with zero tail coverage; mirrored _parse_timestamp + # normalization makes the poll wait for real coverage instead. + assert "never covered the stop request" not in result.stdout + result.stderr + pre, _ = _stop_epochs(tmp_path) + assert _min_covered_tick(tmp_path / "gpu_metrics.csv") >= pre + 1 + _assert_producer_dead(tmp_path) + assert duration < 10 + + +def test_amd_stop_bounds_wait_for_iso_timestamps(tmp_path: Path): + producer = """ +while :; do + emit_row "$(date +%Y-%m-%dT%H:%M:%S)" 0 500 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s=1, interval=30) + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + assert "never covered the stop request" in result.stderr + assert "exited before covering" not in result.stderr + assert duration < 4 + _assert_producer_dead(tmp_path) + + +def test_amd_stop_waits_for_delayed_first_sample(tmp_path: Path): + producer = """ +sleep 4 +while :; do + emit_row "$(date +%s)" 0 500 + sleep 0.2 +done +""" + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s=8) + + assert result.returncode == 0, result.stderr + pre, _ = _stop_epochs(tmp_path) + assert _min_covered_tick(tmp_path / "gpu_metrics.csv") >= pre + 1 + _assert_producer_dead(tmp_path) + + +@pytest.mark.parametrize('required', ['', '0', '1', 'true']) +@pytest.mark.parametrize('failure', ['unsupported', 'collision']) +def test_amd_fifo_failure_respects_power_requirement_and_keeps_existing_path( + tmp_path: Path, required: str, failure: str +): + script = r''' +source "$1" +amd-smi() { echo unexpected-start > "$CALLS"; } +mkfifo() { + if [[ "$FIFO_FAILURE" == collision ]]; then + printf 'another stream\n' > "$1" + fi + printf '%s\n' "$1" > "$FIFO_PATH" + return 1 +} +trap stop_gpu_monitor EXIT +set -e +start_gpu_monitor --output "$2/gpu_metrics.csv" +printf 'benchmark\n' > "$2/benchmark-called" +''' + result = subprocess.run(['bash', '-c', script, 'bash', str(BENCHMARK_LIB), str(tmp_path)], + env={**os.environ, 'PATH': '/usr/bin:/bin', 'REQUIRE_POWER': required, + 'FIFO_FAILURE': failure, 'FIFO_PATH': str(tmp_path / 'fifo-path'), + 'CALLS': str(tmp_path / 'collector-called')}, + capture_output=True, text=True, timeout=5) + is_required = required in {'1', 'true'} + assert result.returncode == int(is_required), result.stderr + assert (tmp_path / 'benchmark-called').exists() is not is_required + assert not (tmp_path / 'collector-called').exists() + fifo = Path((tmp_path / 'fifo-path').read_text().strip()) + if failure == 'collision': + assert fifo.read_text() == 'another stream\n' + else: + assert not fifo.exists() diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 7bc8dc4d9..8e7bd1047 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -5,6 +5,7 @@ import signal import subprocess import sys +import time from pathlib import Path import pytest @@ -259,6 +260,7 @@ def run_script_with_broken_aggregator( wrapper = f""" import runpy import sys +import time import json import builtins from pathlib import Path @@ -1149,8 +1151,13 @@ def test_stop_gpu_monitor_drops_truncated_row_before_final_sample(self, tmp_path final_sample, ] - def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path): - """AMD stop lets the watch stream bracket the window, then snapshots energy.""" + def test_stop_gpu_monitor_amd_covers_stop_request_and_snapshots_energy(self, tmp_path): + """AMD stop returns once telemetry covers the stop entry, then snapshots energy. + + A usable tick stamped past the stop request satisfies the coverage + poll on its first pass: the legacy fixed tail sleep never runs, the + stream is not mutated, and the end-side accumulator snapshot is + written.""" fake_bin = tmp_path / "bin" fake_bin.mkdir() args_log = tmp_path / "amd_args.txt" @@ -1162,7 +1169,8 @@ def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path ) fake_amd_smi.chmod(0o755) sleep_log = tmp_path / "sleep_args.txt" - contents = "timestamp,gpu,socket_power\n1785881113,0,238\n" + covered_tick = int(time.time()) + 30 + contents = f"timestamp,gpu,socket_power\n{covered_tick},0,238\n" metrics = tmp_path / "gpu_metrics.csv" metrics.write_text(contents) benchmark_lib = Path(__file__).parents[1] / "benchmarks/benchmark_lib.sh" @@ -1170,7 +1178,7 @@ def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path source {str(benchmark_lib)!r} kill() {{ return 0; }} wait() {{ return 0; }} -sleep() {{ printf '%s\\n' "$1" > {str(sleep_log)!r}; }} +sleep() {{ printf '%s\\n' "$1" >> {str(sleep_log)!r}; }} GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd GPU_MONITOR_INTERVAL=3 @@ -1191,7 +1199,8 @@ def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path ) assert result.returncode == 0, result.stderr - assert sleep_log.read_text().strip() == "5" + assert not sleep_log.exists() + assert "never covered the stop request" not in result.stderr assert metrics.read_text() == contents assert "metric -E --csv" in args_log.read_text() energy_end = tmp_path / "gpu_metrics_energy_end.csv" @@ -1220,6 +1229,7 @@ def test_stop_gpu_monitor_amd_drops_truncated_row_without_append(self, tmp_path) GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd GPU_METRICS_CSV={str(metrics)!r} +AMD_MONITOR_STOP_TIMEOUT_S=0 stop_gpu_monitor """ env = { From 07d1cf1a37699b37245d55865423236a80b6f87a Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:32:30 -0700 Subject: [PATCH 05/17] test: synchronize replay readiness with signal handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 由实际重放进程在设置信号处理后发布就绪标记，消除 exec 前 SIGINT 竞态。 --- utils/agentic/aggregation/test_power_lifecycle.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/utils/agentic/aggregation/test_power_lifecycle.py b/utils/agentic/aggregation/test_power_lifecycle.py index 97688f4f9..6b7c5f6db 100644 --- a/utils/agentic/aggregation/test_power_lifecycle.py +++ b/utils/agentic/aggregation/test_power_lifecycle.py @@ -283,8 +283,13 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( printf 'monitor-stop:%s\n' "${{AMD_MONITOR_STOP_TIMEOUT_S:-unset}}" >> {str(event_log)!r} }} fake_replay() {{ - printf 'replay-ready\n' >> {str(event_log)!r} - exec sleep 30 + exec {sys.executable!r} -c ' +import signal, sys, time +signal.signal(signal.SIGINT, signal.SIG_DFL) +signal.signal(signal.SIGTERM, signal.SIG_DFL) +print("replay-ready", file=open(sys.argv[1], "a"), flush=True) +time.sleep(30) +' {str(event_log)!r} }} trap 'printf "parent-exit\\n" >> {str(event_log)!r}' EXIT trap 'printf "parent-int\\n" >> {str(event_log)!r}; exit 130' INT @@ -304,7 +309,8 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( ) try: # The monitor starts before the production signal traps are installed. - # Wait for replay so the signal actually exercises those traps. + # Publish readiness from the execed process after restoring signal handling; + # a shell marker before exec races with the group SIGINT. deadline = time.monotonic() + 5 while time.monotonic() < deadline: if event_log.exists() and "replay-ready" in event_log.read_text().splitlines(): From b230e7077dc24a58b4773457de911946b4a6bb2f Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:41:07 -0700 Subject: [PATCH 06/17] fix: preserve native role metrics and abort receipts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 独立生成 prefill 与 decode 角色指标，并在 AMD 采集器被回收前完成取消状态记录。 --- benchmarks/benchmark_lib.sh | 2 +- infx/results/power/native_multinode.py | 10 ++-- utils/test_native_multinode_power.py | 76 ++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 9d99a4cc9..ab6699c9f 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -225,7 +225,7 @@ stop_gpu_monitor() { # tick in the same second as the window end still fails bracketing — # the stream needs a tick at the NEXT whole second (measured on MI355X: # end=...153.325 vs last sample ...153.0). - if [[ "$GPU_MONITOR_VENDOR" == "amd" ]]; then + if [[ "$GPU_MONITOR_VENDOR" == "amd" && "${AMD_MONITOR_STOP_TIMEOUT_S:-}" != "0" ]]; then sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 )) fi kill "$GPU_MONITOR_PID" 2>/dev/null diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py index 1ffae5bd9..86b5d683b 100644 --- a/infx/results/power/native_multinode.py +++ b/infx/results/power/native_multinode.py @@ -221,13 +221,15 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, metrics = {} if not reasons and integration is not None and benchmark is not None: metrics = _derived_metrics(integration, benchmark) - if expected_decode_gpus: + if roles["prefill"]: prefill = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["prefill"]) - decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) - metrics.update(prefill_gpu_energy_j=prefill, decode_gpu_energy_j=decode, + metrics.update(prefill_gpu_energy_j=prefill, prefill_avg_power_w=prefill / benchmark.integration_duration_s / expected_prefill_gpus, + prefill_joules_per_input_token=prefill / benchmark.total_input_tokens) + if roles["decode"]: + decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) + metrics.update(decode_gpu_energy_j=decode, decode_avg_power_w=decode / benchmark.integration_duration_s / expected_decode_gpus, - prefill_joules_per_input_token=prefill / benchmark.total_input_tokens, decode_joules_per_output_token=decode / benchmark.total_output_tokens) valid = not reasons try: diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py index 0aeee2511..c2945cb75 100644 --- a/utils/test_native_multinode_power.py +++ b/utils/test_native_multinode_power.py @@ -199,3 +199,79 @@ def test_result_processor_discovers_staged_native_package(tmp_path, monkeypatch) invalid = json.loads(agg.read_text()) assert invalid["power_valid"] == 0 assert "total_gpu_energy_j" not in invalid + + +@pytest.mark.parametrize('role', ['prefill', 'decode']) +def test_native_single_role_preserves_whole_fleet_and_role_metrics(tmp_path, role): + root, bench, agg = _package(tmp_path) + for path in root.glob('*/manifest.json'): + manifest = json.loads(path.read_text()) + manifest['role'] = role + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=2 if role == 'prefill' else 0, + expected_decode_gpus=2 if role == 'decode' else 0, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual['total_gpu_energy_j'] == 800 + assert actual[f'{role}_gpu_energy_j'] == 800 + assert actual[f'{role}_avg_power_w'] == 200 + opposite = 'decode' if role == 'prefill' else 'prefill' + assert f'{opposite}_gpu_energy_j' not in actual + assert json.loads((tmp_path / 'power_validation_result.json').read_text())['power_valid'] + + +def test_native_amd_abort_skips_legacy_tail_wait(tmp_path): + result = subprocess.run(['bash', '-c', '''source "$1" +kill() { return 0; }; wait() { return 0; } +sleep() { echo unexpected-tail-wait >&2; } +_write_amd_smi_sidecar() { return 0; } +GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd AMD_MONITOR_STOP_TIMEOUT_S=0 +GPU_METRICS_CSV="$2/missing.csv" +stop_gpu_monitor +''', 'bash', str(REPO / 'benchmarks/benchmark_lib.sh'), str(tmp_path)], + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert 'unexpected-tail-wait' not in result.stderr + + +def test_native_amd_abort_publishes_receipt_before_reaper_deadline(tmp_path): + binary = tmp_path / 'bin' + binary.mkdir() + (binary / 'python3').symlink_to(sys.executable) + fake = binary / 'amd-smi' + fake.write_text(f'''#!{sys.executable} +import sys, time +if "-w" in sys.argv: + print("timestamp,gpu,socket_power", flush=True) + while True: + print(str(int(time.time())) + ",0,100", flush=True) + time.sleep(0.1) +else: + print("[]") +''') + fake.chmod(0o755) + control = tmp_path / 'control' + control.mkdir() + node = tmp_path / 'node-0' + process = subprocess.Popen(['bash', '-c', '\n'.join([ + 'source "$1"', + 'bash "$4" "$2" "$3" amd 0 aggregate 0 1 &', + 'POWERX_COLLECTOR_PID=$! POWERX_CONTROL_DIR=$3 POWERX_NUM_NODES=1', + 'POWERX_BARRIER_TIMEOUT_S=5', + 'powerx_wait_collectors ready || exit 1', + 'POWERX_BARRIER_TIMEOUT_S=0', + 'powerx_reap_collector', + ]), 'bash', str(REPO / 'benchmarks/native_power_lifecycle.sh'), str(node), str(control), str(REPO / 'benchmarks/native_power_collect.sh')], + env={**os.environ, 'PATH': f'{binary}:/usr/bin:/bin', 'SLURM_JOB_ID': 'test-job'}, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) + try: + stdout, stderr = process.communicate(timeout=8) + assert process.returncode == 143, (stdout, stderr) + assert (control / 'done-0').read_text().strip() == '143' + assert json.loads((node / 'manifest.json').read_text())['lifecycle'] == 'failed' + finally: + import signal + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() From 880f1153b489da0965261ededb090932d5a374dd Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:43:51 -0700 Subject: [PATCH 07/17] ci: run native collector contract regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在现有 CPU 工作流中执行原生采集契约回归，并跟踪对应文件变更。 --- .github/workflows/test-process-result.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 4d4cf0adf..7d5071770 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -10,6 +10,11 @@ on: - '.github/workflows/e2e-tests.yml' - '.github/workflows/test-process-result.yml' - 'benchmarks/benchmark_lib.sh' + - 'benchmarks/native_power_collect.sh' + - 'benchmarks/native_power_lifecycle.sh' + - 'runners/test_native_collector_barriers.py' + - 'runners/test_native_collector_receipts.py' + - 'utils/test_native_multinode_power.py' - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/**/*.yaml' - 'benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/**/*.yaml' - 'benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/**/*.yaml' @@ -57,4 +62,4 @@ jobs: run: | cd utils uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \ - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py -v From 654c33b2f71584f81a4c11a65e97a3dbd0fc41a9 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:48:30 -0700 Subject: [PATCH 08/17] test: synchronize shared replay signal readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将已验证的信号就绪屏障移至共享原生采集前置，避免重放 exec 前的竞态。 --- utils/agentic/aggregation/test_power_lifecycle.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/utils/agentic/aggregation/test_power_lifecycle.py b/utils/agentic/aggregation/test_power_lifecycle.py index 20710e68e..6d955af12 100644 --- a/utils/agentic/aggregation/test_power_lifecycle.py +++ b/utils/agentic/aggregation/test_power_lifecycle.py @@ -280,8 +280,13 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( }} stop_gpu_monitor() {{ printf 'monitor-stop\n' >> {str(event_log)!r}; }} fake_replay() {{ - printf 'replay-ready\n' >> {str(event_log)!r} - exec sleep 30 + exec {sys.executable!r} -c ' +import signal, sys, time +signal.signal(signal.SIGINT, signal.SIG_DFL) +signal.signal(signal.SIGTERM, signal.SIG_DFL) +print("replay-ready", file=open(sys.argv[1], "a"), flush=True) +time.sleep(30) +' {str(event_log)!r} }} trap 'printf "parent-exit\\n" >> {str(event_log)!r}' EXIT trap 'printf "parent-int\\n" >> {str(event_log)!r}; exit 130' INT @@ -301,7 +306,8 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( ) try: # The monitor starts before the production signal traps are installed. - # Wait for replay so the signal actually exercises those traps. + # Publish readiness from the execed process after restoring signal handling; + # a shell marker before exec races with the group SIGINT. deadline = time.monotonic() + 5 while time.monotonic() < deadline: if event_log.exists() and "replay-ready" in event_log.read_text().splitlines(): From 1422f53707d370c42333354c656bd1d962fec492 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:48:45 -0700 Subject: [PATCH 09/17] fix: preserve serving after optional AMD power startup failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 可选功耗初始化失败时保留服务启动及退出状态，仅在 REQUIRE_POWER 开启时阻止服务。 --- benchmarks/multi_node/amd_utils/server.sh | 7 +++++- runners/test_amd_power_lifecycle.py | 30 +++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/server.sh b/benchmarks/multi_node/amd_utils/server.sh index 70c2d5cbb..15fad699b 100755 --- a/benchmarks/multi_node/amd_utils/server.sh +++ b/benchmarks/multi_node/amd_utils/server.sh @@ -12,7 +12,12 @@ WS_PATH="${WS_PATH:-${SGLANG_WS_PATH:-${VLLM_WS_PATH:-${ATOM_WS_PATH:-$(dirname export WS_PATH ENGINE source "$WS_PATH/power.sh" -start_amd_multinode_power || exit 1 +if ! start_amd_multinode_power; then + case "${REQUIRE_POWER:-0}" in + 1|true|TRUE|yes|YES) exit 1 ;; + esac + echo 'PowerX: continuing without optional worker telemetry' >&2 +fi echo "[DISPATCHER] ENGINE=$ENGINE WS_PATH=$WS_PATH" diff --git a/runners/test_amd_power_lifecycle.py b/runners/test_amd_power_lifecycle.py index 87e3d90c2..cb2667380 100644 --- a/runners/test_amd_power_lifecycle.py +++ b/runners/test_amd_power_lifecycle.py @@ -39,3 +39,33 @@ def test_amd_collector_failure_respects_requirement(tmp_path, phase, required, s expected = 1 if required and phase == 'ready' else serving_rc or int(bool(required)) assert result.returncode == expected, result.stderr assert receipt.exists() == (not required or phase != 'ready') + + +@pytest.mark.parametrize('fault', ['topology', 'uneven_tp']) +@pytest.mark.parametrize('required', ['', '0', 'false', '1', 'true', 'YES']) +@pytest.mark.parametrize('serving_rc', [0, 7]) +def test_amd_start_failure_respects_requirement(tmp_path, fault, required, serving_rc): + scripts = tmp_path / 'amd_utils' + scripts.mkdir() + for name in ['server.sh', 'power.sh']: + shutil.copyfile(ROOT / 'benchmarks/multi_node/amd_utils' / name, scripts / name) + receipt = tmp_path / 'called' + (scripts / 'server_sglang.sh').write_text('''printf 'called\\n' > "$CALL_RECEIPT" +exit "$SERVING_RC" +''') + result = subprocess.run(['bash', str(scripts / 'server.sh')], + env={**os.environ, 'WS_PATH': str(scripts), + 'ENGINE': 'sglang-disagg', 'BENCH_INPUT_LEN': '8192', + 'BENCH_OUTPUT_LEN': '1024', 'EVAL_ONLY': 'false', + 'IS_AGENTIC': '0', 'DRY_RUN': '0', + 'GPUS_PER_NODE': '8', 'PREFILL_TP_SIZE': '9', + 'DECODE_TP_SIZE': '8', 'xP': '1', 'yD': '1', + 'NNODES': '2' if fault == 'topology' else '3', + 'NODE_RANK': '0', 'REQUIRE_POWER': required, + 'SERVING_RC': str(serving_rc), 'CALL_RECEIPT': str(receipt)}, + capture_output=True, text=True, timeout=5) + is_required = required in ['1', 'true', 'YES'] + assert result.returncode == (1 if is_required else serving_rc), result.stderr + assert receipt.exists() == (not is_required) + assert ('inconsistent AMD node topology' if fault == 'topology' else + 'uneven per-node TP layout') in result.stderr From d141e71c8be2bb6c57c37a2a623bfb6c274e5921 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:02:16 -0700 Subject: [PATCH 10/17] fix: preserve diagnostic sidecars and legacy result processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 只忽略已定义的诊断 sidecar，保留损坏与缺失点门禁；旧处理器逐点处理并累计失败，新版导入错误不回退。 --- .../workflows/benchmark-multinode-tmpl.yml | 42 ++++++++- infx/results/fixed_sequence.py | 9 +- utils/test_process_result.py | 87 +++++++++++++++++++ 3 files changed, 136 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 80b0842f8..d26d4e0bb 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -480,7 +480,47 @@ jobs: export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA" fi - python3 utils/process_result.py --all + # Older measured refs predate the batch CLI. Import failures in a + # present processor are errors, not a reason to downgrade validation. + has_batch=$(python3 - <<'PY' + import importlib + from pathlib import Path + if Path('infx/results/fixed_sequence.py').is_file(): + module = importlib.import_module('infx.results.fixed_sequence') + print(int(hasattr(module, 'process_multinode_results'))) + else: + print(0) + PY + ) + if [ "$has_batch" = 1 ]; then + python3 utils/process_result.py --all + else + batch_rc=0 + observed=() + for result_file in "${RESULT_FILENAME}"_*.json; do + [ -f "$result_file" ] || continue + case "$result_file" in + *.pytorch.json|"${RESULT_FILENAME}_gpu_metrics_context.json"|"${RESULT_FILENAME}_gpu_metrics_identity.json") continue ;; + esac + if [[ "$result_file" =~ _(c|conc|concurrency_)([0-9]+)(_req_rate_[^_]+)?_gpus_([0-9]+)(_ctx_([0-9]+)_gen_([0-9]+))?\.json$ ]]; then + observed+=("${BASH_REMATCH[2]}") + gpus=${BASH_REMATCH[4]} + prefill_gpus=${BASH_REMATCH[6]:-$gpus} + decode_gpus=${BASH_REMATCH[7]:-0} + RESULT_FILENAME=${result_file%.json} IS_MULTINODE=true PREFILL_GPUS="$prefill_gpus" DECODE_GPUS="$decode_gpus" \ + python3 utils/process_result.py || batch_rc=1 + else + echo "Unrecognized result filename: $result_file" >&2 + batch_rc=1 + fi + done + read -r -a expected <<< "$CONC_LIST" + if [[ "$(printf '%s\n' "${observed[@]}" | sort -n)" != "$(printf '%s\n' "${expected[@]}" | sort -n)" ]]; then + echo 'Legacy result batch has missing, duplicate or unexpected concurrencies' >&2 + batch_rc=1 + fi + exit "$batch_rc" + fi - name: Upload result if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index f6fa76102..3db53a3b7 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -318,9 +318,16 @@ def process_multinode_results(env: Mapping[str, str]) -> int: if not expected or min(expected) <= 0: raise ValueError('CONC_LIST must contain positive concurrencies') points: list[dict[str, Any]] = [] + ignored_sidecars: list[str] = [] observed: set[int] = set() status = 0 for path in sorted(Path('.').glob(f"{env['RESULT_FILENAME']}_*.json")): + if path.name.endswith('.pytorch.json') or path.name in { + f"{env['RESULT_FILENAME']}_gpu_metrics_context.json", + f"{env['RESULT_FILENAME']}_gpu_metrics_identity.json", + }: + ignored_sidecars.append(path.name) + continue point: dict[str, Any] = {'source': path.name} try: match = re.search( @@ -364,7 +371,7 @@ def process_multinode_results(env: Mapping[str, str]) -> int: status = max(status, int(bool(missing or unexpected))) summary = {'expected_concurrencies': sorted(expected), 'missing_concurrencies': missing, 'unexpected_concurrencies': unexpected, - 'points': points, 'exit_code': status} + 'points': points, 'ignored_sidecars': ignored_sidecars, 'exit_code': status} Path(f"result_processing_{env['RESULT_FILENAME']}.json").write_text(json.dumps(summary, indent=2)) if missing or unexpected: print(f'[process_result] incomplete sweep: missing={missing}, unexpected={unexpected}', file=sys.stderr) diff --git a/utils/test_process_result.py b/utils/test_process_result.py index d14d6a87a..2b394e3bb 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -1623,3 +1623,90 @@ def test_public_power_audit_bounds_text_and_device_identifiers(): assert audit['exporter_image_sha256'] == 'a' * 64 assert len(audit['observed_gpu_ids']) == 1024 assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] + + +@pytest.mark.parametrize('sidecar', ['run_recipe_conc4_gpus_4_ctx_2_gen_2.pytorch.json', + 'run_gpu_metrics_context.json', 'run_gpu_metrics_identity.json']) +@pytest.mark.parametrize('point_state', ['valid', 'missing', 'malformed']) +def test_multinode_batch_retains_sidecars_without_counting_them_as_points( + tmp_path, multinode_env_vars, sample_benchmark_result, sidecar, point_state, +): + (tmp_path / sidecar).write_text('{"diagnostic": true}') + if point_state != 'missing': + (tmp_path / 'run_recipe_conc4_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': 4}) + if point_state == 'valid' else '{broken') + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4', + 'PYTHONPATH': str(REPO_ROOT)} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, env=env, + capture_output=True, text=True) + assert result.returncode == int(point_state != 'valid'), result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['ignored_sidecars'] == [sidecar] + assert receipt['missing_concurrencies'] == ([] if point_state == 'valid' else [4]) + assert (tmp_path / sidecar).read_text() == '{"diagnostic": true}' + + +def test_multinode_batch_rejects_unknown_point_filename( + tmp_path, multinode_env_vars, sample_benchmark_result, +): + for name in ['run_conc4_gpus_4_ctx_2_gen_2.json', 'run_conc16_gpus_bad.json']: + (tmp_path / name).write_text(json.dumps({**sample_benchmark_result, 'max_concurrency': 4})) + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4 16', 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1 + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [16] + assert any('filename lacks' in point.get('error', '') for point in receipt['points']) + + +@pytest.mark.parametrize('first_rc', [0, 7]) +@pytest.mark.parametrize('expected_concs', ['4 16', '4 8 16', '4']) +def test_multinode_workflow_processes_every_point_with_legacy_processor(tmp_path, first_rc, expected_concs): + (tmp_path / 'utils').mkdir() + (tmp_path / 'utils/process_result.py').write_text('''import json, os, sys +from pathlib import Path +stem = os.environ['RESULT_FILENAME'] +data = json.loads(Path(stem + '.json').read_text()) +Path('agg_' + stem + '.json').write_text(json.dumps(data)) +sys.exit(data['exit_code']) +''') + for conc, rc in [(4, first_rc), (16, 0)]: + (tmp_path / f'run_conc{conc}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({'exit_code': rc})) + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == 'Process result') + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step['run']], cwd=tmp_path, + env={**os.environ, 'RESULT_FILENAME': 'run', 'POWER_PRODUCER_SHA': '', + 'CONC_LIST': expected_concs, 'PYTHONPATH': '', + 'PATH': f"{Path(sys.executable).parent}:{os.environ['PATH']}"}, + capture_output=True, text=True, timeout=10) + assert result.returncode == int(bool(first_rc) or expected_concs != '4 16'), result.stderr + for conc in [4, 16]: + assert (tmp_path / f'agg_run_conc{conc}_gpus_4_ctx_2_gen_2.json').is_file() + + +def test_multinode_workflow_does_not_downgrade_processor_import_errors(tmp_path): + (tmp_path / 'infx/results').mkdir(parents=True) + (tmp_path / 'infx/__init__.py').touch() + (tmp_path / 'infx/results/__init__.py').touch() + (tmp_path / 'infx/results/fixed_sequence.py').write_text( + 'raise RuntimeError("broken processor import")\n') + (tmp_path / 'utils').mkdir() + (tmp_path / 'utils/process_result.py').write_text( + 'from pathlib import Path\nPath("legacy-called").touch()\n') + (tmp_path / 'run_conc4_gpus_4.json').write_text('{}') + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == 'Process result') + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step['run']], cwd=tmp_path, + env={**os.environ, 'RESULT_FILENAME': 'run', 'POWER_PRODUCER_SHA': '', + 'CONC_LIST': '4', 'PYTHONPATH': '', + 'PATH': f"{Path(sys.executable).parent}:{os.environ['PATH']}"}, + capture_output=True, text=True, timeout=10) + assert result.returncode != 0 + assert 'broken processor import' in result.stderr + assert not (tmp_path / 'legacy-called').exists() From 8408779c22817e681238ac323f7b4c4ad34977b0 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:02:55 -0700 Subject: [PATCH 11/17] test: preserve existing result test section formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 恢复合并时意外改动的测试注释分隔线与空行。 --- utils/test_process_result.py | 42 ++++++++++++------------------------ 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 350856ea7..9d3ba8ddf 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -155,11 +155,9 @@ def test_fixed_topology_rejects_empty_parallelism( build_result(sample_benchmark_result, {**single_node_env_vars, name: ""}) -# ====================================================================== - +# ============================================================================= # Test Fixtures - Based on real benchmark output structure -# ====================================================================== - +# ============================================================================= @pytest.fixture def sample_benchmark_result(): @@ -298,11 +296,9 @@ def failing_import(name, *args, **kwargs): ) -# ====================================================================== - +# ============================================================================= # Test script execution via subprocess -# ====================================================================== - +# ============================================================================= class TestProcessResultScript: """Tests for process_result.py script execution.""" @@ -532,11 +528,9 @@ def test_missing_result_file(self, tmp_path, single_node_env_vars): assert result.returncode != 0 -# ====================================================================== - +# ============================================================================= # Test latency and throughput calculations -# ====================================================================== - +# ============================================================================= class TestCalculations: """Tests for throughput and latency calculations.""" @@ -665,11 +659,9 @@ def test_multinode_zero_total_gpus_fails(self, tmp_path, sample_benchmark_result assert "Multinode results require at least one GPU" in result.stderr -# ====================================================================== - +# ============================================================================= # Test output file generation -# ====================================================================== - +# ============================================================================= class TestOutputFile: """Tests for output file generation.""" @@ -698,11 +690,9 @@ def test_output_file_has_correct_prefix(self, tmp_path, sample_benchmark_result, assert output_file.exists() -# ====================================================================== - +# ============================================================================= # Test edge cases -# ====================================================================== - +# ============================================================================= class TestEdgeCases: """Tests for edge cases and special scenarios.""" @@ -751,11 +741,9 @@ def test_integer_conversion(self, tmp_path, single_node_env_vars): assert isinstance(output_data["isl"], int) assert isinstance(output_data["osl"], int) -# ====================================================================== - +# ============================================================================= # Integration: power aggregation patches the agg JSON -# ====================================================================== - +# ============================================================================= class TestPowerAggregationIntegration: """End-to-end wiring: process_result.py invokes aggregate_power.py and @@ -1335,11 +1323,9 @@ def test_start_stop_gpu_monitor_amd_lifecycle(self, tmp_path): assert identity == {"gpu_data": []} -# ====================================================================== - +# ============================================================================= # Integration: multinode power aggregation patches the agg JSON -# ====================================================================== - +# ============================================================================= class TestMultinodePower: From f5c749d008571646d9600a1d8a9005126995f359 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:05:34 -0700 Subject: [PATCH 12/17] fix: preserve serving status when optional AMD telemetry staging fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 遥测拷贝失败只在必需功耗且服务成功时使作业失败，保留已有服务错误码。 --- benchmarks/multi_node/amd_utils/job.slurm | 8 +++++++- runners/test_amd_power_lifecycle.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 06bedd9e4..3d8a1d46c 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -813,7 +813,13 @@ exit \$DOCKER_EXIT_CODE BENCHMARK_STEP_RC=$? # Each host copies its own node-local, root-created artifacts as the runner user. # No raw telemetry is written as root into the shared checkout. -stage_native_power || BENCHMARK_STEP_RC=1 +if ! stage_native_power; then + echo 'PowerX: failed to stage native telemetry' >&2 + case "${REQUIRE_POWER:-0}" in + 1|true|TRUE|yes|YES) + if [[ "$BENCHMARK_STEP_RC" == 0 ]]; then BENCHMARK_STEP_RC=1; fi ;; + esac +fi if [[ "${KEEP_CONTAINERS}" != "1" ]]; then srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' '"$CLIENT_CONT_NAME"' 2>/dev/null || true' diff --git a/runners/test_amd_power_lifecycle.py b/runners/test_amd_power_lifecycle.py index cb2667380..e5f34ff7e 100644 --- a/runners/test_amd_power_lifecycle.py +++ b/runners/test_amd_power_lifecycle.py @@ -69,3 +69,18 @@ def test_amd_start_failure_respects_requirement(tmp_path, fault, required, servi assert receipt.exists() == (not is_required) assert ('inconsistent AMD node topology' if fault == 'topology' else 'uneven per-node TP layout') in result.stderr + + +@pytest.mark.parametrize('required', ['', '0', 'false', '1', 'true', 'YES']) +@pytest.mark.parametrize('serving_rc', [0, 7]) +@pytest.mark.parametrize('staging_rc', [0, 9]) +def test_amd_staging_failure_preserves_serving_outcome(tmp_path, required, serving_rc, staging_rc): + tail = (ROOT / 'benchmarks/multi_node/amd_utils/job.slurm').read_text().split( + 'BENCHMARK_STEP_RC=$?', 1)[1] + script = 'BENCHMARK_STEP_RC=$SERVING_RC\nstage_native_power() { return "$STAGING_RC"; }\n' + tail + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', script], + env={**os.environ, 'REQUIRE_POWER': required, 'SERVING_RC': str(serving_rc), + 'STAGING_RC': str(staging_rc), 'KEEP_CONTAINERS': '1'}, + capture_output=True, text=True, timeout=5) + expected = serving_rc or int(staging_rc != 0 and required in ['1', 'true', 'YES']) + assert result.returncode == expected, result.stderr From 6ed8f8b5ed69e71fdd6972ceaeaa8d97a160f347 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:05:34 -0700 Subject: [PATCH 13/17] fix: validate native role counts and retain boundary sample audits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 独立核对每种角色的 GPU 数量，并记录窗口外异常样本，窗口内异常仍阻止功耗有效性。 --- infx/results/power/native_multinode.py | 7 +++--- utils/test_native_multinode_power.py | 30 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py index 86b5d683b..78ff89889 100644 --- a/infx/results/power/native_multinode.py +++ b/infx/results/power/native_multinode.py @@ -202,10 +202,8 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, if expected_aggregate_gpus: if roles["prefill"] or roles["decode"] or len(roles["aggregate"]) != expected_aggregate_gpus: reasons.append("native_role_gpu_count_mismatch") - elif expected_decode_gpus: - if roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or len(roles["decode"]) != expected_decode_gpus: - reasons.append("native_role_gpu_count_mismatch") - elif roles["decode"] or len(roles["aggregate"] + roles["prefill"]) != expected_prefill_gpus: + elif (roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or + len(roles["decode"]) != expected_decode_gpus): reasons.append("native_role_gpu_count_mismatch") combined_path = validation_result.with_name(f"{validation_result.stem}_native.csv") combined_path.parent.mkdir(parents=True, exist_ok=True) @@ -247,6 +245,7 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, "observed_gpu_count": integration.observed_num_gpus if integration else 0, "per_gpu_role": {uuid: role for role, uuids in roles.items() for uuid in uuids}, "per_gpu_sample_counts": integration.per_gpu_sample_counts if integration else {}, + "boundary_degenerate_rows": integration.boundary_degenerate_rows if integration else {}, "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s if integration else {}, "producer": {"name": "inferencex-native-smi", "revisions": sorted(revisions), "producer_git_commit": next(iter(revisions)) if len(revisions) == 1 else None}, diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py index c2945cb75..e4234facc 100644 --- a/utils/test_native_multinode_power.py +++ b/utils/test_native_multinode_power.py @@ -275,3 +275,33 @@ def test_native_amd_abort_publishes_receipt_before_reaper_deadline(tmp_path): except ProcessLookupError: pass process.communicate() + + +def test_native_prefill_only_rejects_aggregate_role_devices(tmp_path): + root, bench, agg = _package(tmp_path) + manifest_path = root / 'node-1/manifest.json' + manifest = json.loads(manifest_path.read_text()) + manifest['role'] = 'aggregate' + manifest_path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=2, expected_decode_gpus=0, + require_power=True) == 1 + audit = json.loads((tmp_path / 'power_validation_result.json').read_text()) + assert 'native_role_gpu_count_mismatch' in audit['reasons'] + assert json.loads(agg.read_text())['power_valid'] == 0 + assert 'prefill_avg_power_w' not in json.loads(agg.read_text()) + + +@pytest.mark.parametrize('tick', [2, 4]) +def test_native_audit_retains_boundary_noise_without_relaxing_in_window_errors(tmp_path, tick): + root, bench, agg = _package(tmp_path) + for rank in [0, 1]: + with (root / f'node-{rank}/gpu_metrics.csv').open('a') as stream: + stream.write(f'{tick},0,N/A\n') + if rank == 0: + stream.write(f'{tick},0,0\n') + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, + require_power=True) == int(tick == 2) + audit = json.loads((tmp_path / 'power_validation_result.json').read_text()) + assert audit['boundary_degenerate_rows'] == ({'uuid-0': 2, 'uuid-1': 1} if tick == 4 else {}) + if tick == 4: + assert json.loads(agg.read_text())['total_gpu_energy_j'] == 800 From 85384dd3ba00223513ec5ef65a2f27b422b8c7ae Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:32:14 -0700 Subject: [PATCH 14/17] fix: validate native collector readiness and identity completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 发布就绪收据前检查采集进程仍存活，并将 AMD 结束身份采集失败写入失败收据。补充真实进程与清单回归测试。 --- benchmarks/native_power_collect.sh | 3 +- runners/test_native_collector_receipts.py | 77 +++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/benchmarks/native_power_collect.sh b/benchmarks/native_power_collect.sh index 69df07951..445d00ac3 100644 --- a/benchmarks/native_power_collect.sh +++ b/benchmarks/native_power_collect.sh @@ -35,7 +35,7 @@ finish() { if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi stop_gpu_monitor if [[ "$vendor" == amd ]]; then - _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json + amd-smi list --json > "$power_dir/gpu_metrics_devices_end.json" || collector_rc=1 else nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \ > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1 @@ -65,6 +65,7 @@ start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 if [[ "$vendor" == amd ]]; then _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json fi +_background_process_is_running "$GPU_MONITOR_PID" || exit 1 write_control "ready-$rank" ready while [[ ! -f "$control_dir/stop" ]]; do _background_process_is_running "$GPU_MONITOR_PID" || exit 1 diff --git a/runners/test_native_collector_receipts.py b/runners/test_native_collector_receipts.py index b981f57ce..63d9de677 100644 --- a/runners/test_native_collector_receipts.py +++ b/runners/test_native_collector_receipts.py @@ -14,3 +14,80 @@ def test_native_control_receipt_is_owned_before_publication(tmp_path): assert result.returncode == 0, result.stderr assert (tmp_path / 'done-0').read_text() == '7\n' assert not list(tmp_path.glob('*.tmp')) + + +def _collector_with_monitor(tmp_path, *, alive, end_identity_rc=0): + import shutil + import sys + + scripts = tmp_path / 'repo/benchmarks' + scripts.mkdir(parents=True) + shutil.copyfile(ROOT / 'benchmarks/native_power_collect.sh', scripts / 'native_power_collect.sh') + library = (ROOT / 'benchmarks/benchmark_lib.sh').read_text() + functions = [] + for name in ['_background_process_is_running', '_write_amd_smi_sidecar']: + start = library.index(name + '() {') + functions.append(library[start:library.index('\n}', start) + 2]) + (scripts / 'benchmark_lib.sh').write_text('\n'.join(functions) + ''' +GPU_MONITOR_PID="" GPU_MONITOR_VENDOR="" +start_gpu_monitor() { + GPU_MONITOR_VENDOR=amd + if [[ "$MONITOR_ALIVE" == 1 ]]; then + sleep 30 & + GPU_MONITOR_PID=$! + else + false & + GPU_MONITOR_PID=$! + wait "$GPU_MONITOR_PID" || true + fi +} +stop_gpu_monitor() { + kill "$GPU_MONITOR_PID" 2>/dev/null || true + wait "$GPU_MONITOR_PID" 2>/dev/null || true +} +amd-smi() { + if [[ -f "$IDENTITY_CALLED" && "$END_IDENTITY_RC" != 0 ]]; then return "$END_IDENTITY_RC"; fi + touch "$IDENTITY_CALLED" + printf '[{"gpu":0,"uuid":"test-gpu"}]\\n' +} +''') + control = tmp_path / 'control' + control.mkdir() + (control / 'stop').touch() + power = tmp_path / 'power' + result = subprocess.run(['bash', str(scripts / 'native_power_collect.sh'), str(power), + str(control), 'amd', '0', 'prefill', '0', '1'], + env={'PATH': f'{Path(sys.executable).parent}:/usr/bin:/bin', + 'PYTHONPATH': str(ROOT), 'MONITOR_ALIVE': str(int(alive)), + 'END_IDENTITY_RC': str(end_identity_rc), + 'IDENTITY_CALLED': str(tmp_path / 'identity-called')}, + capture_output=True, text=True, timeout=5) + return result, control, power + + +def test_dead_monitor_never_publishes_ready(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=False) + assert not (control / 'ready-0').exists(), result.stderr + assert (control / 'done-0').read_text().strip() != '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'failed' + + +def test_amd_end_identity_failure_is_not_a_successful_done_receipt(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=True, end_identity_rc=7) + assert (control / 'ready-0').exists(), result.stderr + assert (control / 'done-0').read_text().strip() != '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'failed' + + +def test_live_monitor_with_successful_identity_completes(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=True) + assert result.returncode == 0, result.stderr + assert (control / 'ready-0').exists() + assert (control / 'done-0').read_text().strip() == '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'complete' From 609cedc21b7c1a1c571f7e71effbd5410eebc34a Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:32:20 -0700 Subject: [PATCH 15/17] fix: forward required power into AMD worker containers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在实际 Docker 环境参数中传递 REQUIRE_POWER，并用嵌套 shell 命令展开结果验证服务与压测的严格失败策略。 --- benchmarks/multi_node/amd_utils/job.slurm | 1 + runners/test_amd_power_lifecycle.py | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 3d8a1d46c..2df345612 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -429,6 +429,7 @@ DOCKER_ENV_COMMON=( -e SLURM_JOB_ID=\$SLURM_JOB_ID -e SLURM_JOB_NODELIST=\$SLURM_JOB_NODELIST -e NNODES=\$NNODES + -e REQUIRE_POWER=\${REQUIRE_POWER:-0} -e POWERX_HOST_UID=\$POWERX_HOST_UID -e POWERX_HOST_GID=\$POWERX_HOST_GID -e POWERX_COLLECTOR_REVISION=\$POWERX_COLLECTOR_REVISION diff --git a/runners/test_amd_power_lifecycle.py b/runners/test_amd_power_lifecycle.py index e5f34ff7e..27f6ae189 100644 --- a/runners/test_amd_power_lifecycle.py +++ b/runners/test_amd_power_lifecycle.py @@ -9,6 +9,20 @@ ROOT = Path(__file__).resolve().parents[1] +def _docker_container_env(required): + job = (ROOT / 'benchmarks/multi_node/amd_utils/job.slurm').read_text() + start = job.index('DOCKER_ENV_COMMON=(') + block = job[start:job.index('\n)', start) + 2] + # Execute the submit-shell array and the node-shell expansion used by docker. + script = block + '\ndocker() { printf \'%s\\0\' "$@"; }; export -f docker;\n' + ( + 'bash -c "docker run ${DOCKER_ENV_COMMON[*]} test-image"') + result = subprocess.run(['bash', '-e', '-c', script], + env={'PATH': '/usr/bin:/bin', 'WS_PATH': '/workspace', + 'REQUIRE_POWER': required}, capture_output=True, check=True) + args = result.stdout.decode().split('\0') + return dict(args[index + 1].split('=', 1) for index, arg in enumerate(args) if arg == '-e') + + @pytest.mark.parametrize('phase', ['ready', 'done']) @pytest.mark.parametrize('required', ['', '1', 'true', 'YES']) @pytest.mark.parametrize('serving_rc', [0, 7]) @@ -33,7 +47,7 @@ def test_amd_collector_failure_respects_requirement(tmp_path, phase, required, s result = subprocess.run(['bash', str(scripts / 'bench.sh'), '1', '1', '1', '1', '/model', 'test', str(tmp_path / 'logs'), '8192', '1024', '1'], env={**os.environ, 'POWERX_CONTROL_DIR': str(control), 'NNODES': '1', - 'REQUIRE_POWER': required, 'SERVING_RC': str(serving_rc), + 'REQUIRE_POWER': _docker_container_env(required).get('REQUIRE_POWER', ''), 'SERVING_RC': str(serving_rc), 'CALL_RECEIPT': str(receipt), 'POWERX_HOST_UID': str(os.getuid()), 'POWERX_HOST_GID': str(os.getgid())}, capture_output=True, text=True, timeout=10) expected = 1 if required and phase == 'ready' else serving_rc or int(bool(required)) @@ -61,7 +75,7 @@ def test_amd_start_failure_respects_requirement(tmp_path, fault, required, servi 'GPUS_PER_NODE': '8', 'PREFILL_TP_SIZE': '9', 'DECODE_TP_SIZE': '8', 'xP': '1', 'yD': '1', 'NNODES': '2' if fault == 'topology' else '3', - 'NODE_RANK': '0', 'REQUIRE_POWER': required, + 'NODE_RANK': '0', 'REQUIRE_POWER': _docker_container_env(required).get('REQUIRE_POWER', ''), 'SERVING_RC': str(serving_rc), 'CALL_RECEIPT': str(receipt)}, capture_output=True, text=True, timeout=5) is_required = required in ['1', 'true', 'YES'] From ebaba2a05f438559dcb4b7881d9cf1ced9c7059f Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sun, 13 Sep 2026 21:47:54 -0700 Subject: [PATCH 16/17] test: derive AMD power lifecycle expectations from the required-power rule --- runners/test_amd_power_lifecycle.py | 74 +++++++++++++++++++---------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/runners/test_amd_power_lifecycle.py b/runners/test_amd_power_lifecycle.py index 27f6ae189..55606aa57 100644 --- a/runners/test_amd_power_lifecycle.py +++ b/runners/test_amd_power_lifecycle.py @@ -1,4 +1,11 @@ -"""Keep optional collector failures separate from AMD serving outcomes.""" +"""Keep optional collector failures separate from AMD serving outcomes. + +Expected exit codes below follow the required-power rule the AMD scripts +implement: REQUIRE_POWER in {1, true, TRUE, yes, YES} makes a collector +failure fail the run with rc 1 but never masks a nonzero serving rc; any +other value downgrades the failure to a warning and the run rc is the +serving rc. Failures before the benchmark starts abort it when required. +""" import os from pathlib import Path import shutil @@ -9,7 +16,7 @@ ROOT = Path(__file__).resolve().parents[1] -def _docker_container_env(required): +def test_job_slurm_forwards_require_power_into_the_container(): job = (ROOT / 'benchmarks/multi_node/amd_utils/job.slurm').read_text() start = job.index('DOCKER_ENV_COMMON=(') block = job[start:job.index('\n)', start) + 2] @@ -18,15 +25,22 @@ def _docker_container_env(required): 'bash -c "docker run ${DOCKER_ENV_COMMON[*]} test-image"') result = subprocess.run(['bash', '-e', '-c', script], env={'PATH': '/usr/bin:/bin', 'WS_PATH': '/workspace', - 'REQUIRE_POWER': required}, capture_output=True, check=True) + 'REQUIRE_POWER': 'sentinel-policy'}, capture_output=True, check=True) args = result.stdout.decode().split('\0') - return dict(args[index + 1].split('=', 1) for index, arg in enumerate(args) if arg == '-e') + forwarded = dict(args[index + 1].split('=', 1) for index, arg in enumerate(args) if arg == '-e') + assert forwarded['REQUIRE_POWER'] == 'sentinel-policy' -@pytest.mark.parametrize('phase', ['ready', 'done']) -@pytest.mark.parametrize('required', ['', '1', 'true', 'YES']) -@pytest.mark.parametrize('serving_rc', [0, 7]) -def test_amd_collector_failure_respects_requirement(tmp_path, phase, required, serving_rc): +@pytest.mark.parametrize(('phase', 'required', 'serving_rc', 'expected_rc', 'benchmark_runs'), [ + ('ready', '', 0, 0, True), + ('ready', '', 7, 7, True), + ('ready', '1', 0, 1, False), + ('ready', 'YES', 7, 1, False), + ('done', '', 7, 7, True), + ('done', '1', 0, 1, True), + ('done', 'true', 7, 7, True), +]) +def test_amd_collector_failure_respects_requirement(tmp_path, phase, required, serving_rc, expected_rc, benchmark_runs): benchmark_root = tmp_path / 'benchmarks' scripts = benchmark_root / 'multi_node/amd_utils' scripts.mkdir(parents=True) @@ -47,18 +61,23 @@ def test_amd_collector_failure_respects_requirement(tmp_path, phase, required, s result = subprocess.run(['bash', str(scripts / 'bench.sh'), '1', '1', '1', '1', '/model', 'test', str(tmp_path / 'logs'), '8192', '1024', '1'], env={**os.environ, 'POWERX_CONTROL_DIR': str(control), 'NNODES': '1', - 'REQUIRE_POWER': _docker_container_env(required).get('REQUIRE_POWER', ''), 'SERVING_RC': str(serving_rc), + 'REQUIRE_POWER': required, 'SERVING_RC': str(serving_rc), 'CALL_RECEIPT': str(receipt), 'POWERX_HOST_UID': str(os.getuid()), 'POWERX_HOST_GID': str(os.getgid())}, capture_output=True, text=True, timeout=10) - expected = 1 if required and phase == 'ready' else serving_rc or int(bool(required)) - assert result.returncode == expected, result.stderr - assert receipt.exists() == (not required or phase != 'ready') + assert result.returncode == expected_rc, result.stderr + assert receipt.exists() is benchmark_runs @pytest.mark.parametrize('fault', ['topology', 'uneven_tp']) -@pytest.mark.parametrize('required', ['', '0', 'false', '1', 'true', 'YES']) -@pytest.mark.parametrize('serving_rc', [0, 7]) -def test_amd_start_failure_respects_requirement(tmp_path, fault, required, serving_rc): +@pytest.mark.parametrize(('required', 'serving_rc', 'expected_rc', 'benchmark_runs'), [ + ('', 0, 0, True), + ('0', 7, 7, True), + ('false', 0, 0, True), + ('1', 0, 1, False), + ('true', 7, 1, False), + ('YES', 0, 1, False), +]) +def test_amd_start_failure_respects_requirement(tmp_path, fault, required, serving_rc, expected_rc, benchmark_runs): scripts = tmp_path / 'amd_utils' scripts.mkdir() for name in ['server.sh', 'power.sh']: @@ -75,20 +94,26 @@ def test_amd_start_failure_respects_requirement(tmp_path, fault, required, servi 'GPUS_PER_NODE': '8', 'PREFILL_TP_SIZE': '9', 'DECODE_TP_SIZE': '8', 'xP': '1', 'yD': '1', 'NNODES': '2' if fault == 'topology' else '3', - 'NODE_RANK': '0', 'REQUIRE_POWER': _docker_container_env(required).get('REQUIRE_POWER', ''), + 'NODE_RANK': '0', 'REQUIRE_POWER': required, 'SERVING_RC': str(serving_rc), 'CALL_RECEIPT': str(receipt)}, capture_output=True, text=True, timeout=5) - is_required = required in ['1', 'true', 'YES'] - assert result.returncode == (1 if is_required else serving_rc), result.stderr - assert receipt.exists() == (not is_required) + assert result.returncode == expected_rc, result.stderr + assert receipt.exists() is benchmark_runs assert ('inconsistent AMD node topology' if fault == 'topology' else 'uneven per-node TP layout') in result.stderr -@pytest.mark.parametrize('required', ['', '0', 'false', '1', 'true', 'YES']) -@pytest.mark.parametrize('serving_rc', [0, 7]) -@pytest.mark.parametrize('staging_rc', [0, 9]) -def test_amd_staging_failure_preserves_serving_outcome(tmp_path, required, serving_rc, staging_rc): +@pytest.mark.parametrize(('required', 'serving_rc', 'staging_rc', 'expected_rc'), [ + ('', 0, 0, 0), + ('', 0, 9, 0), + ('', 7, 9, 7), + ('false', 0, 9, 0), + ('1', 0, 0, 0), + ('1', 0, 9, 1), + ('1', 7, 9, 7), + ('YES', 0, 9, 1), +]) +def test_amd_staging_failure_preserves_serving_outcome(tmp_path, required, serving_rc, staging_rc, expected_rc): tail = (ROOT / 'benchmarks/multi_node/amd_utils/job.slurm').read_text().split( 'BENCHMARK_STEP_RC=$?', 1)[1] script = 'BENCHMARK_STEP_RC=$SERVING_RC\nstage_native_power() { return "$STAGING_RC"; }\n' + tail @@ -96,5 +121,4 @@ def test_amd_staging_failure_preserves_serving_outcome(tmp_path, required, servi env={**os.environ, 'REQUIRE_POWER': required, 'SERVING_RC': str(serving_rc), 'STAGING_RC': str(staging_rc), 'KEEP_CONTAINERS': '1'}, capture_output=True, text=True, timeout=5) - expected = serving_rc or int(staging_rc != 0 and required in ['1', 'true', 'YES']) - assert result.returncode == expected, result.stderr + assert result.returncode == expected_rc, result.stderr From b2852be361127c811c0e97092692e91689c5cdb1 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Mon, 14 Sep 2026 01:06:36 -0700 Subject: [PATCH 17/17] fix: scope AMD exit changes to active server paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：暂缓未启用的 ATOM/vLLM 多节点退出改动，明确当前 SGLang 验证及 AgentX 采集边界。 --- benchmarks/multi_node/amd_utils/server_atom.sh | 5 ++--- benchmarks/multi_node/amd_utils/server_vllm.sh | 5 ++--- docs/configuration-procedures.md | 2 ++ docs/configuration-procedures_zh.md | 2 ++ 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/server_atom.sh b/benchmarks/multi_node/amd_utils/server_atom.sh index 81cdaa6e0..5bad8fd39 100755 --- a/benchmarks/multi_node/amd_utils/server_atom.sh +++ b/benchmarks/multi_node/amd_utils/server_atom.sh @@ -378,8 +378,7 @@ if [ "$NODE_RANK" -eq 0 ]; then echo "DRY RUN: $BENCH_CMD" else set -x - BENCHMARK_EXIT_CODE=0 - eval "$BENCH_CMD" || BENCHMARK_EXIT_CODE=$? + eval "$BENCH_CMD" set +x fi @@ -620,4 +619,4 @@ else fi echo "Script completed successfully" -exit "${BENCHMARK_EXIT_CODE:-0}" +exit 0 \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh index 6a65bbe8a..5a6ce2357 100755 --- a/benchmarks/multi_node/amd_utils/server_vllm.sh +++ b/benchmarks/multi_node/amd_utils/server_vllm.sh @@ -323,8 +323,7 @@ if [ "$NODE_RANK" -eq 0 ]; then echo "DRY RUN: $BENCH_CMD" else set -x - BENCHMARK_EXIT_CODE=0 - eval "$BENCH_CMD" || BENCHMARK_EXIT_CODE=$? + eval "$BENCH_CMD" set +x fi @@ -540,4 +539,4 @@ fi # pkill -f etcd 2>/dev/null || true echo "Script completed successfully" -exit "${BENCHMARK_EXIT_CODE:-0}" +exit 0 diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 1eccf2bf7..5883b7b98 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -126,6 +126,8 @@ directory is not a completion signal. The AMD SGLang/ATOM/vLLM launchers enable native SMI collection for 8192-input/1024-output runs. Every serving node starts `benchmarks/native_power_collect.sh`; the client waits for all `ready-` receipts, then requests `stop` and waits for all `done-` receipts before tearing down servers. The shared collector also accepts NVIDIA SMI for launchers that do not use the srt-slurm/DCGM contract. +The selected active multi-node qualification uses SGLang. Inactive ATOM/vLLM multi-node exit changes are deferred; single-node ATOM/vLLM checks do not qualify those server paths. Native multi-node AgentX collection remains disabled. + Keep `native_power/node-/gpu_metrics.csv`, the start/end device identity snapshots and `manifest.json` together. Select the actual serving GPU indices, preserve physical node counts when workers span nodes, and stage node-local files as the host runner user into `LOGS/native_power`. The result processor uses each client's formal window, validates all node/role counts and UUID membership, and reuses the shared integration/percentile math. Aggregate deployments emit whole-deployment metrics; role metrics require real separate prefill/decode pools. Host `timedatectl NTPSynchronized` is recorded as clock context. The shared collector accepts `yes` or `true` as synchronized; other or missing values remain unsynchronized. It does not measure the offset between nodes; common-window trace coverage is still required, and runtime clock alignment remains part of fleet qualification. Missing clock context, a replaced UUID, a missing node or an incomplete collector lifecycle makes power unavailable. Local fixtures prove the format and failure behavior, not GPU runtime or dashboard publication. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 982bc23f9..6d13ca306 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -124,6 +124,8 @@ B300 DSXE 的 Kimi-K3 AgentX 路径在 `/scratch/models` 下挂载预置目标 AMD SGLang/ATOM/vLLM launcher 为 8192 输入、1024 输出的运行启用原生 SMI 采集。每个服务节点启动 `benchmarks/native_power_collect.sh`；客户端等待全部 `ready-` 回执，然后在基准结束后请求 `stop`，等待全部 `done-` 回执，再关闭服务。共享采集器也支持 NVIDIA SMI，供未采用 srt-slurm/DCGM 契约的 launcher 使用。 +当前所选现役多节点验证使用 SGLang。未启用的 ATOM/vLLM 多节点退出改动暂缓；单节点 ATOM/vLLM 检查不能证明这些服务路径。多节点 AgentX 原生采集仍未开启。 + 将 `native_power/node-/gpu_metrics.csv`、开始和结束时的设备身份快照及 `manifest.json` 一起保留。选择实际服务进程使用的 GPU 索引，worker 跨节点时保留真实物理节点数，并由宿主机 runner 用户将节点本地文件暂存到 `LOGS/native_power`。结果处理器使用每个客户端的正式窗口，验证全部节点、角色数量及 UUID 归属，再复用共享积分与百分位计算。聚合部署只输出全部署指标；角色指标要求实际分离的 prefill/decode 池。 宿主机的 `timedatectl NTPSynchronized` 状态作为时钟上下文记录。共享采集器将 `yes` 或 `true` 视为已同步；其他值或缺失值均视为未同步。它不测量节点间时钟偏移；仍需验证共同窗口的轨迹覆盖，并在集群运行验证中检查时钟对齐。缺失时钟上下文、UUID 被替换、节点缺失或采集生命周期未完成都会使功耗不可用。本地 fixture 只证明格式和失败处理行为，不能证明 GPU 运行或 dashboard 发布完成。