From c3040728c629feac18cdd26aca7cdbc246b03b73 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:23:08 -0700 Subject: [PATCH 01/17] fix: preserve complete request outcomes and diagnostic batches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留完整请求结果状态与诊断批次，拆分 Slurm 生命周期和原生功耗采集。 --- .../workflows/benchmark-multinode-tmpl.yml | 26 +-- .github/workflows/benchmark-tmpl.yml | 4 +- docs/results-and-ingestion.md | 12 +- docs/results-and-ingestion_zh.md | 12 +- infx/bench_serving/benchmark_outcome.py | 31 +++ infx/bench_serving/benchmark_serving.py | 17 +- infx/results/fixed_sequence.py | 109 ++++++++- infx/results/power/__init__.py | 1 + infx/results/power/audit.py | 50 ++++ perf-changelog.yaml | 11 + utils/test_process_result.py | 217 +++++++++++++++++- 11 files changed, 444 insertions(+), 46 deletions(-) create mode 100644 infx/bench_serving/benchmark_outcome.py create mode 100644 infx/results/power/audit.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index bae33211b..80b0842f8 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -470,7 +470,7 @@ jobs: fi - name: Process result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ always() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} env: RUNNER_TYPE: ${{ inputs.runner }} run: | @@ -480,29 +480,10 @@ jobs: export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA" fi - # Process each result file - for result_file in ${RESULT_FILENAME}_*.json; do - if [ -f "$result_file" ]; then - echo "Processing $result_file" - # Extract GPU count, prefill_gpus and decode_gpus from filename for tp_size calculation - gpus=$(echo "$result_file" | sed -n "s/.*_gpus_\([0-9][0-9]*\).*\.json/\1/p") - prefill_gpus=$(echo "$result_file" | sed -n "s/.*_ctx_\([0-9]*\).*\.json/\1/p") - decode_gpus=$(echo "$result_file" | sed -n "s/.*_gen_\([0-9]*\).*\.json/\1/p") - - if [ -n "$gpus" ]; then - # Non-disagg results lack _ctx_/_gen_ fields; all GPUs belong to the single worker - if [ -z "$prefill_gpus" ] || [ -z "$decode_gpus" ]; then - prefill_gpus=$gpus - decode_gpus=0 - fi - echo "Extracted: gpus=$gpus, prefill_gpus=$prefill_gpus, decode_gpus=$decode_gpus" - RESULT_FILENAME=${result_file%.json} IS_MULTINODE=true PREFILL_GPUS="$prefill_gpus" DECODE_GPUS="$decode_gpus" python3 utils/process_result.py - fi - fi - done + python3 utils/process_result.py --all - name: Upload result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bmk_${{ env.RESULT_FILENAME }} @@ -518,6 +499,7 @@ jobs: agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json LOGS/power/** + result_processing_${{ env.RESULT_FILENAME }}.json LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json LOGS/agentic/**/agentic_power_timezone_offset.txt diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index b08ef0775..4d1b817ea 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -395,14 +395,14 @@ jobs: fi - name: Process result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ always() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} env: RUNNER_TYPE: ${{ inputs.runner }} run: | python3 utils/process_result.py - name: Upload result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bmk_${{ env.RESULT_FILENAME }} diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index daba2715b..a1e357b30 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -92,10 +92,20 @@ The fixed-sequence transformer requires runner, framework, precision, speculativ | Latency and interactivity | Each benchmark input key ending in `ms` is converted from milliseconds to seconds with `_ms` removed. Keys containing `tpot` also produce an `intvty` reciprocal. | | Optional runtime metadata | `router` as exactly `{name, version}`, `kv_p2p_transfer`, and measured power patched from `gpu_metrics.csv` when available | -Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physical GPU count. Multinode per-GPU denominators use the declared prefill and decode GPU counts. Invalid or missing required metadata fails transformation. Power aggregation is explicitly best effort and cannot fail the benchmark aggregate. +Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physical GPU count. Multinode per-GPU denominators use the declared prefill and decode GPU counts. Invalid or missing required metadata fails transformation. Power aggregation is best effort by default; `REQUIRE_POWER=1` fails the job after preserving available results and audits when power validation fails. InferenceX-app treats routing fields as columns or config dimensions and stores numeric measurements in `benchmark_results.metrics` JSONB. The mapper supports v1 shared topology, v2 split prefill/decode topology, and nested v3 AgentX metrics. Unknown numeric metrics are retained and warned about, which permits schema growth without silently losing numeric data. +### Fixed-sequence outcomes and PowerX audits + +The serving client records `benchmark_outcome` before saving its raw result. It retains the existing maximum request-failure rate of 5%, including the requested/completed/failed counts. The processor verifies this record, copies it to the aggregate, and returns failure even when telemetry is valid. Zero successful requests retain a diagnostic aggregate without fabricated reciprocal latency. Legacy results without outcome metadata remain distinguishable; power validity alone never establishes benchmark success or answer quality. + +`power_invalid_reasons` and `power_audit` carry a bounded summary alongside numeric metrics. The summary includes the available measurement window, expected/observed GPU counts, sampling diagnostics, observed device identifiers and producer pin. Its `source` names the retained `power_validation_*.json` sidecar. Device identifiers retain the collector's semantics; local SMI indices are not physical UUID proof. + +For multinode fixed-sequence jobs, `utils/process_result.py --all` processes every available result before returning failure. It accepts `_c_gpus_...`, `_conc_gpus_...`, and AMD `_concurrency__req_rate__gpus_...` filenames, including `inf` request rates. It compares result concurrencies with `CONC_LIST`, rejects duplicate or contradictory point identities, and records omissions/errors in `result_processing_.json`. Aggregate workers pass `AGGREGATE_GPUS` with zero role GPU counts to telemetry validation; separate prefill/decode energy remains absent. Existing layout fields remain compatible, and `num_aggregate_gpu` marks the shared allocation. + +Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index cc5c9e302..6a054d5f0 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -92,10 +92,20 @@ shape: array of benchmark row objects | 延迟和交互性 | 基准输入中每个以 `ms` 结尾的键都会从毫秒换算为秒，并移除 `_ms`。包含 `tpot` 的键还会产生其倒数 `intvty`。 | | 可选运行时元数据 | 形式必须精确为 `{name, version}` 的 `router`、`kv_p2p_transfer`，以及在可用时由 `gpu_metrics.csv` 补入的实测功耗 | -单节点 GPU 数为 `tp * pp * pcp_size`。DCP 不会增加物理 GPU 数。多节点每 GPU 指标的分母使用声明的 prefill 和 decode GPU 数。无效或缺失的必需元数据会使转换失败。功耗聚合明确采用尽力而为模式，不能导致基准聚合失败。 +单节点 GPU 数为 `tp * pp * pcp_size`。DCP 不会增加物理 GPU 数。多节点每 GPU 指标的分母使用声明的 prefill 和 decode GPU 数。无效或缺失的必需元数据会使转换失败。功耗聚合默认尽力而为；当设置 `REQUIRE_POWER=1` 时，功耗验证失败会在保留已有结果和审计后使任务失败。 InferenceX-app 将路由字段作为列或配置维度，并把数值测量存入 `benchmark_results.metrics` JSONB。映射器支持共享拓扑的 v1、拆分 prefill/decode 拓扑的 v2，以及嵌套 AgentX 指标的 v3。未知数值指标会被保留并产生警告，因此架构可以扩展，同时不会无提示地丢失数值数据。 +### 固定序列基准结果状态与 PowerX 审计 + +服务客户端在保存原始结果前写入 `benchmark_outcome`，保留现有的 5% 最大请求失败率，以及请求总数、完成数和失败数。处理器检查该记录并复制到聚合结果中；即使遥测有效，请求失败率超限仍返回失败。零成功请求会保留诊断聚合结果，但不会生成不存在的延迟倒数。没有状态元数据的历史结果仍可区分；功耗有效不能证明基准成功或答案质量。 + +`power_invalid_reasons` 和 `power_audit` 在数值指标旁携带有界摘要，包括可用的测量窗口、预期与观测 GPU 数、采样诊断、观测设备标识和生产者版本。`source` 指向保留的 `power_validation_*.json` 工件名称。设备标识保留采集器原有语义，本地 SMI 序号不是物理 UUID 的证明。 + +对于多节点固定序列任务，`utils/process_result.py --all` 先处理所有已有结果，再返回失败。它接受 `_c_gpus_...`、`_conc_gpus_...` 和 AMD 的 `_concurrency__req_rate__gpus_...` 文件名，也支持 `inf` 请求速率。它将结果并发度与 `CONC_LIST` 比较，拒绝重复或矛盾的点身份，并将遗漏和错误记录到 `result_processing_.json`。共享工作池通过 `AGGREGATE_GPUS` 及零值角色 GPU 数进行遥测验证；独立的 prefill/decode 能耗保持缺失。现有布局字段保持兼容，`num_aggregate_gpu` 明确标记共享分配。 + +启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 + ## 评测工件 ### 单配置身份和收集 diff --git a/infx/bench_serving/benchmark_outcome.py b/infx/bench_serving/benchmark_outcome.py new file mode 100644 index 000000000..e14dca470 --- /dev/null +++ b/infx/bench_serving/benchmark_outcome.py @@ -0,0 +1,31 @@ +"""The serving client's existing request-failure gate, preserved with results.""" + +from __future__ import annotations + +from typing import Literal, TypedDict + + +MAX_FAILURE_RATE = 0.05 + + +class BenchmarkOutcome(TypedDict): + status: Literal["passed", "failed"] + requested: int + completed: int + failed: int + max_failure_rate: float + + +def benchmark_outcome(requested: int, completed: int) -> BenchmarkOutcome: + """Record the same five-percent gate before cleanup or artifact upload.""" + if (type(requested) is not int or type(completed) is not int + or requested <= 0 or not 0 <= completed <= requested): + raise ValueError("Benchmark request counts must satisfy 0 <= completed <= requested > 0") + failed = requested - completed + return { + "status": "failed" if failed / requested > MAX_FAILURE_RATE else "passed", + "requested": requested, + "completed": completed, + "failed": failed, + "max_failure_rate": MAX_FAILURE_RATE, + } diff --git a/infx/bench_serving/benchmark_serving.py b/infx/bench_serving/benchmark_serving.py index 82dfec973..1619d28ae 100644 --- a/infx/bench_serving/benchmark_serving.py +++ b/infx/bench_serving/benchmark_serving.py @@ -62,6 +62,7 @@ from argparse import ArgumentParser as FlexibleArgumentParser from .benchmark_utils import convert_to_pytorch_benchmark_format +from .benchmark_outcome import benchmark_outcome from .encoding_dsv4 import encode_messages as dsv4_encode_messages MILLISECONDS_TO_SECONDS_CONVERSION = 1000 @@ -898,6 +899,11 @@ def main(args: argparse.Namespace): lora_modules=args.lora_modules, )) + # Preserve the request gate before writing results: cleanup or a surrounding + # launcher must not turn an error-containing benchmark into a passed point. + outcome = benchmark_outcome(args.num_prompts, benchmark_result["completed"]) + benchmark_result["benchmark_outcome"] = outcome + # Save config and results to json if args.save_result: result_json: Dict[str, Any] = {} @@ -957,14 +963,11 @@ def main(args: argparse.Namespace): json.dump(result_json, outfile) save_to_pytorch_benchmark_format(args, result_json, file_name) - max_failure_rate = 0.05 - completed = benchmark_result["completed"] - failure_rate = 1 - completed / args.num_prompts - if failure_rate > max_failure_rate: + if outcome["status"] == "failed": raise SystemExit( - f"FAIL: request failure rate {failure_rate:.1%} exceeds " - f"{max_failure_rate:.0%} threshold " - f"({completed}/{args.num_prompts} completed)" + f"FAIL: request failure rate {outcome['failed'] / outcome['requested']:.1%} exceeds " + f"{outcome['max_failure_rate']:.0%} threshold " + f"({outcome['completed']}/{outcome['requested']} completed)" ) diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index ef6ad2b8e..f6fa76102 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -3,7 +3,9 @@ from __future__ import annotations import json +import math import os +import re import sys from collections.abc import Iterable, Mapping from pathlib import Path @@ -12,6 +14,7 @@ from .metadata import parse_component_metadata from .power import ALL_POWER_METRIC_KEYS, POWER_METRIC_SCHEMA_VERSION, with_power_metrics from .topology import Parallelism, validate_parallelism +from infx.bench_serving.benchmark_outcome import benchmark_outcome _BASE_ENV_VARS = ( @@ -49,6 +52,14 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s 'isl': int(env['ISL']), 'osl': int(env['OSL']), } + if 'benchmark_outcome' in benchmark: + outcome = benchmark['benchmark_outcome'] + expected = benchmark_outcome(outcome['requested'], outcome['completed']) + if (outcome != expected + or benchmark.get('completed', expected['completed']) != expected['completed'] + or benchmark.get('num_prompts', expected['requested']) != expected['requested']): + raise ValueError("Benchmark outcome does not match the recorded request counts and gate") + data['benchmark_outcome'] = expected router = parse_component_metadata(env.get('ROUTER_METADATA'), 'ROUTER_METADATA') if router is not None: @@ -61,9 +72,6 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s is_multinode = env.get('IS_MULTINODE', 'false').lower() == 'true' if is_multinode: - # TODO: Eventually will have to have a separate condition in here for multinode disagg and - # multinode agg. For now, just assume that multinode implies disagg. - multinode_vars = ['PREFILL_GPUS', 'DECODE_GPUS', 'PREFILL_NUM_WORKERS', 'PREFILL_TP', 'PREFILL_EP', 'PREFILL_DP_ATTN', 'DECODE_NUM_WORKERS', 'DECODE_TP', 'DECODE_EP', 'DECODE_DP_ATTN'] @@ -76,6 +84,14 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s ) prefill_gpus = int(env['PREFILL_GPUS']) decode_gpus = int(env['DECODE_GPUS']) + aggregate_gpus = int(env.get('AGGREGATE_GPUS', '0')) + if aggregate_gpus: + if aggregate_gpus < 0 or prefill_gpus or decode_gpus or disagg: + raise ValueError("Aggregate GPUs require non-disaggregated topology without role GPUs") + # Preserve the existing multinode layout fields for consumers, but + # expose the actual shared role and validate telemetry as `agg`. + prefill_gpus = aggregate_gpus + data['num_aggregate_gpu'] = aggregate_gpus prefill_num_workers = int(env['PREFILL_NUM_WORKERS']) prefill = Parallelism( tp=int(env['PREFILL_TP']), @@ -154,9 +170,9 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s data = data | single_node_data for key, value in benchmark.items(): - if key.endswith('ms'): + if key.endswith('ms') and math.isfinite(float(value)): data[key.replace('_ms', '')] = float(value) / 1000.0 - if 'tpot' in key: + if 'tpot' in key and math.isfinite(float(value)) and float(value) > 0: data[key.replace('_ms', '').replace( 'tpot', 'intvty')] = 1000.0 / float(value) return data @@ -221,7 +237,8 @@ def aggregate_power_result( source = Path(env.get('POWER_ARTIFACT_DIR', 'LOGS/power')) prefill_gpus = int(env['PREFILL_GPUS']) decode_gpus = int(env['DECODE_GPUS']) - expected_num_gpus = prefill_gpus + decode_gpus + aggregate_gpus = int(env.get('AGGREGATE_GPUS', '0')) + expected_num_gpus = prefill_gpus + decode_gpus + aggregate_gpus else: candidates = [env.get('GPU_METRICS_CSV'), 'gpu_metrics.csv', '/workspace/gpu_metrics.csv'] source = next( @@ -237,6 +254,7 @@ def aggregate_power_result( source, bench_path, agg_path, prefill_gpus=prefill_gpus, decode_gpus=decode_gpus, + aggregate_gpus=aggregate_gpus, expected_producer_sha=env.get('POWER_PRODUCER_SHA') or None, logs_root=Path(env.get('POWER_RESULT_ROOT', 'LOGS')), validation_result=validation_path, @@ -265,8 +283,7 @@ def aggregate_power_result( return int(require_power) -def main() -> int: - env = os.environ +def process_result(env: Mapping[str, str]) -> int: require_environment(env, _BASE_ENV_VARS) result_filename = env['RESULT_FILENAME'] bench_path = Path(f'{result_filename}.json') @@ -277,10 +294,86 @@ def main() -> int: with open(agg_path, 'w') as f: json.dump(data, f, indent=2) status = aggregate_power_result(env, bench_path, agg_path) + validation_path = Path(f'power_validation_{result_filename}.json') + from .power.audit import audit_summary + + result = json.loads(agg_path.read_text()) + try: + validation = json.loads(validation_path.read_text()) + result.update(audit_summary(validation, validation_path.name)) + except (OSError, ValueError, TypeError) as exc: + print(f'[process_result] audit summary unavailable: {exc}', file=sys.stderr) + result['power_invalid_reasons'] = ['validation_artifact_unavailable'] + # A required run must preserve its audit as well as numeric metrics. + status = max(status, int(env.get('REQUIRE_POWER', '').lower() in {'1', 'true', 'yes'})) + agg_path.write_text(json.dumps(result, indent=2)) with open(agg_path) as f: print(json.dumps(json.load(f), indent=2)) + return max(status, int(data.get('benchmark_outcome', {}).get('status') == 'failed')) + + +def process_multinode_results(env: Mapping[str, str]) -> int: + """Process every available point and preserve sweep omissions before failing.""" + expected = {int(value) for value in env['CONC_LIST'].split()} + if not expected or min(expected) <= 0: + raise ValueError('CONC_LIST must contain positive concurrencies') + points: list[dict[str, Any]] = [] + observed: set[int] = set() + status = 0 + for path in sorted(Path('.').glob(f"{env['RESULT_FILENAME']}_*.json")): + point: dict[str, Any] = {'source': path.name} + try: + match = re.search( + r'_(?:c|conc|concurrency_)(\d+)(?:_req_rate_[^_]+)?_gpus_(\d+)' + r'(?:_ctx_(\d+)_gen_(\d+))?$', path.stem) + if match is None: + raise ValueError('Result filename lacks concurrency and physical GPU counts') + concurrency, total = int(match[1]), int(match[2]) + raw = json.loads(path.read_text()) + if raw['max_concurrency'] != concurrency: + raise ValueError('Result concurrency does not match its filename') + if concurrency in observed: + raise ValueError('Duplicate concurrency result within one recipe') + observed.add(concurrency) + point['concurrency'] = concurrency + point_env = {**env, 'RESULT_FILENAME': path.stem, 'IS_MULTINODE': 'true'} + disagg = env['DISAGG'].lower() == 'true' + # Some disaggregated recipe groups include a shared-worker point. + # Its zero decode workers and filename must both describe aggregate + # execution; the group-level flag cannot manufacture role energy. + aggregate = not disagg or int(env['DECODE_NUM_WORKERS']) == 0 + if aggregate: + if match[3] is not None and (int(match[3]) != total or int(match[4]) != 0): + raise ValueError('Aggregate result contains separate role GPU counts') + point_env.update(DISAGG='false', PREFILL_GPUS='0', DECODE_GPUS='0', + AGGREGATE_GPUS=str(total)) + else: + if match[3] is None: + raise ValueError('Disaggregated result lacks prefill/decode GPU counts') + prefill, decode = int(match[3]), int(match[4]) + if prefill + decode != total: + raise ValueError('Role GPU counts do not equal total GPU count') + point_env.update(PREFILL_GPUS=str(prefill), DECODE_GPUS=str(decode), AGGREGATE_GPUS='0') + point['exit_code'] = process_result(point_env) + except (OSError, ValueError, KeyError, TypeError) as exc: + point.update(exit_code=1, error=str(exc)) + print(f'[process_result] {path}: {exc}', file=sys.stderr) + status = max(status, point['exit_code']) + points.append(point) + missing, unexpected = sorted(expected - observed), sorted(observed - expected) + status = max(status, int(bool(missing or unexpected))) + summary = {'expected_concurrencies': sorted(expected), + 'missing_concurrencies': missing, 'unexpected_concurrencies': unexpected, + 'points': points, 'exit_code': status} + Path(f"result_processing_{env['RESULT_FILENAME']}.json").write_text(json.dumps(summary, indent=2)) + if missing or unexpected: + print(f'[process_result] incomplete sweep: missing={missing}, unexpected={unexpected}', file=sys.stderr) return status +def main() -> int: + return process_multinode_results(os.environ) if sys.argv[1:] == ['--all'] else process_result(os.environ) + + if __name__ == '__main__': sys.exit(main()) diff --git a/infx/results/power/__init__.py b/infx/results/power/__init__.py index f0e1da95c..8007f5a91 100644 --- a/infx/results/power/__init__.py +++ b/infx/results/power/__init__.py @@ -59,6 +59,7 @@ def with_power_metrics( data["power_metric_schema_version"] = schema_version data["power_valid"] = int(power_valid) data.pop("power_invalid_reasons", None) + data.pop("power_audit", None) if power_valid: for key, value in metrics.items(): if value is None or not math.isfinite(value): diff --git a/infx/results/power/audit.py b/infx/results/power/audit.py new file mode 100644 index 000000000..6463976cf --- /dev/null +++ b/infx/results/power/audit.py @@ -0,0 +1,50 @@ +"""Bounded public audit metadata from retained power-validation sidecars.""" + +from __future__ import annotations + +import math +import re +from collections.abc import Mapping +from typing import Any + + +def audit_summary(validation: Mapping[str, Any], source: str) -> dict[str, Any]: + """Project the shared app audit contract without publishing raw telemetry.""" + audit: dict[str, Any] = {"source": source} + window = validation.get("benchmark_window") or {} + producer = validation.get("producer") or {} + fields = { + "window_start_unix": window.get("start_time_unix"), + "window_end_unix": window.get("end_time_unix"), + "expected_gpu_count": validation.get("expected_gpu_count"), + "observed_gpu_count": validation.get("observed_gpu_count"), + } + counts = validation.get("per_gpu_sample_counts") or {} + if counts: + fields["sample_count"] = sum(counts.values()) + gaps = validation.get("per_gpu_max_sample_gap_s") or {} + finite_gaps = [value for value in gaps.values() + if type(value) in (int, float) and math.isfinite(value)] + if finite_gaps: + fields["max_sample_gap_s"] = max(finite_gaps) + for key, value in fields.items(): + if type(value) in (int, float) and math.isfinite(value) and value >= 0: + audit[key] = value + for target, original in (("producer_sha", "producer_git_commit"), + ("exporter_image_sha256", "exporter_image_sha256")): + value = producer.get(original) + if isinstance(value, str) and 0 < len(value) <= 128: + audit[target] = value + ids = validation.get("observed_gpu_ids") + if ids is None: + ids = list((validation.get("per_gpu_role") or {}).keys()) + if ids: + audit["observed_gpu_ids"] = list(dict.fromkeys( + str(value) for value in ids if 0 < len(str(value)) <= 128 + ))[:1024] + return { + "power_invalid_reasons": [reason for reason in validation.get("reasons", []) + if isinstance(reason, str) + and re.fullmatch(r"[a-z][a-z0-9_]{0,63}", reason)][:32], + "power_audit": audit, + } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 88c163461..f3b85de24 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7399,3 +7399,14 @@ - "Disable adaptive verification in the EVAL_ONLY DSpark config as well. It trims verification requests on device, which the ROCm DeepseekV4IndexerBackend does not support, so the eval-only engine refused to start (run 34651830283, c32). Evals keep real block rejection; throughput settings are unchanged." - "EVAL_ONLY 的 DSpark 配置同样关闭自适应验证：它会在设备端裁剪验证请求，而 ROCm 的 DeepseekV4IndexerBackend 不支持该操作，导致仅评测引擎拒绝启动（运行 34651830283，c32）。评测仍保留真实块拒绝采样；吞吐设置不变。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Preserve request failure outcomes, retain all available fixed-sequence diagnostics, and reject incomplete concurrency batches before publishing benchmark artifacts." + - "保留请求失败状态及全部可用的固定序列诊断信息；发布基准产物前拒绝缺失并发点的批次。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 485021b36..d14d6a87a 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -28,12 +28,13 @@ def test_workflow_processes_results_through_compatibility_entrypoint( step = next(step for job in workflow["jobs"].values() for step in job.get("steps", []) if step.get("name", "").startswith("Process result")) multinode = workflow_name == "benchmark-multinode-tmpl.yml" - stem = "fixture_gpus_28_ctx_20_gen_8" if multinode else "fixture" + stem = "fixture_conc64_gpus_28_ctx_20_gen_8" if multinode else "fixture" (tmp_path / f"{stem}.json").write_text(json.dumps({ **sample_benchmark_result, "total_token_throughput": 56, "output_throughput": 28, })) env = {**(multinode_env_vars if multinode else single_node_env_vars), "RESULT_FILENAME": "fixture" if multinode else stem, "POWER_PRODUCER_SHA": "", + "CONC_LIST": "64", "PATH": f"{Path(sys.executable).parent}:{os.environ['PATH']}"} result = subprocess.run(["bash", "-euo", "pipefail", "-c", step["run"]], cwd=tmp_path, env=env, capture_output=True, text=True, timeout=10) @@ -845,7 +846,7 @@ def test_missing_csv_does_not_break_process_result(self, tmp_path, single_node_e assert "avg_power_w" not in patched assert "joules_per_output_token" not in patched assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] validation = json.loads( (tmp_path / "power_validation_benchmark_result.json").read_text() @@ -876,7 +877,7 @@ def test_missing_bench_timestamps_does_not_patch(self, tmp_path, single_node_env assert "avg_power_w" not in patched assert "joules_per_output_token" not in patched assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] def test_expected_gpu_count_mismatch_is_invalid(self, tmp_path, single_node_env_vars): """TP/PP/PCP topology is checked against the observed device IDs.""" @@ -902,7 +903,7 @@ def test_expected_gpu_count_mismatch_is_invalid(self, tmp_path, single_node_env_ assert result.returncode == 0, f"Script failed: {result.stderr}" patched = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] assert "total_gpu_energy_j" not in patched def test_require_power_propagates_validation_failure( @@ -1009,7 +1010,7 @@ def test_internal_aggregation_error_is_always_auditable( agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 0 - assert "power_invalid_reasons" not in agg + assert agg["power_invalid_reasons"] == ["aggregation_internal_error"] validation = json.loads( (tmp_path / "power_validation_benchmark_result.json").read_text() ) @@ -1416,3 +1417,209 @@ def test_strict_mode_passes_on_valid_package(self, tmp_path, power_env): assert result.returncode == 0, f"Script failed: {result.stderr}" agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert agg["power_valid"] == 1 + + +@pytest.mark.parametrize('completed,status', [(100, 'passed'), (95, 'passed'), (94, 'failed')]) +def test_request_outcome_preserves_existing_failure_threshold(completed, status): + from infx.bench_serving.benchmark_outcome import benchmark_outcome + + outcome = benchmark_outcome(100, completed) + assert outcome['status'] == status + assert outcome['failed'] == 100 - completed + assert outcome['max_failure_rate'] == 0.05 + + +def test_failed_client_is_preserved_with_valid_power(tmp_path, single_node_env_vars): + start, end = 1_700_000_100.0, 1_700_000_110.0 + TestPowerAggregationIntegration._write_nvidia_csv(tmp_path / 'gpu_metrics.csv', start, end, num_gpus=2) + outcome = {'status': 'failed', 'requested': 100, 'completed': 94, 'failed': 6, + 'max_failure_rate': 0.05} + raw = {'model_id': 'fixture', 'max_concurrency': 4, 'total_token_throughput': 500, + 'output_throughput': 100, 'benchmark_start_time_unix': start, + 'benchmark_end_time_unix': end, 'duration': 10, 'completed': 94, + 'total_input_tokens': 8192, 'total_output_tokens': 1024, + 'benchmark_outcome': outcome} + result = run_script(tmp_path, {**single_node_env_vars, 'TP': '2'}, raw) + assert result.returncode == 1, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['benchmark_outcome'] == outcome + assert aggregate['power_valid'] == 1 + assert aggregate['power_invalid_reasons'] == [] + assert aggregate['power_audit']['window_start_unix'] == start + assert aggregate['power_audit']['expected_gpu_count'] == 2 + assert aggregate['power_audit']['observed_gpu_count'] == 2 + assert aggregate['power_audit']['source'] == 'power_validation_benchmark_result.json' + + +def test_request_outcome_cannot_disagree_with_raw_counts(single_node_env_vars, sample_benchmark_result): + from infx.results.fixed_sequence import build_result + + raw = {**sample_benchmark_result, 'completed': 94, + 'benchmark_outcome': {'status': 'passed', 'requested': 100, 'completed': 100, + 'failed': 0, 'max_failure_rate': 0.05}} + with pytest.raises(ValueError, match='request counts and gate'): + build_result(raw, single_node_env_vars) + + +def test_multinode_aggregate_role_through_result_processor(tmp_path, multinode_env_vars): + pkg = build_package(tmp_path, bench_extra=TestMultinodePower.BENCH_EXTRA) + manifest_path = pkg.power_dir / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + for device in manifest['expected_devices']: + for assignment in device['assignments']: + assignment.update(worker_role='agg', het_group=None) + manifest_path.write_text(json.dumps(manifest)) + env = {**multinode_env_vars, 'DISAGG': 'false', 'PREFILL_GPUS': '0', + 'DECODE_GPUS': '0', 'AGGREGATE_GPUS': '4', 'POWER_PRODUCER_SHA': PRODUCER_SHA, + 'REQUIRE_POWER': '1'} + result = run_script(tmp_path, env, json.loads(pkg.original_result.read_text())) + assert result.returncode == 0, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['power_valid'] == 1 + assert aggregate['num_aggregate_gpu'] == 4 + assert aggregate['avg_power_w'] == 350 + assert aggregate['power_audit']['producer_sha'] == PRODUCER_SHA + assert set(ROLE_METRIC_KEYS).isdisjoint(aggregate) + + +@pytest.mark.parametrize('conc_token,rate_suffix', [ + ('c', ''), ('conc', ''), + ('concurrency_', '_req_rate_1'), ('concurrency_', '_req_rate_inf'), +]) +@pytest.mark.parametrize('expected_concs,missing', [('4 8 16', [8]), ('4 16', [])]) +def test_multinode_batch_preserves_points_and_checks_completeness( + tmp_path, multinode_env_vars, sample_benchmark_result, conc_token, rate_suffix, + expected_concs, missing, +): + for conc in (4, 16): + (tmp_path / f'run_recipe_{conc_token}{conc}{rate_suffix}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': conc})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': expected_concs, 'REQUIRE_POWER': '0'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == int(bool(missing)), result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == missing + assert receipt['unexpected_concurrencies'] == [] + assert len(receipt['points']) == 2 + for conc in (4, 16): + stem = f'run_recipe_{conc_token}{conc}{rate_suffix}_gpus_4_ctx_2_gen_2' + aggregate = json.loads((tmp_path / f'agg_{stem}.json').read_text()) + assert aggregate['power_valid'] == 0 + assert aggregate['power_invalid_reasons'] + assert (tmp_path / f'power_validation_{stem}.json').is_file() + + +@pytest.mark.parametrize('role_suffix', ['', '_ctx_4_gen_0']) +def test_multinode_batch_normalizes_legacy_zero_decode_aggregate( + tmp_path, multinode_env_vars, role_suffix, +): + pkg = build_package(tmp_path, bench_extra=TestMultinodePower.BENCH_EXTRA) + manifest_path = pkg.power_dir / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + for device in manifest['expected_devices']: + for assignment in device['assignments']: + assignment.update(worker_role='agg', het_group=None) + manifest_path.write_text(json.dumps(manifest)) + stem = f'run_recipe_c4_gpus_4{role_suffix}' + (tmp_path / f'{stem}.json').write_text(pkg.original_result.read_text()) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4', 'DECODE_NUM_WORKERS': '0', 'REQUIRE_POWER': '1', + 'POWER_PRODUCER_SHA': PRODUCER_SHA} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 0, result.stderr + aggregate = json.loads((tmp_path / f'agg_{stem}.json').read_text()) + assert aggregate['disagg'] is False + assert aggregate['num_aggregate_gpu'] == 4 + assert aggregate['num_decode_gpu'] == 0 + assert aggregate['decode_num_workers'] == 0 + assert aggregate['power_valid'] == 1 + assert aggregate['avg_power_w'] == 350 + assert set(ROLE_METRIC_KEYS).isdisjoint(aggregate) + + +@pytest.mark.parametrize('decode_workers,role_suffix', [ + ('0', '_ctx_2_gen_2'), ('0', '_ctx_3_gen_0'), ('1', ''), +]) +def test_multinode_batch_rejects_misdeclared_aggregate_role_counts( + tmp_path, multinode_env_vars, sample_benchmark_result, decode_workers, role_suffix, +): + stem = f'run_recipe_c4_gpus_4{role_suffix}' + (tmp_path / f'{stem}.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': 4})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4', 'DECODE_NUM_WORKERS': decode_workers} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['points'][0]['exit_code'] == 1 + assert not (tmp_path / f'agg_{stem}.json').exists() + + +def test_zero_successful_requests_preserves_diagnostic_result(tmp_path, single_node_env_vars, sample_benchmark_result): + raw = {**sample_benchmark_result, 'tpot_p50_ms': 0, 'tpot_p99_ms': float('nan'), + 'completed': 0, 'num_prompts': 4, + 'benchmark_outcome': {'status': 'failed', 'requested': 4, 'completed': 0, + 'failed': 4, 'max_failure_rate': 0.05}} + result = run_script(tmp_path, single_node_env_vars, raw) + assert result.returncode == 1, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['benchmark_outcome']['failed'] == 4 + assert aggregate['power_valid'] == 0 + assert 'intvty_p50' not in aggregate + assert 'tpot_p99' not in aggregate + + +@pytest.mark.parametrize('extra_conc,error', [(4, 'Duplicate concurrency'), (8, None)]) +def test_multinode_batch_rejects_extra_results_without_losing_other_points( + tmp_path, multinode_env_vars, sample_benchmark_result, extra_conc, error, +): + for label, conc in [('a', 4), ('b', extra_conc), ('c', 16)]: + (tmp_path / f'run_{label}_conc{conc}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': conc})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4 16'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [] + assert receipt['unexpected_concurrencies'] == ([] if error else [8]) + if error: + assert error in receipt['points'][1]['error'] + assert (tmp_path / 'agg_run_a_conc4_gpus_4_ctx_2_gen_2.json').is_file() + assert (tmp_path / 'agg_run_c_conc16_gpus_4_ctx_2_gen_2.json').is_file() + + +def test_multinode_empty_sweep_records_every_missing_point(tmp_path, multinode_env_vars): + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4 8'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [4, 8] + assert receipt['points'] == [] + + +def test_public_power_audit_bounds_text_and_device_identifiers(): + from infx.results.power.audit import audit_summary + + summary = audit_summary({ + 'producer': {'producer_git_commit': 'x' * 129, 'exporter_image_sha256': 'a' * 64}, + 'observed_gpu_ids': ['gpu0', 'gpu0', 'x' * 129] + [f'gpu{i}' for i in range(1, 1025)], + 'reasons': ['window_missing', ''] + [f'reason_{i}' for i in range(40)], + }, 'power_validation_run.json') + assert len(summary['power_invalid_reasons']) == 32 + assert '' not in summary['power_invalid_reasons'] + audit = summary['power_audit'] + assert 'producer_sha' not in audit + assert audit['exporter_image_sha256'] == 'a' * 64 + assert len(audit['observed_gpu_ids']) == 1024 + assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] From 57ae800333ac5d85c6cfb422e1b336a442cc2716 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:23:55 -0700 Subject: [PATCH 02/17] feat: add scoped fixed-sequence power requirements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为固定序列场景增加默认关闭的功耗要求，并贯通矩阵及工作流；保持 AgentX 和评估隔离。 --- .github/workflows/e2e-tests.yml | 4 +-- .github/workflows/run-sweep.yml | 3 ++ docs/configuration-procedures.md | 2 ++ docs/configuration-procedures_zh.md | 2 ++ infx/matrix/generate.py | 7 +++++ infx/matrix/validation.py | 5 ++++ .../test_generate_sweep_configs.py | 29 +++++++++++++++++++ 7 files changed, 50 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 0718ef718..f3dc97527 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -368,7 +368,7 @@ jobs: decode-dp-attn: ${{ matrix.config.decode.dp-attn }} decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} run-eval: false - require-power: ${{ inputs.require-power }} + require-power: ${{ inputs.require-power || (matrix.config['require-power'] == true && matrix.config['eval-only'] != true) }} power-producer-sha: ${{ inputs.power-producer-sha }} ref: ${{ inputs.ref }} @@ -692,7 +692,7 @@ jobs: spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} run-eval: false - require-power: ${{ inputs.require-power }} + require-power: ${{ inputs.require-power || (matrix.config['require-power'] == true && matrix.config['eval-only'] != true) }} ref: ${{ inputs.ref }} test-sweep-evals: diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 747017020..88afbba16 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -413,6 +413,7 @@ jobs: klaud-run: &klaud-run ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login == 'Klaud-Cold' && (startsWith(github.head_ref, 'klaud/auto-') || startsWith(github.head_ref, 'klaude/auto-')) }} exp-name: ${{ matrix.config.exp-name }} recipe-fingerprint: ${{ matrix.config['recipe-fingerprint'] || '' }} + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} isl: ${{ matrix.config.isl }} osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} @@ -474,6 +475,7 @@ jobs: kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} exp-name: ${{ matrix.config.exp-name }} recipe-fingerprint: ${{ matrix.config['recipe-fingerprint'] || '' }} + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} @@ -540,6 +542,7 @@ jobs: klaud-run: *klaud-run exp-name: ${{ matrix.config.exp-name }} recipe-fingerprint: ${{ matrix.config['recipe-fingerprint'] || '' }} + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} isl: ${{ matrix.config.isl }} osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index d853f6493..581e5d09a 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -88,6 +88,8 @@ Sources: [`configs/CONFIGS.md`](../configs/CONFIGS.md), [`validation.py`](../uti 6. For srt-slurm, update recipe and master entry together. For llm-d, update the llm-d recipe/orchestration and master entry together. 7. Append the trigger entry, generate only the affected key first, and inspect every emitted point. +Fixed-sequence `8192/1024` scenarios may set `require-power: true` to opt into validated measured power. The matrix passes this flag to standard sweeps and manual E2E throughput jobs; eval-only and AgentX rows do not inherit it. Omit the field to preserve existing behavior. Enable it only alongside the corresponding runtime and result adapter, then qualify the complete selected scope. + ## Register and set up a runner Setup source: [`utils/runner_setup/RUNNER_SETUP.md`](../utils/runner_setup/RUNNER_SETUP.md). Config source: [`configs/CONFIGS.md#runners`](../configs/CONFIGS.md#runners). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 883f1f8e2..aeff271a8 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -88,6 +88,8 @@ STP（Single Token Prediction，单 Token 预测）是每次前向传播生成 6. srt-slurm 必须同时更新配方和主条目；llm-d 必须同时更新 llm-d 配方/编排和主条目。 7. 追加触发条目，先只生成受影响的 key，并检查每个生成点。 +固定序列 `8192/1024` 场景可设置 `require-power: true`，要求经过验证的实测功耗。矩阵将此标记传递给标准 sweep 和手动 E2E 吞吐作业；eval-only 和 AgentX 行不继承该标记。省略此字段可保留现有行为。仅在对应 runtime 和结果适配器同时交付时启用，然后验证完整选定范围。 + ## 注册并设置 runner 设置来源：[`utils/runner_setup/RUNNER_SETUP.md`](../utils/runner_setup/RUNNER_SETUP.md)。配置来源：[`configs/CONFIGS.md#runners`](../configs/CONFIGS.md#runners)。 diff --git a/infx/matrix/generate.py b/infx/matrix/generate.py index 55c46da58..ee6166045 100644 --- a/infx/matrix/generate.py +++ b/infx/matrix/generate.py @@ -136,6 +136,7 @@ def smoke_entries(entries: list[dict]) -> list[dict]: if not row.get('run-eval'): continue row = {**row, 'eval-only': True} + row.pop(Fields.REQUIRE_POWER.value, None) if row.get('prefill') is not None: row['conc'] = [row['eval-conc']] row['eval-all-concs'] = False @@ -808,6 +809,9 @@ def _fixed_sequence_entries( is_multinode = config.get(Fields.MULTINODE.value, False) disagg = config.get(Fields.DISAGG.value, False) isl, osl = sequence[Fields.ISL.value], sequence[Fields.OSL.value] + require_power = sequence.get(Fields.REQUIRE_POWER.value, False) + if require_power and (isl, osl) != (8192, 1024): + raise ValueError("require-power rollout supports only fixed-sequence 8192/1024") model_code = config[Fields.MODEL_PREFIX.value] spec_decoding = benchmark.get(Fields.SPEC_DECODING.value, "none") if is_multinode: @@ -831,6 +835,8 @@ def _fixed_sequence_entries( Fields.ISL.value: isl, Fields.OSL.value: osl, } + if require_power: + entry[Fields.REQUIRE_POWER.value] = True if is_multinode: entry.update({ Fields.SPEC_DECODING.value: spec_decoding, @@ -1316,6 +1322,7 @@ def select_matrix_evals( rows = [row for row in rows if row.get(Fields.RUN_EVAL.value, False)] for row in rows: row[Fields.EVAL_ONLY.value] = True + row.pop(Fields.REQUIRE_POWER.value, None) return rows diff --git a/infx/matrix/validation.py b/infx/matrix/validation.py index a3614b0bc..8ea0acffb 100644 --- a/infx/matrix/validation.py +++ b/infx/matrix/validation.py @@ -76,6 +76,7 @@ class Fields(Enum): NUM_NODES = 'num-nodes' NODE_COUNT = 'node-count' DURATION = 'duration' + REQUIRE_POWER = 'require-power' # Matrix entry fields CONC = 'conc' @@ -168,6 +169,7 @@ class SingleNodeMatrixEntry(BaseModel): runner: str isl: int osl: int + require_power: bool = Field(default=False, alias=Fields.REQUIRE_POWER.value, strict=True) tp: int pp: int = Field(gt=0, strict=True) dcp_size: int = Field(alias=Fields.DCP_SIZE.value, gt=0, strict=True) @@ -268,6 +270,7 @@ class MultiNodeMatrixEntry(BaseModel): node_count: int = Field(alias=Fields.NODE_COUNT.value, gt=0, strict=True) isl: int osl: int + require_power: bool = Field(default=False, alias=Fields.REQUIRE_POWER.value, strict=True) prefill: WorkerConfig decode: WorkerConfig conc: List[int] @@ -629,6 +632,7 @@ class SingleNodeSeqLenConfig(BaseModel): isl: int osl: int + require_power: bool = Field(default=False, alias=Fields.REQUIRE_POWER.value, strict=True) search_space: List[SingleNodeSearchSpaceEntry] = Field( alias=Fields.SEARCH_SPACE.value) @@ -639,6 +643,7 @@ class MultiNodeSeqLenConfig(BaseModel): isl: int osl: int + require_power: bool = Field(default=False, alias=Fields.REQUIRE_POWER.value, strict=True) search_space: List[MultiNodeSearchSpaceEntry] = Field( alias=Fields.SEARCH_SPACE.value) diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 67699d254..eeae0b83d 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -3241,3 +3241,32 @@ def test_empty_matrix_has_no_jobs(self, split_e2e_configs): output = split_e2e_configs([]) assert output and all(rows == [] for rows in output.values()) + + +@pytest.mark.parametrize("multinode", [False, True]) +def test_require_power_is_scoped_to_one_fixed_sequence(multinode, sample_single_node_config, + sample_multinode_config, sample_runner_config): + from infx.matrix.generate import expand_full_sweep, select_matrix_evals + from infx.matrix.validation import MultiNodeSeqLenConfig, SingleNodeSeqLenConfig + + config = sample_multinode_config if multinode else sample_single_node_config + entry = next(iter(config.values())) + sequences = entry["scenarios"]["fixed-seq-len"] + if multinode: + sequences.append(copy.deepcopy(sequences[0])) + sequences[-1]["isl"] = 8192 + before = expand_full_sweep(config, sample_runner_config) + assert all("require-power" not in row for row in before) + sequences[-1]["require-power"] = True + schema = MultiNodeSeqLenConfig if multinode else SingleNodeSeqLenConfig + schema.model_validate(sequences[-1]) + after = expand_full_sweep(config, sample_runner_config) + assert len(before) == len(after) + for original, row in zip(before, after): + assert row == ({**original, "require-power": True} if original["isl"] == 8192 else original) + evals = select_matrix_evals(copy.deepcopy(after), mode="subset") + assert evals + assert all("require-power" not in row for row in evals) + sequences[0]["require-power"] = True + with pytest.raises(ValueError, match="only fixed-sequence 8192/1024"): + expand_full_sweep(config, sample_runner_config) From b44efcd09eb7fb63061adb1e883ae4802b28241f Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:24:13 -0700 Subject: [PATCH 03/17] feat: separate native multinode power collection contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 拆分原生多节点功耗采集契约、UTC 上下文和生命周期屏障，不启用新路径。 --- .../workflows/benchmark-multinode-tmpl.yml | 1 + .github/workflows/benchmark-tmpl.yml | 4 + benchmarks/benchmark_lib.sh | 7 +- benchmarks/native_power_collect.sh | 73 +++++ benchmarks/native_power_lifecycle.sh | 77 +++++ docs/results-and-ingestion.md | 6 + docs/results-and-ingestion_zh.md | 6 + infx/results/fixed_sequence.py | 15 + infx/results/power/native_multinode.py | 286 ++++++++++++++++++ infx/results/power/single_node.py | 59 +++- perf-changelog.yaml | 11 + runners/test_native_collector_barriers.py | 46 +++ runners/test_native_collector_receipts.py | 16 + utils/test_aggregate_power.py | 255 +++++++++++++++- utils/test_native_multinode_power.py | 201 ++++++++++++ 15 files changed, 1045 insertions(+), 18 deletions(-) create mode 100644 benchmarks/native_power_collect.sh create mode 100644 benchmarks/native_power_lifecycle.sh create mode 100644 infx/results/power/native_multinode.py create mode 100644 runners/test_native_collector_barriers.py create mode 100644 runners/test_native_collector_receipts.py create mode 100644 utils/test_native_multinode_power.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 80b0842f8..e9f3a3c90 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -499,6 +499,7 @@ jobs: agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json LOGS/power/** + LOGS/native_power/** result_processing_${{ env.RESULT_FILENAME }}.json LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 4d1b817ea..1da933f63 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -445,11 +445,13 @@ jobs: name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }} path: | gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json if-no-files-found: ignore @@ -462,12 +464,14 @@ jobs: ${{ env.RESULT_FILENAME }}.json agg_${{ env.RESULT_FILENAME }}.json gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json results/agentic_power_window.json results/agentic_power_timezone_offset.txt diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 305a118ef..9d99a4cc9 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -177,6 +177,7 @@ start_gpu_monitor() { GPU_METRICS_CSV="$output" GPU_MONITOR_INTERVAL="$interval" export GPU_METRICS_CSV + printf '{"timestamp_timezone":"UTC"}\n' > "${output%.csv}_context.json" if command -v nvidia-smi &>/dev/null; then GPU_MONITOR_VENDOR="nvidia" @@ -185,7 +186,7 @@ start_gpu_monitor() { rm -f "${output%.csv}_identity.csv" echo "[GPU Monitor] Warning: NVIDIA identity sidecar failed" >&2 fi - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv -l "$interval" > "$output" 2>/dev/null & GPU_MONITOR_PID=$! echo "[GPU Monitor] Started NVIDIA (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)" @@ -196,7 +197,7 @@ start_gpu_monitor() { # Python; measured on MI355X: trailing ticks were lost at kill without it). # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated # headers, and flush every row so killing the pipe cannot discard buffered samples. - PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ + TZ=UTC PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ | awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" & GPU_MONITOR_PID=$! # Hardware energy-accumulator + identity snapshots; the end-side twin in @@ -232,7 +233,7 @@ stop_gpu_monitor() { case "$GPU_MONITOR_VENDOR" in nvidia) if _repair_truncated_gpu_metrics_tail; then - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv,noheader >> "$GPU_METRICS_CSV" 2>/dev/null || echo "[GPU Monitor] Warning: final NVIDIA sample failed" >&2 fi diff --git a/benchmarks/native_power_collect.sh b/benchmarks/native_power_collect.sh new file mode 100644 index 000000000..69df07951 --- /dev/null +++ b/benchmarks/native_power_collect.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# One native SMI collector per serving node. Raw files stay on node-local scratch; +# the launcher stages them as its host user after containers stop. +set -uo pipefail +power_dir=$1 +control_dir=$2 +vendor=$3 +rank=$4 +role=$5 +gpu_indices=$6 +num_nodes=$7 +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" +source "$repo_root/benchmarks/benchmark_lib.sh" +mkdir -p "$power_dir" +collector_rc=0 +finished=0 + +write_control() { + local path="$control_dir/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + # The control directory is created by the host user before Docker starts. + # Containers must not strand root-owned files in the shared runner tree. + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +finish() { + local incoming_rc=$? + [[ "$finished" == 0 ]] || return + if [[ "$incoming_rc" != 0 ]]; then collector_rc=$incoming_rc; fi + if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi + stop_gpu_monitor + if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json + else + nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \ + > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1 + fi + python3 -m infx.results.power.native_multinode end --directory "$power_dir" \ + --collector-exit-code "$collector_rc" || collector_rc=1 + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown -R "$POWERX_HOST_UID:$POWERX_HOST_GID" "$power_dir" || collector_rc=1 + fi + write_control "done-$rank" "$collector_rc" + finished=1 +} +trap finish EXIT +trap 'collector_rc=130; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 130' INT +trap 'collector_rc=143; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 143' TERM HUP + +case "${POWERX_CLOCK_SYNCHRONIZED:-false}" in + yes|true) clock_synchronized=true ;; + *) clock_synchronized=false ;; +esac + +python3 -m infx.results.power.native_multinode begin --directory "$power_dir" \ + --vendor "$vendor" --rank "$rank" --role "$role" --gpu-indices "$gpu_indices" \ + --num-nodes "$num_nodes" --clock-synchronized "$clock_synchronized" || exit 1 +start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 +[[ "$GPU_MONITOR_VENDOR" == "$vendor" ]] || exit 1 +if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json +fi +write_control "ready-$rank" ready +while [[ ! -f "$control_dir/stop" ]]; do + _background_process_is_running "$GPU_MONITOR_PID" || exit 1 + sleep 1 & + wait $! || true +done diff --git a/benchmarks/native_power_lifecycle.sh b/benchmarks/native_power_lifecycle.sh new file mode 100644 index 000000000..b24d1dada --- /dev/null +++ b/benchmarks/native_power_lifecycle.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Shared barriers for native collectors. Launchers own host scratch and mounts. + +powerx_start_collector() { + local power_dir="$1" control_dir="$2" vendor="$3" rank="$4" role="$5" gpus="$6" nodes="$7" + local indices + indices=$(seq -s, 0 "$((gpus - 1))") || return 1 + POWERX_CONTROL_DIR="$control_dir" + POWERX_NUM_NODES="$nodes" + bash "$(dirname "${BASH_SOURCE[0]}")/native_power_collect.sh" \ + "$power_dir" "$control_dir" "$vendor" "$rank" "$role" "$indices" "$nodes" & + POWERX_COLLECTOR_PID=$! +} + +powerx_write_control() { + local path="$POWERX_CONTROL_DIR/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +powerx_wait_collectors() { + local phase="$1" deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rank pending failed + while :; do + pending=0 + failed=0 + for ((rank=0; rank&2 + return 1 + fi + if [[ ! -f "$POWERX_CONTROL_DIR/$phase-$rank" ]]; then + pending=1 + elif [[ "$phase" == done && "$(cat "$POWERX_CONTROL_DIR/done-$rank")" != 0 ]]; then + failed=1 + fi + done + if [[ "$pending" == 0 ]]; then + [[ "$failed" == 0 ]] || echo "One or more PowerX collectors failed" >&2 + return "$failed" + fi + if (( SECONDS >= deadline )); then + echo "Timed out waiting for PowerX $phase receipts" >&2 + return 1 + fi + sleep 1 + done +} + +powerx_stop_collectors() { + local rc=0 + powerx_write_control stop stop || rc=$? + powerx_wait_collectors done || rc=$? + powerx_reap_collector || rc=$? + return "$rc" +} + +powerx_reap_collector() { + [[ -n "${POWERX_COLLECTOR_PID:-}" ]] || return 0 + local deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rc=0 + while kill -0 "$POWERX_COLLECTOR_PID" 2>/dev/null; do + if (( SECONDS >= deadline )); then + kill -TERM "$POWERX_COLLECTOR_PID" 2>/dev/null || true + sleep 2 + kill -KILL "$POWERX_COLLECTOR_PID" 2>/dev/null || true + rc=1 + break + fi + sleep 1 + done + wait "$POWERX_COLLECTOR_PID" || rc=$? + POWERX_COLLECTOR_PID="" + return "$rc" +} diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index a1e357b30..6da11624d 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -106,6 +106,12 @@ For multinode fixed-sequence jobs, `utils/process_result.py --all` processes eve Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. +### Native multinode telemetry + +`native_power_collect.sh` and `native_power_lifecycle.sh` provide per-node collection and bounded ready/stop receipts. Launchers opt into the native package under `LOGS/native_power`; this prerequisite enables no new recipe. The adapter validates serving GPU identity, synchronized clocks, collector completion, and complete formal-window coverage. It preserves per-node failures, sample counts, and collector revision in the audit. + +SMI collection records UTC context beside each CSV for portable replay. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index 6a054d5f0..fd3fa2a2d 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -106,6 +106,12 @@ InferenceX-app 将路由字段作为列或配置维度，并把数值测量存 启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 +### 原生多节点遥测 + +`native_power_collect.sh` 和 `native_power_lifecycle.sh` 提供每节点采集及有时限的就绪/停止状态文件。启动器可使用 `LOGS/native_power` 下的原生产物；此前置改动不会启用新 recipe。适配器验证服务 GPU 身份、时钟同步、采集完成及正式窗口完整覆盖，并在审计中保留节点故障、样本数和采集器版本。 + +SMI 采集在 CSV 旁记录 UTC 上下文以支持跨环境回放。正式窗口外的无效样本不能构成覆盖；`boundary_degenerate_rows` 保留其逐 GPU 计数。 + ## 评测工件 ### 单配置身份和收集 diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index f6fa76102..5ae8a65ee 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -248,6 +248,21 @@ def aggregate_power_result( expected_num_gpus = int(env['TP']) * int(env.get('PP_SIZE', '1')) * int(env.get('PCP_SIZE', '1')) try: if is_multinode: + native_dir = Path(env.get('POWERX_NATIVE_DIR', 'LOGS/native_power')) + if env.get('POWERX_NATIVE_DIR') or native_dir.is_dir(): + if source.is_dir() and source != native_dir: + raise ValueError('Both native and SRT power packages are present') + source = native_dir + from .power.native_multinode import run + + return run( + native_dir, bench_path, agg_path, + expected_prefill_gpus=prefill_gpus, + expected_decode_gpus=decode_gpus, + expected_aggregate_gpus=aggregate_gpus, + validation_result=validation_path, + require_power=require_power, + ) from .power.multinode import run return run( diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py new file mode 100644 index 000000000..1ffae5bd9 --- /dev/null +++ b/infx/results/power/native_multinode.py @@ -0,0 +1,286 @@ +"""Validate native SMI traces from each node of one fixed-sequence deployment. + +This format is owned by InferenceX. It does not claim the srt-slurm/DCGM wire +contract. The launcher records real serving-device membership and synchronized +host clocks; the normal result processor binds every trace to the client window. +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import os +import socket +import time +from datetime import timezone +from pathlib import Path + +from . import ALL_POWER_METRIC_KEYS +from .common import ( + _load_benchmark_data, _write_json_atomic, benchmark_window_payload, + patch_power_metrics, +) +from .single_node import _derived_metrics, _detect_columns, _parse_timestamp, integrate_power + + +def _identity(path: Path, vendor: str) -> dict[str, str]: + """Map the vendor enumeration index to a physical UUID; never use PCI alone.""" + if vendor == "nvidia": + with path.open(newline="") as stream: + rows = [] + for row in csv.DictReader(stream, skipinitialspace=True): + if any(not isinstance(k, str) or not isinstance(v, str) for k, v in row.items()): + raise ValueError("invalid_device_identity") + rows.append({k.strip().lower(): v.strip() for k, v in row.items()}) + elif vendor == "amd": + payload = json.loads(path.read_text()) + rows = [] + + def visit(value: object) -> None: + if isinstance(value, list): + for item in value: + visit(item) + elif isinstance(value, dict): + row = {str(k).lower(): v for k, v in value.items()} + if "gpu" in row and "uuid" in row: + rows.append(row) + else: + for item in value.values(): + visit(item) + + visit(payload) + else: + raise ValueError("unsupported_native_vendor") + identities = {} + for row in rows: + index = str(row.get("index", row.get("gpu", ""))).strip() + uuid = str(row.get("uuid", "")).strip() + if not index.isdigit() or not uuid or uuid.lower() in {"n/a", "none", "null"}: + raise ValueError("invalid_device_identity") + if index in identities or uuid in identities.values(): + raise ValueError("duplicate_device_identity") + identities[index] = uuid + if not identities: + raise ValueError("device_identity_missing") + return identities + + +def record_begin(directory: Path, *, vendor: str, node: str, rank: int, + role: str, gpu_indices: list[int], num_nodes: int, job_id: str, + clock_synchronized: bool, revision: str) -> None: + if (role not in {"prefill", "decode", "aggregate"} or rank < 0 or + num_nodes <= rank or not gpu_indices or min(gpu_indices) < 0 or + len(set(gpu_indices)) != len(gpu_indices)): + raise ValueError("invalid_native_topology") + directory.mkdir(parents=True, exist_ok=True) + _write_json_atomic(directory / "manifest.json", { + "schema_version": 1, "collector": "inferencex-native-smi", + "vendor": vendor, "node": node, "rank": rank, "role": role, + "selected_gpu_indices": gpu_indices, "expected_num_nodes": num_nodes, + "job_id": job_id, "collector_revision": revision, + "clock_source": "utc_ntp", "clock_synchronized": clock_synchronized, + "clock_observation": "timedatectl NTPSynchronized on serving host; no measured clock offset", + "collection_start_unix": time.time(), "lifecycle": "collecting", + }) + + +def record_end(directory: Path, *, collector_exit_code: int) -> None: + manifest = json.loads((directory / "manifest.json").read_text()) + manifest.update(collection_end_unix=time.time(), + collector_exit_code=collector_exit_code, + lifecycle="complete" if collector_exit_code == 0 else "failed") + _write_json_atomic(directory / "manifest.json", manifest) + + +def run(power_dir: Path, bench_result: Path, agg_result: Path, *, + expected_prefill_gpus: int, expected_decode_gpus: int, expected_aggregate_gpus: int = 0, + validation_result: Path | None = None, require_power: bool = False) -> int: + validation_result = validation_result or bench_result.with_name( + f"power_validation_{bench_result.stem}.json") + benchmark, reasons = _load_benchmark_data(bench_result) + expected_gpus = expected_prefill_gpus + expected_decode_gpus + expected_aggregate_gpus + if expected_aggregate_gpus and (expected_prefill_gpus or expected_decode_gpus): + reasons.append("native_mixed_aggregate_role_topology") + roles: dict[str, list[str]] = {"prefill": [], "decode": [], "aggregate": []} + receipts = [] + node_errors = [] + samples = [] + ranks = [] + nodes = [] + jobs = set() + revisions = set() + expected_nodes = set() + paths = sorted(power_dir.glob("node-*/manifest.json")) + if not paths: + reasons.append("native_manifests_missing") + if expected_gpus <= 0 or min(expected_prefill_gpus, expected_decode_gpus, expected_aggregate_gpus) < 0: + reasons.append("invalid_expected_gpu_count") + for path in paths: + try: + manifest = json.loads(path.read_text()) + rank = manifest["rank"] + role = manifest["role"] + if (manifest.get("schema_version") != 1 or + manifest.get("collector") != "inferencex-native-smi" or + type(rank) is not int or rank < 0 or role not in roles or + path.parent.name != f"node-{rank}"): + raise ValueError("invalid_native_manifest") + if (not isinstance(manifest.get("node"), str) or not manifest["node"] or + not isinstance(manifest.get("job_id"), str) or + not isinstance(manifest.get("collector_revision"), str) or + type(manifest.get("expected_num_nodes")) is not int or + manifest["expected_num_nodes"] <= 0): + raise ValueError("invalid_native_run_identity") + ranks.append(rank) + nodes.append(manifest["node"]) + expected_nodes.add(manifest["expected_num_nodes"]) + jobs.add(manifest["job_id"]) + revisions.add(manifest["collector_revision"]) + if (manifest.get("lifecycle") != "complete" or + manifest.get("collector_exit_code") != 0): + reasons.append("native_collector_incomplete") + if (manifest.get("clock_source") != "utc_ntp" or + manifest.get("clock_synchronized") is not True): + reasons.append("native_clock_not_synchronized") + start, end = manifest["collection_start_unix"], manifest["collection_end_unix"] + if (not all(type(x) in (int, float) and math.isfinite(x) for x in (start, end)) + or end <= start or (benchmark is not None and + (start > benchmark.start_unix or end < benchmark.end_unix))): + reasons.append("native_collection_window_mismatch") + selected = manifest["selected_gpu_indices"] + if (not isinstance(selected, list) or not selected or + any(type(i) is not int or i < 0 for i in selected) or + len(set(selected)) != len(selected)): + raise ValueError("invalid_native_gpu_selection") + suffix = "csv" if manifest["vendor"] == "nvidia" else "json" + stem = "gpu_metrics_identity" if suffix == "csv" else "gpu_metrics_devices" + first_path = path.parent / f"{stem}.{suffix}" + last_path = path.parent / f"{stem}_end.{suffix}" + first = _identity(first_path, manifest["vendor"]) + last = _identity(last_path, manifest["vendor"]) + selected_ids = {str(i): first[str(i)] for i in selected} + if any(last.get(i) != uuid for i, uuid in selected_ids.items()): + reasons.append("native_device_identity_changed") + previous = {uuid for members in roles.values() for uuid in members} + if previous.intersection(selected_ids.values()): + reasons.append("native_duplicate_physical_gpu") + roles[role].extend(selected_ids.values()) + csv_path = path.parent / "gpu_metrics.csv" + with csv_path.open(newline="") as stream: + reader = csv.DictReader(stream, skipinitialspace=True) + reader.fieldnames = [c.strip() for c in (reader.fieldnames or [])] + t_col, p_col, g_col = _detect_columns(reader.fieldnames) + if not all((t_col, p_col, g_col)): + raise ValueError("native_telemetry_columns_missing") + for row in reader: + gpu = (row.get(g_col) or "").strip() + if gpu not in first: + reasons.append("native_unknown_device_index") + continue + if gpu not in selected_ids: + continue + timestamp = _parse_timestamp((row.get(t_col) or ""), naive_timezone=timezone.utc) + # Invalid rows are retained for the common validator to reject. + samples.append((timestamp, selected_ids[gpu], row.get(p_col) or "")) + receipts.append({**manifest, "physical_gpu_ids": selected_ids, + "manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "telemetry_sha256": hashlib.sha256(csv_path.read_bytes()).hexdigest(), + "identity_sha256": hashlib.sha256(first_path.read_bytes()).hexdigest(), + "identity_end_sha256": hashlib.sha256(last_path.read_bytes()).hexdigest()}) + except (OSError, ValueError, KeyError, TypeError, csv.Error) as exc: + reasons.append("native_node_invalid") + node_errors.append({"node": path.parent.name, "type": type(exc).__name__, "detail": str(exc)}) + if (len(expected_nodes) != 1 or not expected_nodes or + type(next(iter(expected_nodes), None)) is not int or + set(ranks) != set(range(next(iter(expected_nodes), 0))) or + len(set(nodes)) != len(nodes) or len(ranks) != len(set(ranks))): + reasons.append("native_node_topology_mismatch") + if len(jobs) != 1 or "" in jobs or len(revisions) != 1 or "" in revisions: + reasons.append("native_run_identity_mismatch") + if expected_aggregate_gpus: + if roles["prefill"] or roles["decode"] or len(roles["aggregate"]) != expected_aggregate_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif expected_decode_gpus: + if roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or len(roles["decode"]) != expected_decode_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif roles["decode"] or len(roles["aggregate"] + roles["prefill"]) != expected_prefill_gpus: + reasons.append("native_role_gpu_count_mismatch") + combined_path = validation_result.with_name(f"{validation_result.stem}_native.csv") + combined_path.parent.mkdir(parents=True, exist_ok=True) + with combined_path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["timestamp", "gpu", "power"]) + writer.writerows(samples) + integration = None + if benchmark is not None: + integration = integrate_power(combined_path, start_unix=benchmark.start_unix, + end_unix=benchmark.end_unix, expected_num_gpus=expected_gpus) + reasons.extend(integration.invalid_reasons) + metrics = {} + if not reasons and integration is not None and benchmark is not None: + metrics = _derived_metrics(integration, benchmark) + if expected_decode_gpus: + prefill = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["prefill"]) + decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) + metrics.update(prefill_gpu_energy_j=prefill, decode_gpu_energy_j=decode, + prefill_avg_power_w=prefill / benchmark.integration_duration_s / expected_prefill_gpus, + decode_avg_power_w=decode / benchmark.integration_duration_s / expected_decode_gpus, + prefill_joules_per_input_token=prefill / benchmark.total_input_tokens, + decode_joules_per_output_token=decode / benchmark.total_output_tokens) + valid = not reasons + try: + patch_power_metrics(agg_result, metric_keys=ALL_POWER_METRIC_KEYS, power_valid=valid, metrics=metrics) + except (OSError, ValueError): + reasons.append("aggregate_result_unwritable") + valid, metrics = False, {} + audit = {"schema_version": 1, "telemetry_kind": "native_multinode_smi", + "power_valid": valid, "reasons": list(dict.fromkeys(reasons)), + "benchmark_result": str(bench_result), + "benchmark_result_sha256": hashlib.sha256(bench_result.read_bytes()).hexdigest() if bench_result.is_file() else None, + "benchmark_window": benchmark_window_payload(benchmark), + "expected_gpu_count": expected_gpus, "nodes": receipts, + "node_errors": node_errors, + "observed_gpu_count": integration.observed_num_gpus if integration else 0, + "per_gpu_role": {uuid: role for role, uuids in roles.items() for uuid in uuids}, + "per_gpu_sample_counts": integration.per_gpu_sample_counts if integration else {}, + "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s if integration else {}, + "producer": {"name": "inferencex-native-smi", "revisions": sorted(revisions), + "producer_git_commit": next(iter(revisions)) if len(revisions) == 1 else None}, + "integration_method": "per_device_trapezoidal_with_linear_boundary_interpolation", + "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear", + "metrics": metrics} + _write_json_atomic(validation_result, audit) + return int(require_power and not valid) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + begin = sub.add_parser("begin") + begin.add_argument("--directory", type=Path, required=True) + begin.add_argument("--vendor", choices=("amd", "nvidia"), required=True) + begin.add_argument("--node", default=os.environ.get("POWERX_NODE_NAME", socket.gethostname())) + begin.add_argument("--rank", type=int, required=True) + begin.add_argument("--role", choices=("prefill", "decode", "aggregate"), required=True) + begin.add_argument("--gpu-indices", required=True) + begin.add_argument("--num-nodes", type=int, required=True) + begin.add_argument("--job-id", default=os.environ.get("SLURM_JOB_ID", "")) + begin.add_argument("--revision", default=os.environ.get("POWERX_COLLECTOR_REVISION", "")) + begin.add_argument("--clock-synchronized", choices=("true", "false"), default="false") + end = sub.add_parser("end") + end.add_argument("--directory", type=Path, required=True) + end.add_argument("--collector-exit-code", type=int, required=True) + args = vars(parser.parse_args()) + action = args.pop("action") + if action == "begin": + args["gpu_indices"] = [int(i) for i in args["gpu_indices"].split(",")] + args["clock_synchronized"] = args["clock_synchronized"] == "true" + record_begin(**args) + else: + record_end(**args) + + +if __name__ == "__main__": + main() diff --git a/infx/results/power/single_node.py b/infx/results/power/single_node.py index 54669a1a8..4ce1bcaaf 100644 --- a/infx/results/power/single_node.py +++ b/infx/results/power/single_node.py @@ -9,7 +9,11 @@ aggregate and a validation sidecar, but does not fail the benchmark. Power studies can set ``REQUIRE_POWER=1`` to fail after those audit artifacts exist. The aggregate carries numeric ``power_valid`` (1/0) for metric ingestion; the -sidecar is the canonical source for boolean validity and reason codes. +sidecar is the canonical source for boolean validity and reason codes. Rows in +the ingest band but outside the formal window whose power is missing, +non-finite, or <= 0 are teardown noise: they are skipped and counted in the +sidecar's ``boundary_degenerate_rows`` instead of poisoning validity or faking +window bracketing. """ from __future__ import annotations @@ -22,7 +26,7 @@ import os import re import sys -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from statistics import mean @@ -65,6 +69,10 @@ class PowerIntegration: per_gpu_max_sample_gap_s: dict[str, float] per_gpu_energy_j: dict[str, float] device_issues: dict[str, list[str]] + # Rows in the ingest band but outside the formal window whose power was + # missing/N-A/non-finite/<=0, skipped and counted per GPU; "unknown" + # buckets rows without a GPU identity. + boundary_degenerate_rows: dict[str, int] = field(default_factory=dict) avg_power_w: float | None = None p75_power_w: float | None = None p75_total_gpu_power_w: float | None = None @@ -78,7 +86,7 @@ def observed_num_gpus(self) -> int: return len(self.observed_gpu_ids) -def _parse_timestamp(value: str) -> float | None: +def _parse_timestamp(value: str, *, naive_timezone: timezone | None = None) -> float | None: """Best-effort timestamp parse to Unix epoch seconds (local wall clock). Handles the formats observed in practice: @@ -96,7 +104,7 @@ def _parse_timestamp(value: str) -> float | None: # nvidia-smi: "YYYY/MM/DD HH:MM:SS.ffffff" for fmt in ("%Y/%m/%d %H:%M:%S.%f", "%Y/%m/%d %H:%M:%S"): try: - return datetime.strptime(value, fmt).timestamp() + return datetime.strptime(value, fmt).replace(tzinfo=naive_timezone).timestamp() except ValueError: pass # ISO 8601 (amd-smi variants). fromisoformat tolerates 'T' or space separator @@ -108,7 +116,7 @@ def _parse_timestamp(value: str) -> float | None: return None if dt.tzinfo is None: # Treat naive timestamps as local time (matches nvidia-smi convention). - return dt.timestamp() + return dt.replace(tzinfo=naive_timezone).timestamp() return dt.astimezone(timezone.utc).timestamp() @@ -146,6 +154,16 @@ def _detect_columns(header: list[str]) -> tuple[str | None, str | None, str | No return timestamp_col, power_col, gpu_col +def _telemetry_timezone(csv_path: Path) -> timezone | None: + context = csv_path.with_name(f"{csv_path.stem}_context.json") + if not context.is_file(): + return None + payload = json.loads(context.read_text()) + if not isinstance(payload, dict) or payload.get("timestamp_timezone") != "UTC": + raise ValueError("unsupported_telemetry_timezone") + return timezone.utc + + def aggregate_power( csv_path: Path, start_unix: float, @@ -162,6 +180,7 @@ def aggregate_power( return None try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [c.strip() for c in (reader.fieldnames or [])] @@ -189,7 +208,7 @@ def aggregate_power( for row in reader: ts_raw = (row.get(timestamp_col) or "").strip() pw_raw = (row.get(power_col) or "").strip() - ts = _parse_timestamp(ts_raw) + ts = _parse_timestamp(ts_raw, naive_timezone=timestamp_timezone) pw = _parse_power(pw_raw) if ts is None or pw is None: continue @@ -204,7 +223,7 @@ def aggregate_power( if gpu_id: per_sample_gpus.setdefault(bucket, set()).add(gpu_id) gpu_keys.add(gpu_id) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): return None if not per_sample_total: @@ -237,6 +256,7 @@ def _empty_integration( *, expected_num_gpus: int | None, reasons: list[str], + boundary_degenerate_rows: dict[str, int] | None = None, ) -> PowerIntegration: """Build an invalid integration result when no device data is available.""" return PowerIntegration( @@ -248,6 +268,7 @@ def _empty_integration( per_gpu_max_sample_gap_s={}, per_gpu_energy_j={}, device_issues={}, + boundary_degenerate_rows=boundary_degenerate_rows or {}, ) @@ -308,8 +329,10 @@ def integrate_power( # expose timestamps at lower resolution than their sampling cadence, so # duplicate-timestamp readings are averaged rather than treated as corrupt. raw_samples: dict[str, dict[float, list[float]]] = {} + boundary_degenerate: dict[str, int] = {} saw_missing_gpu_identity = False try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -332,7 +355,7 @@ def integrate_power( ) for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) if timestamp is None or not math.isfinite(timestamp): _append_reason(reasons, "invalid_timestamp_sample") continue @@ -348,6 +371,15 @@ def integrate_power( power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() + if (power is None or not math.isfinite(power) or power <= 0.0) and ( + timestamp < start_unix or timestamp > end_unix + ): + # SMI teardown rows can carry N/A or 0 W cells: outside the + # formal window they are counted, never used to satisfy + # bracketing or to poison in-window validity. + key = gpu_id or "unknown" + boundary_degenerate[key] = boundary_degenerate.get(key, 0) + 1 + continue if power is None: _append_reason(reasons, "invalid_power_sample") continue @@ -359,11 +391,12 @@ def integrate_power( continue values = raw_samples.setdefault(gpu_id, {}).setdefault(timestamp, []) values.append(power) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): _append_reason(reasons, "telemetry_file_unreadable") return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) if saw_missing_gpu_identity: @@ -373,6 +406,7 @@ def integrate_power( return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) observed_gpu_ids = tuple(sorted(raw_samples, key=_gpu_sort_key)) @@ -450,6 +484,7 @@ def integrate_power( per_gpu_max_sample_gap_s=per_gpu_max_sample_gap_s, per_gpu_energy_j=per_gpu_energy_j, device_issues=device_issues, + boundary_degenerate_rows=boundary_degenerate, avg_power_w=avg_power_w, p75_power_w=p75_total / len(observed_gpu_ids) if p75_total is not None else None, p75_total_gpu_power_w=p75_total, @@ -485,6 +520,7 @@ def _read_energy_snapshot(path: Path) -> dict[str, float] | None: def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float]]]: """Group every parseable (timestamp, watt) sample per GPU, sorted in time.""" samples: dict[str, list[tuple[float, float]]] = {} + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -493,7 +529,7 @@ def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float] if not timestamp_col or not power_col or not gpu_col: return {} for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() if timestamp is None or power is None or not gpu_id: @@ -679,6 +715,7 @@ def _validation_payload( "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s, "per_gpu_energy_j": integration.per_gpu_energy_j, "device_issues": integration.device_issues, + "boundary_degenerate_rows": integration.boundary_degenerate_rows, "accumulator_check": accumulator_check, "metrics": audit_metrics(metrics), } @@ -754,7 +791,7 @@ def run( try: accumulator_check = cross_check_accumulator(csv_path) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): accumulator_check = {"available": False, "reason": "cross_check_error"} try: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f3b85de24..10db254b5 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7410,3 +7410,14 @@ - "Preserve request failure outcomes, retain all available fixed-sequence diagnostics, and reject incomplete concurrency batches before publishing benchmark artifacts." - "保留请求失败状态及全部可用的固定序列诊断信息；发布基准产物前拒绝缺失并发点的批次。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Retain UTC SMI telemetry context and validate native multinode collector artifacts without enabling new recipes." + - "保留 UTC SMI 遥测上下文并验证原生多节点采集产物，不启用新 recipe。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 diff --git a/runners/test_native_collector_barriers.py b/runners/test_native_collector_barriers.py new file mode 100644 index 000000000..986016a78 --- /dev/null +++ b/runners/test_native_collector_barriers.py @@ -0,0 +1,46 @@ +"""CPU-only lifecycle checks with controlled external collector/Slurm processes.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +LIFECYCLE = ROOT / 'benchmarks/native_power_lifecycle.sh' +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' + + +@pytest.mark.parametrize('failed_rank', [None, 1]) +def test_stop_waits_for_every_collector_and_retains_failure(tmp_path, failed_rank): + command = ''' +source "$1" +export POWERX_CONTROL_DIR="$2" POWERX_NUM_NODES=2 POWERX_BARRIER_TIMEOUT_S=5 +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.1; echo 0 > "$2/done-0") & +POWERX_COLLECTOR_PID=$! +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.2; echo "$3" > "$2/done-1") & +remote_pid=$! +powerx_stop_collectors +rc=$? +wait "$remote_pid" +exit "$rc" +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(LIFECYCLE), + str(tmp_path), '1' if failed_rank else '0'], + capture_output=True, text=True, timeout=10) + assert result.returncode == (1 if failed_rank else 0), result.stderr + assert (tmp_path / 'done-0').read_text().strip() == '0' + assert (tmp_path / 'done-1').read_text().strip() == ('1' if failed_rank else '0') + + +def test_ready_barrier_rejects_collector_that_already_stopped(tmp_path): + (tmp_path / 'ready-0').write_text('ready') + (tmp_path / 'done-0').write_text('1') + result = subprocess.run(['bash', '-c', 'source "$1"; POWERX_CONTROL_DIR="$2"; ' + 'POWERX_NUM_NODES=1; powerx_wait_collectors ready', + 'bash', str(LIFECYCLE), str(tmp_path)], capture_output=True, text=True) + assert result.returncode == 1 + assert 'stopped before benchmark readiness' in result.stderr + + diff --git a/runners/test_native_collector_receipts.py b/runners/test_native_collector_receipts.py new file mode 100644 index 000000000..b981f57ce --- /dev/null +++ b/runners/test_native_collector_receipts.py @@ -0,0 +1,16 @@ +import subprocess +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] + +def test_native_control_receipt_is_owned_before_publication(tmp_path): + source = (ROOT / 'benchmarks/native_power_collect.sh').read_text() + function = source[source.index('write_control() {'):source.index('\nfinish() {')] + result = subprocess.run(['bash', '-c', function + ''' +control_dir=$1 +POWERX_HOST_UID=1000 POWERX_HOST_GID=1000 +chown() { [[ ! -e "$control_dir/done-0" ]]; } +write_control done-0 7 +''', 'bash', str(tmp_path)], capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert (tmp_path / 'done-0').read_text() == '7\n' + assert not list(tmp_path.glob('*.tmp')) diff --git a/utils/test_aggregate_power.py b/utils/test_aggregate_power.py index aa3bf23f9..5b8b76121 100644 --- a/utils/test_aggregate_power.py +++ b/utils/test_aggregate_power.py @@ -574,6 +574,201 @@ def test_run_rejects_malformed_telemetry_inside_window( assert audit["reasons"] == [expected_reason] + +# AMDSMI 26.2.0 `metric -p -c -t -u -w 1 --csv` header (order-faithful subset, +# measured on MI355X; see test_detect_columns_amd_watch_mode_real_header). +_MI355X_WATCH_HEADER = ( + "timestamp,gpu,gfx_activity,umc_activity,mm_activity,vcn_activity," + "jpeg_activity,gfx_busy_inst_xcp_0,jpeg_busy_xcp_0,vcn_busy_xcp_0," + "socket_power,gfx_voltage,soc_voltage,mem_voltage,throttle_status," + "power_management,gfx_0_clk,mem_0_clk,edge,hotspot,mem" +) + + +def _mi355x_watch_row(timestamp: int, gpu: int, socket_power: str) -> str: + """One data row in the shape captured from run 32433563482 (conc1): + integer-second epoch, quoted list cells with embedded commas, N/A cells, + and a trailing carriage return.""" + return ( + f"{timestamp},{gpu},0,0,N/A,\"['N/A', 'N/A', 'N/A', 'N/A']\"," + "\"['N/A', 'N/A']\",\"[0, 0, 0, 0, 0, 0, 0, 0]\",\"[0, 0]\",\"[0, 0]\"," + f"{socket_power},N/A,N/A,N/A,N/A,ENABLED,1404,2000,N/A,40,25\r" + ) + + +def test_integrate_power_skips_na_power_rows_outside_window(tmp_path: Path): + """N/A-power teardown rows past the window end are counted, not poisonous.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-2, 13): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},N/A,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + assert result.total_gpu_energy_j == pytest.approx(10_000.0) + + +def test_integrate_power_does_not_bracket_with_zero_power_tail(tmp_path: Path): + """A 0 W teardown row past the window end must not fake end bracketing. + + Legacy behavior accepted the 0 W row as a valid boundary sample, corrupting + the end interpolation with a bogus value; this intentionally flips that + case to an explicit benchmark_window_not_bracketed failure.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + # Good rows stop at end-4; the only post-end sample per GPU has power 0. + for offset in range(-1, 7): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},0,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "benchmark_window_not_bracketed" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + + +def test_integrate_power_preserves_boundary_counts_without_usable_samples(tmp_path: Path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text( + "timestamp,gpu,socket_power\n" + "1699999999,0,N/A\n" + "1700000011,0,0\n" + "1700000011,1,N/A\n", + encoding="utf-8", + ) + + result = integrate_power( + csv, + start_unix=1_700_000_000, + end_unix=1_700_000_010, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "no_usable_power_samples" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 2, "1": 1} + + +def test_integrate_power_keeps_zero_power_semantics_inside_window(tmp_path: Path): + """Frozen legacy behavior: an in-window 0 W sample still integrates.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-1, 12): + watts = "0.0" if offset == 5 else "500.0" + lines.append(f"{base + offset},0,{watts},65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=1, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {} + # Trapezoids dip to 0 at t=5: 8 x 500 + 2 x 250 = 4500 J. + assert result.total_gpu_energy_j == pytest.approx(4_500.0) + + +def test_run_writes_boundary_degenerate_rows_to_sidecar(tmp_path: Path): + base = 1_700_000_000.0 + csv = tmp_path / "gpu_metrics.csv" + _write_constant_window_samples( + csv, + start=base, + end=base + 10, + watts_per_gpu=500.0, + num_gpus=2, + ) + bench = tmp_path / "bench.json" + agg = tmp_path / "agg.json" + validation = tmp_path / "power_validation.json" + _write_bench_result( + bench, + start=base, + end=base + 10, + duration=10.0, + total_output=2_000, + total_input=10_000, + ) + agg.write_text(json.dumps({"hw": "mi355x"}), encoding="utf-8") + + exit_code = run(csv, bench, agg, expected_num_gpus=2, validation_result=validation) + + assert exit_code == 0 + audit = json.loads(validation.read_text()) + # Present-and-empty for clean streams: readers can rely on the key. + assert audit["boundary_degenerate_rows"] == {} + + +def test_integrate_power_regression_mi355x_integer_ticks_end_gap(tmp_path: Path): + """Run-32433563482 conc1 regression: amd-smi integer-second ticks stop 4 s + before the fractional aiperf window end (last tick 1787277605 vs end + ...609.157497), so bracketing fails and the producer-side telemetry loss is + attributed as benchmark_window_not_bracketed on every GPU. + + The retrieved artifact's trailing rows all carry valid socket_power + (254-264 W) with N/A activity/voltage cells; the N/A- and 0-power teardown + rows appended past the window end are the documented synthetic degenerate + shapes, asserting they are counted rather than used for bracketing.""" + csv = tmp_path / "gpu_metrics.csv" + start = 1_787_277_560.155891 + end = 1_787_277_609.157497 + last_tick = 1_787_277_605 + powers = [259, 255, 263, 264, 256, 254, 259, 259] + lines = [_MI355X_WATCH_HEADER] + for tick in range(1_787_277_555, last_tick + 1): + for gpu in range(8): + lines.append(_mi355x_watch_row(tick, gpu, str(powers[gpu]))) + # amd-smi watch mode emits a blank line between tick groups. + lines.append("") + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_610, gpu, "N/A")) + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_611, gpu, "0")) + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=start, + end_unix=end, + expected_num_gpus=8, + ) + + assert result.power_valid is False + assert result.invalid_reasons == ("benchmark_window_not_bracketed",) + assert result.device_issues == { + str(gpu): ["benchmark_window_not_bracketed"] for gpu in range(8) + } + assert result.boundary_degenerate_rows == {str(gpu): 2 for gpu in range(8)} + + def test_run_patches_agg_with_power_and_joules(tmp_path: Path): base = 1_700_000_000.0 csv = tmp_path / "gpu_metrics.csv" @@ -708,6 +903,7 @@ def test_power_replacement_removes_stale_metrics(tmp_path, patch_validated_power path.write_text(json.dumps({ "hw": "fixture", "avg_power_w": 99, "total_gpu_energy_j": 50, "power_invalid_reasons": ["stale"], + "power_audit": {"source": "previous-run.json"}, })) patch_validated_power(path, power_valid=valid, metrics={ "avg_power_w": 12.34567, "joules_per_output_token": 0.12345678, @@ -1433,12 +1629,12 @@ def test_power_percentiles_uses_synchronized_total_not_device_percentiles(tmp_pa def test_power_percentiles_weights_time_and_clips_the_validated_window(tmp_path): csv_path = tmp_path / "power.csv" - # Dense readings near the high end must not bias a uniform linear ramp. - _write_amd_csv(csv_path, [(0, 0, 0), (1, 0, 100), (1.9, 0, 190), (2, 0, 200)]) + # Dense readings must not bias the uniform 150-250 W ramp inside the window. + _write_amd_csv(csv_path, [(0, 0, 100), (1, 0, 200), (1.9, 0, 290), (2, 0, 300)]) result = integrate_power(csv_path, start_unix=0.5, end_unix=1.5, expected_num_gpus=1) assert result.power_valid - assert result.p75_power_w == pytest.approx(125) - assert result.p90_power_w == pytest.approx(140) + assert result.p75_power_w == pytest.approx(225) + assert result.p90_power_w == pytest.approx(240) def test_power_percentiles_is_withheld_for_invalid_telemetry(tmp_path): @@ -1462,3 +1658,54 @@ def test_power_percentiles_aligns_asynchronous_gpu_samples(tmp_path): assert result.p75_power_w == pytest.approx(300) assert result.p90_total_gpu_power_w == pytest.approx(600) assert result.p90_power_w == pytest.approx(300) + + +@pytest.mark.parametrize('step_name', ['Upload GPU metrics', 'Upload power audit bundle']) +@pytest.mark.parametrize('directory', ['', 'results']) +def test_uploaded_telemetry_replays_in_a_different_timezone(tmp_path, step_name, directory): + import os + import shutil + import yaml + + repo = Path(__file__).resolve().parents[1] + source = tmp_path / 'source' + telemetry = source / directory + telemetry.mkdir(parents=True) + (telemetry / 'gpu_metrics.csv').write_text( + 'timestamp,index,power.draw [W]\n' + '2024/01/01 00:00:00.000,0,100 W\n' + '2024/01/01 00:00:01.000,0,100 W\n' + '2024/01/01 00:00:02.000,0,100 W\n') + (telemetry / 'gpu_metrics_context.json').write_text('{"timestamp_timezone":"UTC"}') + workflow = yaml.safe_load((repo / '.github/workflows/benchmark-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job['steps'] + if step.get('name') == step_name) + archive = tmp_path / 'downloaded' + for pattern in step['with']['path'].splitlines(): + for file in source.glob(pattern): + destination = archive / file.relative_to(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(file, destination) + result = subprocess.run( + [sys.executable, '-c', """ +import sys, time +from pathlib import Path +from infx.results.power.single_node import integrate_power +time.tzset() +result = integrate_power(Path(sys.argv[1]), start_unix=1704067200, end_unix=1704067202, + expected_num_gpus=1) +assert result.power_valid, result.invalid_reasons +assert result.total_gpu_energy_j == 200 +""", str(archive / directory / 'gpu_metrics.csv')], + cwd=tmp_path, env={**os.environ, 'TZ': 'Etc/GMT+8', 'PYTHONPATH': str(repo)}, + capture_output=True, text=True, timeout=10, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize('context', ['{"timestamp_timezone":"PST"}', '{}', '{invalid']) +def test_legacy_average_rejects_invalid_telemetry_context(tmp_path, context): + csv = tmp_path / 'gpu_metrics.csv' + csv.write_text('timestamp,index,power.draw [W]\n1,0,100\n2,0,100\n') + csv.with_name('gpu_metrics_context.json').write_text(context) + assert aggregate_power(csv, 1, 2) is None diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py new file mode 100644 index 000000000..0aeee2511 --- /dev/null +++ b/utils/test_native_multinode_power.py @@ -0,0 +1,201 @@ +"""Native collector acceptance uses independent, hand-computed two-node traces.""" +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from infx.results.power.native_multinode import record_begin, record_end, run +from infx.results.power.single_node import integrate_power + +REPO = Path(__file__).resolve().parents[1] + + +def _package(tmp_path, vendor="amd"): + root = tmp_path / "native_power" + for rank, role, watts in ((0, "prefill", 100), (1, "decode", 300)): + node = root / f"node-{rank}" + record_begin(node, vendor=vendor, node=f"host-{rank}", rank=rank, role=role, + gpu_indices=[0], num_nodes=2, job_id="job-123", revision="revision-abc", + clock_synchronized=True) + record_end(node, collector_exit_code=0) + manifest = json.loads((node / "manifest.json").read_text()) + manifest.update(collection_start_unix=0, collection_end_unix=5) + (node / "manifest.json").write_text(json.dumps(manifest)) + for ending in ("", "_end"): + if vendor == "amd": + (node / f"gpu_metrics_devices{ending}.json").write_text(json.dumps([ + {"gpu": 0, "uuid": f"uuid-{rank}"}, {"gpu": 1, "uuid": f"unused-{rank}"}])) + else: + (node / f"gpu_metrics_identity{ending}.csv").write_text( + f"index, uuid, pci.bus_id\n0, uuid-{rank}, 0000:01:00.0\n1, unused-{rank}, 0000:02:00.0\n") + # The spare physical GPU is visible but does not belong to the server. + (node / "gpu_metrics.csv").write_text("timestamp,gpu,power\n" + "".join( + f"{tick},0,{watts}\n{tick},1,900\n" for tick in range(5))) + bench = tmp_path / "result.json" + bench.write_text(json.dumps({"benchmark_start_time_unix": 1, "benchmark_end_time_unix": 3, + "duration": 2, "completed": 2, "total_input_tokens": 20, + "total_output_tokens": 10})) + agg = tmp_path / "agg.json" + agg.write_text(json.dumps({"avg_power_w": 999, "prefill_gpu_energy_j": 999})) + return root, bench, agg + + +@pytest.mark.parametrize("vendor", ["amd", "nvidia"]) +def test_native_whole_fleet_and_role_energy_use_only_serving_devices(tmp_path, vendor): + root, bench, agg = _package(tmp_path, vendor) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 1 + assert actual["total_gpu_energy_j"] == 800 + assert actual["avg_power_w"] == 200 + assert actual["p90_power_w"] == 200 + assert actual["p75_power_w"] == 200 + assert actual["joules_per_successful_query"] == 400 + assert actual["prefill_gpu_energy_j"] == 200 + assert actual["decode_gpu_energy_j"] == 600 + assert actual["prefill_joules_per_input_token"] == 10 + assert actual["decode_joules_per_output_token"] == 60 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert audit["observed_gpu_count"] == 2 + assert audit["per_gpu_role"] == {"uuid-0": "prefill", "uuid-1": "decode"} + assert len(audit["nodes"][0]["telemetry_sha256"]) == 64 + from infx.results.power.audit import audit_summary + summary = audit_summary(audit, "power_validation_result.json")["power_audit"] + assert summary["sample_count"] == 10 # Five samples on each of two participating GPUs. + assert summary["producer_sha"] == "revision-abc" + + +def test_native_parse_failure_has_a_public_reason_and_retained_detail(tmp_path): + from infx.results.power.audit import audit_summary + root, bench, agg = _package(tmp_path) + (root / "node-1/manifest.json").write_text("not JSON") + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + summary = audit_summary(audit, "power_validation_result.json") + assert "native_node_invalid" in summary["power_invalid_reasons"] + assert audit["node_errors"][0]["node"] == "node-1" + assert audit["node_errors"][0]["detail"] + + +@pytest.mark.parametrize(("field", "value", "reason"), [ + ("clock_synchronized", False, "native_clock_not_synchronized"), + ("lifecycle", "collecting", "native_collector_incomplete"), + ("collection_end_unix", 2, "native_collection_window_mismatch"), + ("job_id", "other-job", "native_run_identity_mismatch"), + ("role", "prefill", "native_role_gpu_count_mismatch"), + ("expected_num_nodes", 3, "native_node_topology_mismatch"), +]) +def test_native_invalid_evidence_clears_stale_metrics_and_writes_audit(tmp_path, field, value, reason): + root, bench, agg = _package(tmp_path) + path = root / "node-1/manifest.json" + manifest = json.loads(path.read_text()); manifest[field] = value + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 0 + assert "avg_power_w" not in actual + assert "prefill_gpu_energy_j" not in actual + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert reason in audit["reasons"] + + +def test_native_device_replacement_cannot_preserve_validity(tmp_path): + root, bench, agg = _package(tmp_path) + (root / "node-1/gpu_metrics_devices_end.json").write_text('[{"gpu":0,"uuid":"replacement"}]') + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + assert "native_device_identity_changed" in json.loads((tmp_path / "power_validation_result.json").read_text())["reasons"] + + +def test_native_aggregate_nodes_do_not_invent_prefill_decode_metrics(tmp_path): + root, bench, agg = _package(tmp_path) + for path in root.glob("*/manifest.json"): + manifest = json.loads(path.read_text()); manifest["role"] = "aggregate" + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=0, expected_decode_gpus=0, + expected_aggregate_gpus=2, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["avg_power_w"] == 200 + assert "prefill_gpu_energy_j" not in actual + + +def test_utc_context_replays_in_a_different_timezone(tmp_path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text("timestamp,index,power.draw [W]\n2026/01/01 00:00:00,0,100\n2026/01/01 00:00:02,0,100\n") + (tmp_path / "gpu_metrics_context.json").write_text('{"timestamp_timezone":"UTC"}') + script = "from pathlib import Path; from infx.results.power.single_node import integrate_power; " + \ + f"r=integrate_power(Path({str(csv)!r}),start_unix=1767225600,end_unix=1767225602,expected_num_gpus=1); assert r.power_valid; assert r.total_gpu_energy_j == 200" + subprocess.run([sys.executable, "-c", script], cwd=REPO, + env={**os.environ, "TZ": "America/Los_Angeles"}, check=True, timeout=10) + + +@pytest.mark.parametrize("clock_value, synchronized", [ + ("yes", True), ("true", True), ("no", False), ("false", False), ("", False), +]) +def test_native_supervisor_reaps_monitor_and_writes_completion(tmp_path, clock_value, synchronized): + binary = tmp_path / "bin"; binary.mkdir() + (binary / "python3").symlink_to(sys.executable) + fake = binary / "nvidia-smi" + fake.write_text(f'''#!{sys.executable} +import datetime, os, sys, time +if any("index,uuid" in arg for arg in sys.argv): + print("index, uuid, pci.bus_id"); print("0, gpu-0, 0000:01:00.0") +else: + with open({str(tmp_path / 'monitor.pid')!r}, "w") as stream: stream.write(str(os.getpid())) + if "-l" in sys.argv: print("timestamp,index,power.draw [W]", flush=True) + while True: + print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y/%m/%d %H:%M:%S.%f") + ",0,100", flush=True) + if "-l" not in sys.argv: break + time.sleep(0.1) +''') + fake.chmod(0o755) + control = tmp_path / "control"; control.mkdir() + node = tmp_path / "node-0" + process = subprocess.Popen(["bash", str(REPO / "benchmarks/native_power_collect.sh"), + str(node), str(control), "nvidia", "0", "aggregate", "0", "1"], + env={**os.environ, "PATH": f"{binary}:{os.environ['PATH']}", + "SLURM_JOB_ID": "test-job", "POWERX_COLLECTOR_REVISION": "revision", + "POWERX_CLOCK_SYNCHRONIZED": clock_value}, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + try: + deadline = time.monotonic() + 10 + while not (control / "ready-0").exists(): + assert process.poll() is None + assert time.monotonic() < deadline + time.sleep(0.02) + (control / "stop").write_text("stop") + stdout, stderr = process.communicate(timeout=10) + assert process.returncode == 0, (stdout, stderr) + assert (control / "done-0").read_text().strip() == "0" + manifest = json.loads((node / "manifest.json").read_text()) + assert manifest["lifecycle"] == "complete" + assert manifest["clock_synchronized"] is synchronized + assert (node / "gpu_metrics_identity_end.csv").exists() + finally: + if process.poll() is None: + process.terminate(); process.communicate(timeout=10) + + +def test_result_processor_discovers_staged_native_package(tmp_path, monkeypatch): + from infx.results.fixed_sequence import aggregate_power_result + + root, bench, agg = _package(tmp_path) + logs = tmp_path / "LOGS" + logs.mkdir() + root.rename(logs / "native_power") + monkeypatch.chdir(tmp_path) + env = {"IS_MULTINODE": "true", "PREFILL_GPUS": "1", "DECODE_GPUS": "1", + "RESULT_FILENAME": "result", "REQUIRE_POWER": "1"} + assert aggregate_power_result(env, bench, agg) == 0 + assert json.loads(agg.read_text())["total_gpu_energy_j"] == 800 + assert (tmp_path / "power_validation_result.json").is_file() + + # Two competing formats must not silently select one package. + (logs / "power").mkdir() + assert aggregate_power_result(env, bench, agg) == 1 + invalid = json.loads(agg.read_text()) + assert invalid["power_valid"] == 0 + assert "total_gpu_energy_j" not in invalid From ba0275c4769dc42e606d6fec2e89a3c910b9eb5a Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:24:13 -0700 Subject: [PATCH 04/17] feat: separate native multinode power collection contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 拆分原生多节点功耗采集契约、UTC 上下文和生命周期屏障，不启用新路径。 --- .../workflows/benchmark-multinode-tmpl.yml | 1 + .github/workflows/benchmark-tmpl.yml | 4 + benchmarks/benchmark_lib.sh | 7 +- benchmarks/native_power_collect.sh | 73 +++++ benchmarks/native_power_lifecycle.sh | 77 +++++ docs/results-and-ingestion.md | 6 + docs/results-and-ingestion_zh.md | 6 + infx/results/fixed_sequence.py | 15 + infx/results/power/native_multinode.py | 286 ++++++++++++++++++ infx/results/power/single_node.py | 59 +++- perf-changelog.yaml | 11 + runners/test_native_collector_barriers.py | 58 ++++ runners/test_native_collector_receipts.py | 16 + utils/test_aggregate_power.py | 255 +++++++++++++++- utils/test_native_multinode_power.py | 201 ++++++++++++ 15 files changed, 1057 insertions(+), 18 deletions(-) create mode 100644 benchmarks/native_power_collect.sh create mode 100644 benchmarks/native_power_lifecycle.sh create mode 100644 infx/results/power/native_multinode.py create mode 100644 runners/test_native_collector_barriers.py create mode 100644 runners/test_native_collector_receipts.py create mode 100644 utils/test_native_multinode_power.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 80b0842f8..e9f3a3c90 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -499,6 +499,7 @@ jobs: agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json LOGS/power/** + LOGS/native_power/** result_processing_${{ env.RESULT_FILENAME }}.json LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 4d1b817ea..1da933f63 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -445,11 +445,13 @@ jobs: name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }} path: | gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json if-no-files-found: ignore @@ -462,12 +464,14 @@ jobs: ${{ env.RESULT_FILENAME }}.json agg_${{ env.RESULT_FILENAME }}.json gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json results/agentic_power_window.json results/agentic_power_timezone_offset.txt diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 305a118ef..9d99a4cc9 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -177,6 +177,7 @@ start_gpu_monitor() { GPU_METRICS_CSV="$output" GPU_MONITOR_INTERVAL="$interval" export GPU_METRICS_CSV + printf '{"timestamp_timezone":"UTC"}\n' > "${output%.csv}_context.json" if command -v nvidia-smi &>/dev/null; then GPU_MONITOR_VENDOR="nvidia" @@ -185,7 +186,7 @@ start_gpu_monitor() { rm -f "${output%.csv}_identity.csv" echo "[GPU Monitor] Warning: NVIDIA identity sidecar failed" >&2 fi - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv -l "$interval" > "$output" 2>/dev/null & GPU_MONITOR_PID=$! echo "[GPU Monitor] Started NVIDIA (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)" @@ -196,7 +197,7 @@ start_gpu_monitor() { # Python; measured on MI355X: trailing ticks were lost at kill without it). # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated # headers, and flush every row so killing the pipe cannot discard buffered samples. - PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ + TZ=UTC PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ | awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" & GPU_MONITOR_PID=$! # Hardware energy-accumulator + identity snapshots; the end-side twin in @@ -232,7 +233,7 @@ stop_gpu_monitor() { case "$GPU_MONITOR_VENDOR" in nvidia) if _repair_truncated_gpu_metrics_tail; then - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv,noheader >> "$GPU_METRICS_CSV" 2>/dev/null || echo "[GPU Monitor] Warning: final NVIDIA sample failed" >&2 fi diff --git a/benchmarks/native_power_collect.sh b/benchmarks/native_power_collect.sh new file mode 100644 index 000000000..69df07951 --- /dev/null +++ b/benchmarks/native_power_collect.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# One native SMI collector per serving node. Raw files stay on node-local scratch; +# the launcher stages them as its host user after containers stop. +set -uo pipefail +power_dir=$1 +control_dir=$2 +vendor=$3 +rank=$4 +role=$5 +gpu_indices=$6 +num_nodes=$7 +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" +source "$repo_root/benchmarks/benchmark_lib.sh" +mkdir -p "$power_dir" +collector_rc=0 +finished=0 + +write_control() { + local path="$control_dir/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + # The control directory is created by the host user before Docker starts. + # Containers must not strand root-owned files in the shared runner tree. + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +finish() { + local incoming_rc=$? + [[ "$finished" == 0 ]] || return + if [[ "$incoming_rc" != 0 ]]; then collector_rc=$incoming_rc; fi + if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi + stop_gpu_monitor + if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json + else + nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \ + > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1 + fi + python3 -m infx.results.power.native_multinode end --directory "$power_dir" \ + --collector-exit-code "$collector_rc" || collector_rc=1 + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown -R "$POWERX_HOST_UID:$POWERX_HOST_GID" "$power_dir" || collector_rc=1 + fi + write_control "done-$rank" "$collector_rc" + finished=1 +} +trap finish EXIT +trap 'collector_rc=130; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 130' INT +trap 'collector_rc=143; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 143' TERM HUP + +case "${POWERX_CLOCK_SYNCHRONIZED:-false}" in + yes|true) clock_synchronized=true ;; + *) clock_synchronized=false ;; +esac + +python3 -m infx.results.power.native_multinode begin --directory "$power_dir" \ + --vendor "$vendor" --rank "$rank" --role "$role" --gpu-indices "$gpu_indices" \ + --num-nodes "$num_nodes" --clock-synchronized "$clock_synchronized" || exit 1 +start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 +[[ "$GPU_MONITOR_VENDOR" == "$vendor" ]] || exit 1 +if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json +fi +write_control "ready-$rank" ready +while [[ ! -f "$control_dir/stop" ]]; do + _background_process_is_running "$GPU_MONITOR_PID" || exit 1 + sleep 1 & + wait $! || true +done diff --git a/benchmarks/native_power_lifecycle.sh b/benchmarks/native_power_lifecycle.sh new file mode 100644 index 000000000..b24d1dada --- /dev/null +++ b/benchmarks/native_power_lifecycle.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Shared barriers for native collectors. Launchers own host scratch and mounts. + +powerx_start_collector() { + local power_dir="$1" control_dir="$2" vendor="$3" rank="$4" role="$5" gpus="$6" nodes="$7" + local indices + indices=$(seq -s, 0 "$((gpus - 1))") || return 1 + POWERX_CONTROL_DIR="$control_dir" + POWERX_NUM_NODES="$nodes" + bash "$(dirname "${BASH_SOURCE[0]}")/native_power_collect.sh" \ + "$power_dir" "$control_dir" "$vendor" "$rank" "$role" "$indices" "$nodes" & + POWERX_COLLECTOR_PID=$! +} + +powerx_write_control() { + local path="$POWERX_CONTROL_DIR/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +powerx_wait_collectors() { + local phase="$1" deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rank pending failed + while :; do + pending=0 + failed=0 + for ((rank=0; rank&2 + return 1 + fi + if [[ ! -f "$POWERX_CONTROL_DIR/$phase-$rank" ]]; then + pending=1 + elif [[ "$phase" == done && "$(cat "$POWERX_CONTROL_DIR/done-$rank")" != 0 ]]; then + failed=1 + fi + done + if [[ "$pending" == 0 ]]; then + [[ "$failed" == 0 ]] || echo "One or more PowerX collectors failed" >&2 + return "$failed" + fi + if (( SECONDS >= deadline )); then + echo "Timed out waiting for PowerX $phase receipts" >&2 + return 1 + fi + sleep 1 + done +} + +powerx_stop_collectors() { + local rc=0 + powerx_write_control stop stop || rc=$? + powerx_wait_collectors done || rc=$? + powerx_reap_collector || rc=$? + return "$rc" +} + +powerx_reap_collector() { + [[ -n "${POWERX_COLLECTOR_PID:-}" ]] || return 0 + local deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rc=0 + while kill -0 "$POWERX_COLLECTOR_PID" 2>/dev/null; do + if (( SECONDS >= deadline )); then + kill -TERM "$POWERX_COLLECTOR_PID" 2>/dev/null || true + sleep 2 + kill -KILL "$POWERX_COLLECTOR_PID" 2>/dev/null || true + rc=1 + break + fi + sleep 1 + done + wait "$POWERX_COLLECTOR_PID" || rc=$? + POWERX_COLLECTOR_PID="" + return "$rc" +} diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index a1e357b30..6da11624d 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -106,6 +106,12 @@ For multinode fixed-sequence jobs, `utils/process_result.py --all` processes eve Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. +### Native multinode telemetry + +`native_power_collect.sh` and `native_power_lifecycle.sh` provide per-node collection and bounded ready/stop receipts. Launchers opt into the native package under `LOGS/native_power`; this prerequisite enables no new recipe. The adapter validates serving GPU identity, synchronized clocks, collector completion, and complete formal-window coverage. It preserves per-node failures, sample counts, and collector revision in the audit. + +SMI collection records UTC context beside each CSV for portable replay. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index 6a054d5f0..fd3fa2a2d 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -106,6 +106,12 @@ InferenceX-app 将路由字段作为列或配置维度，并把数值测量存 启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 +### 原生多节点遥测 + +`native_power_collect.sh` 和 `native_power_lifecycle.sh` 提供每节点采集及有时限的就绪/停止状态文件。启动器可使用 `LOGS/native_power` 下的原生产物；此前置改动不会启用新 recipe。适配器验证服务 GPU 身份、时钟同步、采集完成及正式窗口完整覆盖，并在审计中保留节点故障、样本数和采集器版本。 + +SMI 采集在 CSV 旁记录 UTC 上下文以支持跨环境回放。正式窗口外的无效样本不能构成覆盖；`boundary_degenerate_rows` 保留其逐 GPU 计数。 + ## 评测工件 ### 单配置身份和收集 diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index f6fa76102..5ae8a65ee 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -248,6 +248,21 @@ def aggregate_power_result( expected_num_gpus = int(env['TP']) * int(env.get('PP_SIZE', '1')) * int(env.get('PCP_SIZE', '1')) try: if is_multinode: + native_dir = Path(env.get('POWERX_NATIVE_DIR', 'LOGS/native_power')) + if env.get('POWERX_NATIVE_DIR') or native_dir.is_dir(): + if source.is_dir() and source != native_dir: + raise ValueError('Both native and SRT power packages are present') + source = native_dir + from .power.native_multinode import run + + return run( + native_dir, bench_path, agg_path, + expected_prefill_gpus=prefill_gpus, + expected_decode_gpus=decode_gpus, + expected_aggregate_gpus=aggregate_gpus, + validation_result=validation_path, + require_power=require_power, + ) from .power.multinode import run return run( diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py new file mode 100644 index 000000000..1ffae5bd9 --- /dev/null +++ b/infx/results/power/native_multinode.py @@ -0,0 +1,286 @@ +"""Validate native SMI traces from each node of one fixed-sequence deployment. + +This format is owned by InferenceX. It does not claim the srt-slurm/DCGM wire +contract. The launcher records real serving-device membership and synchronized +host clocks; the normal result processor binds every trace to the client window. +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import os +import socket +import time +from datetime import timezone +from pathlib import Path + +from . import ALL_POWER_METRIC_KEYS +from .common import ( + _load_benchmark_data, _write_json_atomic, benchmark_window_payload, + patch_power_metrics, +) +from .single_node import _derived_metrics, _detect_columns, _parse_timestamp, integrate_power + + +def _identity(path: Path, vendor: str) -> dict[str, str]: + """Map the vendor enumeration index to a physical UUID; never use PCI alone.""" + if vendor == "nvidia": + with path.open(newline="") as stream: + rows = [] + for row in csv.DictReader(stream, skipinitialspace=True): + if any(not isinstance(k, str) or not isinstance(v, str) for k, v in row.items()): + raise ValueError("invalid_device_identity") + rows.append({k.strip().lower(): v.strip() for k, v in row.items()}) + elif vendor == "amd": + payload = json.loads(path.read_text()) + rows = [] + + def visit(value: object) -> None: + if isinstance(value, list): + for item in value: + visit(item) + elif isinstance(value, dict): + row = {str(k).lower(): v for k, v in value.items()} + if "gpu" in row and "uuid" in row: + rows.append(row) + else: + for item in value.values(): + visit(item) + + visit(payload) + else: + raise ValueError("unsupported_native_vendor") + identities = {} + for row in rows: + index = str(row.get("index", row.get("gpu", ""))).strip() + uuid = str(row.get("uuid", "")).strip() + if not index.isdigit() or not uuid or uuid.lower() in {"n/a", "none", "null"}: + raise ValueError("invalid_device_identity") + if index in identities or uuid in identities.values(): + raise ValueError("duplicate_device_identity") + identities[index] = uuid + if not identities: + raise ValueError("device_identity_missing") + return identities + + +def record_begin(directory: Path, *, vendor: str, node: str, rank: int, + role: str, gpu_indices: list[int], num_nodes: int, job_id: str, + clock_synchronized: bool, revision: str) -> None: + if (role not in {"prefill", "decode", "aggregate"} or rank < 0 or + num_nodes <= rank or not gpu_indices or min(gpu_indices) < 0 or + len(set(gpu_indices)) != len(gpu_indices)): + raise ValueError("invalid_native_topology") + directory.mkdir(parents=True, exist_ok=True) + _write_json_atomic(directory / "manifest.json", { + "schema_version": 1, "collector": "inferencex-native-smi", + "vendor": vendor, "node": node, "rank": rank, "role": role, + "selected_gpu_indices": gpu_indices, "expected_num_nodes": num_nodes, + "job_id": job_id, "collector_revision": revision, + "clock_source": "utc_ntp", "clock_synchronized": clock_synchronized, + "clock_observation": "timedatectl NTPSynchronized on serving host; no measured clock offset", + "collection_start_unix": time.time(), "lifecycle": "collecting", + }) + + +def record_end(directory: Path, *, collector_exit_code: int) -> None: + manifest = json.loads((directory / "manifest.json").read_text()) + manifest.update(collection_end_unix=time.time(), + collector_exit_code=collector_exit_code, + lifecycle="complete" if collector_exit_code == 0 else "failed") + _write_json_atomic(directory / "manifest.json", manifest) + + +def run(power_dir: Path, bench_result: Path, agg_result: Path, *, + expected_prefill_gpus: int, expected_decode_gpus: int, expected_aggregate_gpus: int = 0, + validation_result: Path | None = None, require_power: bool = False) -> int: + validation_result = validation_result or bench_result.with_name( + f"power_validation_{bench_result.stem}.json") + benchmark, reasons = _load_benchmark_data(bench_result) + expected_gpus = expected_prefill_gpus + expected_decode_gpus + expected_aggregate_gpus + if expected_aggregate_gpus and (expected_prefill_gpus or expected_decode_gpus): + reasons.append("native_mixed_aggregate_role_topology") + roles: dict[str, list[str]] = {"prefill": [], "decode": [], "aggregate": []} + receipts = [] + node_errors = [] + samples = [] + ranks = [] + nodes = [] + jobs = set() + revisions = set() + expected_nodes = set() + paths = sorted(power_dir.glob("node-*/manifest.json")) + if not paths: + reasons.append("native_manifests_missing") + if expected_gpus <= 0 or min(expected_prefill_gpus, expected_decode_gpus, expected_aggregate_gpus) < 0: + reasons.append("invalid_expected_gpu_count") + for path in paths: + try: + manifest = json.loads(path.read_text()) + rank = manifest["rank"] + role = manifest["role"] + if (manifest.get("schema_version") != 1 or + manifest.get("collector") != "inferencex-native-smi" or + type(rank) is not int or rank < 0 or role not in roles or + path.parent.name != f"node-{rank}"): + raise ValueError("invalid_native_manifest") + if (not isinstance(manifest.get("node"), str) or not manifest["node"] or + not isinstance(manifest.get("job_id"), str) or + not isinstance(manifest.get("collector_revision"), str) or + type(manifest.get("expected_num_nodes")) is not int or + manifest["expected_num_nodes"] <= 0): + raise ValueError("invalid_native_run_identity") + ranks.append(rank) + nodes.append(manifest["node"]) + expected_nodes.add(manifest["expected_num_nodes"]) + jobs.add(manifest["job_id"]) + revisions.add(manifest["collector_revision"]) + if (manifest.get("lifecycle") != "complete" or + manifest.get("collector_exit_code") != 0): + reasons.append("native_collector_incomplete") + if (manifest.get("clock_source") != "utc_ntp" or + manifest.get("clock_synchronized") is not True): + reasons.append("native_clock_not_synchronized") + start, end = manifest["collection_start_unix"], manifest["collection_end_unix"] + if (not all(type(x) in (int, float) and math.isfinite(x) for x in (start, end)) + or end <= start or (benchmark is not None and + (start > benchmark.start_unix or end < benchmark.end_unix))): + reasons.append("native_collection_window_mismatch") + selected = manifest["selected_gpu_indices"] + if (not isinstance(selected, list) or not selected or + any(type(i) is not int or i < 0 for i in selected) or + len(set(selected)) != len(selected)): + raise ValueError("invalid_native_gpu_selection") + suffix = "csv" if manifest["vendor"] == "nvidia" else "json" + stem = "gpu_metrics_identity" if suffix == "csv" else "gpu_metrics_devices" + first_path = path.parent / f"{stem}.{suffix}" + last_path = path.parent / f"{stem}_end.{suffix}" + first = _identity(first_path, manifest["vendor"]) + last = _identity(last_path, manifest["vendor"]) + selected_ids = {str(i): first[str(i)] for i in selected} + if any(last.get(i) != uuid for i, uuid in selected_ids.items()): + reasons.append("native_device_identity_changed") + previous = {uuid for members in roles.values() for uuid in members} + if previous.intersection(selected_ids.values()): + reasons.append("native_duplicate_physical_gpu") + roles[role].extend(selected_ids.values()) + csv_path = path.parent / "gpu_metrics.csv" + with csv_path.open(newline="") as stream: + reader = csv.DictReader(stream, skipinitialspace=True) + reader.fieldnames = [c.strip() for c in (reader.fieldnames or [])] + t_col, p_col, g_col = _detect_columns(reader.fieldnames) + if not all((t_col, p_col, g_col)): + raise ValueError("native_telemetry_columns_missing") + for row in reader: + gpu = (row.get(g_col) or "").strip() + if gpu not in first: + reasons.append("native_unknown_device_index") + continue + if gpu not in selected_ids: + continue + timestamp = _parse_timestamp((row.get(t_col) or ""), naive_timezone=timezone.utc) + # Invalid rows are retained for the common validator to reject. + samples.append((timestamp, selected_ids[gpu], row.get(p_col) or "")) + receipts.append({**manifest, "physical_gpu_ids": selected_ids, + "manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "telemetry_sha256": hashlib.sha256(csv_path.read_bytes()).hexdigest(), + "identity_sha256": hashlib.sha256(first_path.read_bytes()).hexdigest(), + "identity_end_sha256": hashlib.sha256(last_path.read_bytes()).hexdigest()}) + except (OSError, ValueError, KeyError, TypeError, csv.Error) as exc: + reasons.append("native_node_invalid") + node_errors.append({"node": path.parent.name, "type": type(exc).__name__, "detail": str(exc)}) + if (len(expected_nodes) != 1 or not expected_nodes or + type(next(iter(expected_nodes), None)) is not int or + set(ranks) != set(range(next(iter(expected_nodes), 0))) or + len(set(nodes)) != len(nodes) or len(ranks) != len(set(ranks))): + reasons.append("native_node_topology_mismatch") + if len(jobs) != 1 or "" in jobs or len(revisions) != 1 or "" in revisions: + reasons.append("native_run_identity_mismatch") + if expected_aggregate_gpus: + if roles["prefill"] or roles["decode"] or len(roles["aggregate"]) != expected_aggregate_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif expected_decode_gpus: + if roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or len(roles["decode"]) != expected_decode_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif roles["decode"] or len(roles["aggregate"] + roles["prefill"]) != expected_prefill_gpus: + reasons.append("native_role_gpu_count_mismatch") + combined_path = validation_result.with_name(f"{validation_result.stem}_native.csv") + combined_path.parent.mkdir(parents=True, exist_ok=True) + with combined_path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["timestamp", "gpu", "power"]) + writer.writerows(samples) + integration = None + if benchmark is not None: + integration = integrate_power(combined_path, start_unix=benchmark.start_unix, + end_unix=benchmark.end_unix, expected_num_gpus=expected_gpus) + reasons.extend(integration.invalid_reasons) + metrics = {} + if not reasons and integration is not None and benchmark is not None: + metrics = _derived_metrics(integration, benchmark) + if expected_decode_gpus: + prefill = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["prefill"]) + decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) + metrics.update(prefill_gpu_energy_j=prefill, decode_gpu_energy_j=decode, + prefill_avg_power_w=prefill / benchmark.integration_duration_s / expected_prefill_gpus, + decode_avg_power_w=decode / benchmark.integration_duration_s / expected_decode_gpus, + prefill_joules_per_input_token=prefill / benchmark.total_input_tokens, + decode_joules_per_output_token=decode / benchmark.total_output_tokens) + valid = not reasons + try: + patch_power_metrics(agg_result, metric_keys=ALL_POWER_METRIC_KEYS, power_valid=valid, metrics=metrics) + except (OSError, ValueError): + reasons.append("aggregate_result_unwritable") + valid, metrics = False, {} + audit = {"schema_version": 1, "telemetry_kind": "native_multinode_smi", + "power_valid": valid, "reasons": list(dict.fromkeys(reasons)), + "benchmark_result": str(bench_result), + "benchmark_result_sha256": hashlib.sha256(bench_result.read_bytes()).hexdigest() if bench_result.is_file() else None, + "benchmark_window": benchmark_window_payload(benchmark), + "expected_gpu_count": expected_gpus, "nodes": receipts, + "node_errors": node_errors, + "observed_gpu_count": integration.observed_num_gpus if integration else 0, + "per_gpu_role": {uuid: role for role, uuids in roles.items() for uuid in uuids}, + "per_gpu_sample_counts": integration.per_gpu_sample_counts if integration else {}, + "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s if integration else {}, + "producer": {"name": "inferencex-native-smi", "revisions": sorted(revisions), + "producer_git_commit": next(iter(revisions)) if len(revisions) == 1 else None}, + "integration_method": "per_device_trapezoidal_with_linear_boundary_interpolation", + "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear", + "metrics": metrics} + _write_json_atomic(validation_result, audit) + return int(require_power and not valid) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + begin = sub.add_parser("begin") + begin.add_argument("--directory", type=Path, required=True) + begin.add_argument("--vendor", choices=("amd", "nvidia"), required=True) + begin.add_argument("--node", default=os.environ.get("POWERX_NODE_NAME", socket.gethostname())) + begin.add_argument("--rank", type=int, required=True) + begin.add_argument("--role", choices=("prefill", "decode", "aggregate"), required=True) + begin.add_argument("--gpu-indices", required=True) + begin.add_argument("--num-nodes", type=int, required=True) + begin.add_argument("--job-id", default=os.environ.get("SLURM_JOB_ID", "")) + begin.add_argument("--revision", default=os.environ.get("POWERX_COLLECTOR_REVISION", "")) + begin.add_argument("--clock-synchronized", choices=("true", "false"), default="false") + end = sub.add_parser("end") + end.add_argument("--directory", type=Path, required=True) + end.add_argument("--collector-exit-code", type=int, required=True) + args = vars(parser.parse_args()) + action = args.pop("action") + if action == "begin": + args["gpu_indices"] = [int(i) for i in args["gpu_indices"].split(",")] + args["clock_synchronized"] = args["clock_synchronized"] == "true" + record_begin(**args) + else: + record_end(**args) + + +if __name__ == "__main__": + main() diff --git a/infx/results/power/single_node.py b/infx/results/power/single_node.py index 54669a1a8..4ce1bcaaf 100644 --- a/infx/results/power/single_node.py +++ b/infx/results/power/single_node.py @@ -9,7 +9,11 @@ aggregate and a validation sidecar, but does not fail the benchmark. Power studies can set ``REQUIRE_POWER=1`` to fail after those audit artifacts exist. The aggregate carries numeric ``power_valid`` (1/0) for metric ingestion; the -sidecar is the canonical source for boolean validity and reason codes. +sidecar is the canonical source for boolean validity and reason codes. Rows in +the ingest band but outside the formal window whose power is missing, +non-finite, or <= 0 are teardown noise: they are skipped and counted in the +sidecar's ``boundary_degenerate_rows`` instead of poisoning validity or faking +window bracketing. """ from __future__ import annotations @@ -22,7 +26,7 @@ import os import re import sys -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from statistics import mean @@ -65,6 +69,10 @@ class PowerIntegration: per_gpu_max_sample_gap_s: dict[str, float] per_gpu_energy_j: dict[str, float] device_issues: dict[str, list[str]] + # Rows in the ingest band but outside the formal window whose power was + # missing/N-A/non-finite/<=0, skipped and counted per GPU; "unknown" + # buckets rows without a GPU identity. + boundary_degenerate_rows: dict[str, int] = field(default_factory=dict) avg_power_w: float | None = None p75_power_w: float | None = None p75_total_gpu_power_w: float | None = None @@ -78,7 +86,7 @@ def observed_num_gpus(self) -> int: return len(self.observed_gpu_ids) -def _parse_timestamp(value: str) -> float | None: +def _parse_timestamp(value: str, *, naive_timezone: timezone | None = None) -> float | None: """Best-effort timestamp parse to Unix epoch seconds (local wall clock). Handles the formats observed in practice: @@ -96,7 +104,7 @@ def _parse_timestamp(value: str) -> float | None: # nvidia-smi: "YYYY/MM/DD HH:MM:SS.ffffff" for fmt in ("%Y/%m/%d %H:%M:%S.%f", "%Y/%m/%d %H:%M:%S"): try: - return datetime.strptime(value, fmt).timestamp() + return datetime.strptime(value, fmt).replace(tzinfo=naive_timezone).timestamp() except ValueError: pass # ISO 8601 (amd-smi variants). fromisoformat tolerates 'T' or space separator @@ -108,7 +116,7 @@ def _parse_timestamp(value: str) -> float | None: return None if dt.tzinfo is None: # Treat naive timestamps as local time (matches nvidia-smi convention). - return dt.timestamp() + return dt.replace(tzinfo=naive_timezone).timestamp() return dt.astimezone(timezone.utc).timestamp() @@ -146,6 +154,16 @@ def _detect_columns(header: list[str]) -> tuple[str | None, str | None, str | No return timestamp_col, power_col, gpu_col +def _telemetry_timezone(csv_path: Path) -> timezone | None: + context = csv_path.with_name(f"{csv_path.stem}_context.json") + if not context.is_file(): + return None + payload = json.loads(context.read_text()) + if not isinstance(payload, dict) or payload.get("timestamp_timezone") != "UTC": + raise ValueError("unsupported_telemetry_timezone") + return timezone.utc + + def aggregate_power( csv_path: Path, start_unix: float, @@ -162,6 +180,7 @@ def aggregate_power( return None try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [c.strip() for c in (reader.fieldnames or [])] @@ -189,7 +208,7 @@ def aggregate_power( for row in reader: ts_raw = (row.get(timestamp_col) or "").strip() pw_raw = (row.get(power_col) or "").strip() - ts = _parse_timestamp(ts_raw) + ts = _parse_timestamp(ts_raw, naive_timezone=timestamp_timezone) pw = _parse_power(pw_raw) if ts is None or pw is None: continue @@ -204,7 +223,7 @@ def aggregate_power( if gpu_id: per_sample_gpus.setdefault(bucket, set()).add(gpu_id) gpu_keys.add(gpu_id) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): return None if not per_sample_total: @@ -237,6 +256,7 @@ def _empty_integration( *, expected_num_gpus: int | None, reasons: list[str], + boundary_degenerate_rows: dict[str, int] | None = None, ) -> PowerIntegration: """Build an invalid integration result when no device data is available.""" return PowerIntegration( @@ -248,6 +268,7 @@ def _empty_integration( per_gpu_max_sample_gap_s={}, per_gpu_energy_j={}, device_issues={}, + boundary_degenerate_rows=boundary_degenerate_rows or {}, ) @@ -308,8 +329,10 @@ def integrate_power( # expose timestamps at lower resolution than their sampling cadence, so # duplicate-timestamp readings are averaged rather than treated as corrupt. raw_samples: dict[str, dict[float, list[float]]] = {} + boundary_degenerate: dict[str, int] = {} saw_missing_gpu_identity = False try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -332,7 +355,7 @@ def integrate_power( ) for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) if timestamp is None or not math.isfinite(timestamp): _append_reason(reasons, "invalid_timestamp_sample") continue @@ -348,6 +371,15 @@ def integrate_power( power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() + if (power is None or not math.isfinite(power) or power <= 0.0) and ( + timestamp < start_unix or timestamp > end_unix + ): + # SMI teardown rows can carry N/A or 0 W cells: outside the + # formal window they are counted, never used to satisfy + # bracketing or to poison in-window validity. + key = gpu_id or "unknown" + boundary_degenerate[key] = boundary_degenerate.get(key, 0) + 1 + continue if power is None: _append_reason(reasons, "invalid_power_sample") continue @@ -359,11 +391,12 @@ def integrate_power( continue values = raw_samples.setdefault(gpu_id, {}).setdefault(timestamp, []) values.append(power) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): _append_reason(reasons, "telemetry_file_unreadable") return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) if saw_missing_gpu_identity: @@ -373,6 +406,7 @@ def integrate_power( return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) observed_gpu_ids = tuple(sorted(raw_samples, key=_gpu_sort_key)) @@ -450,6 +484,7 @@ def integrate_power( per_gpu_max_sample_gap_s=per_gpu_max_sample_gap_s, per_gpu_energy_j=per_gpu_energy_j, device_issues=device_issues, + boundary_degenerate_rows=boundary_degenerate, avg_power_w=avg_power_w, p75_power_w=p75_total / len(observed_gpu_ids) if p75_total is not None else None, p75_total_gpu_power_w=p75_total, @@ -485,6 +520,7 @@ def _read_energy_snapshot(path: Path) -> dict[str, float] | None: def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float]]]: """Group every parseable (timestamp, watt) sample per GPU, sorted in time.""" samples: dict[str, list[tuple[float, float]]] = {} + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -493,7 +529,7 @@ def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float] if not timestamp_col or not power_col or not gpu_col: return {} for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() if timestamp is None or power is None or not gpu_id: @@ -679,6 +715,7 @@ def _validation_payload( "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s, "per_gpu_energy_j": integration.per_gpu_energy_j, "device_issues": integration.device_issues, + "boundary_degenerate_rows": integration.boundary_degenerate_rows, "accumulator_check": accumulator_check, "metrics": audit_metrics(metrics), } @@ -754,7 +791,7 @@ def run( try: accumulator_check = cross_check_accumulator(csv_path) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): accumulator_check = {"available": False, "reason": "cross_check_error"} try: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f3b85de24..6d321aec8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7410,3 +7410,14 @@ - "Preserve request failure outcomes, retain all available fixed-sequence diagnostics, and reject incomplete concurrency batches before publishing benchmark artifacts." - "保留请求失败状态及全部可用的固定序列诊断信息；发布基准产物前拒绝缺失并发点的批次。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Retain UTC SMI telemetry context and validate native multinode collector artifacts without enabling new recipes." + - "保留 UTC SMI 遥测上下文并验证原生多节点采集产物，不启用新 recipe。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3051 diff --git a/runners/test_native_collector_barriers.py b/runners/test_native_collector_barriers.py new file mode 100644 index 000000000..14c8fe2ce --- /dev/null +++ b/runners/test_native_collector_barriers.py @@ -0,0 +1,58 @@ +"""CPU-only lifecycle checks with controlled external collector/Slurm processes.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +LIFECYCLE = ROOT / 'benchmarks/native_power_lifecycle.sh' +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' + + +@pytest.mark.parametrize('failed_rank', [None, 1]) +def test_stop_waits_for_every_collector_and_retains_failure(tmp_path, failed_rank): + command = ''' +source "$1" +export POWERX_CONTROL_DIR="$2" POWERX_NUM_NODES=2 POWERX_BARRIER_TIMEOUT_S=5 +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.1; echo 0 > "$2/done-0") & +POWERX_COLLECTOR_PID=$! +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.2; echo "$3" > "$2/done-1") & +remote_pid=$! +powerx_stop_collectors +rc=$? +wait "$remote_pid" +exit "$rc" +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(LIFECYCLE), + str(tmp_path), '1' if failed_rank else '0'], + capture_output=True, text=True, timeout=10) + assert result.returncode == (1 if failed_rank else 0), result.stderr + assert (tmp_path / 'done-0').read_text().strip() == '0' + assert (tmp_path / 'done-1').read_text().strip() == ('1' if failed_rank else '0') + + +def test_ready_barrier_rejects_collector_that_already_stopped(tmp_path): + (tmp_path / 'ready-0').write_text('ready') + (tmp_path / 'done-0').write_text('1') + result = subprocess.run(['bash', '-c', 'source "$1"; POWERX_CONTROL_DIR="$2"; ' + 'POWERX_NUM_NODES=1; powerx_wait_collectors ready', + 'bash', str(LIFECYCLE), str(tmp_path)], capture_output=True, text=True) + assert result.returncode == 1 + assert 'stopped before benchmark readiness' in result.stderr + + +def test_control_receipt_is_owned_before_publication(tmp_path): + command = '''source "$1" +POWERX_CONTROL_DIR=$2 +POWERX_HOST_UID=1000 POWERX_HOST_GID=1000 +chown() { [[ ! -e "$POWERX_CONTROL_DIR/stop" ]]; } +powerx_write_control stop stop +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(LIFECYCLE), str(tmp_path)], + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert (tmp_path / 'stop').read_text() == 'stop\n' + assert not list(tmp_path.glob('*.tmp')) diff --git a/runners/test_native_collector_receipts.py b/runners/test_native_collector_receipts.py new file mode 100644 index 000000000..b981f57ce --- /dev/null +++ b/runners/test_native_collector_receipts.py @@ -0,0 +1,16 @@ +import subprocess +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] + +def test_native_control_receipt_is_owned_before_publication(tmp_path): + source = (ROOT / 'benchmarks/native_power_collect.sh').read_text() + function = source[source.index('write_control() {'):source.index('\nfinish() {')] + result = subprocess.run(['bash', '-c', function + ''' +control_dir=$1 +POWERX_HOST_UID=1000 POWERX_HOST_GID=1000 +chown() { [[ ! -e "$control_dir/done-0" ]]; } +write_control done-0 7 +''', 'bash', str(tmp_path)], capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert (tmp_path / 'done-0').read_text() == '7\n' + assert not list(tmp_path.glob('*.tmp')) diff --git a/utils/test_aggregate_power.py b/utils/test_aggregate_power.py index aa3bf23f9..5b8b76121 100644 --- a/utils/test_aggregate_power.py +++ b/utils/test_aggregate_power.py @@ -574,6 +574,201 @@ def test_run_rejects_malformed_telemetry_inside_window( assert audit["reasons"] == [expected_reason] + +# AMDSMI 26.2.0 `metric -p -c -t -u -w 1 --csv` header (order-faithful subset, +# measured on MI355X; see test_detect_columns_amd_watch_mode_real_header). +_MI355X_WATCH_HEADER = ( + "timestamp,gpu,gfx_activity,umc_activity,mm_activity,vcn_activity," + "jpeg_activity,gfx_busy_inst_xcp_0,jpeg_busy_xcp_0,vcn_busy_xcp_0," + "socket_power,gfx_voltage,soc_voltage,mem_voltage,throttle_status," + "power_management,gfx_0_clk,mem_0_clk,edge,hotspot,mem" +) + + +def _mi355x_watch_row(timestamp: int, gpu: int, socket_power: str) -> str: + """One data row in the shape captured from run 32433563482 (conc1): + integer-second epoch, quoted list cells with embedded commas, N/A cells, + and a trailing carriage return.""" + return ( + f"{timestamp},{gpu},0,0,N/A,\"['N/A', 'N/A', 'N/A', 'N/A']\"," + "\"['N/A', 'N/A']\",\"[0, 0, 0, 0, 0, 0, 0, 0]\",\"[0, 0]\",\"[0, 0]\"," + f"{socket_power},N/A,N/A,N/A,N/A,ENABLED,1404,2000,N/A,40,25\r" + ) + + +def test_integrate_power_skips_na_power_rows_outside_window(tmp_path: Path): + """N/A-power teardown rows past the window end are counted, not poisonous.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-2, 13): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},N/A,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + assert result.total_gpu_energy_j == pytest.approx(10_000.0) + + +def test_integrate_power_does_not_bracket_with_zero_power_tail(tmp_path: Path): + """A 0 W teardown row past the window end must not fake end bracketing. + + Legacy behavior accepted the 0 W row as a valid boundary sample, corrupting + the end interpolation with a bogus value; this intentionally flips that + case to an explicit benchmark_window_not_bracketed failure.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + # Good rows stop at end-4; the only post-end sample per GPU has power 0. + for offset in range(-1, 7): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},0,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "benchmark_window_not_bracketed" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + + +def test_integrate_power_preserves_boundary_counts_without_usable_samples(tmp_path: Path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text( + "timestamp,gpu,socket_power\n" + "1699999999,0,N/A\n" + "1700000011,0,0\n" + "1700000011,1,N/A\n", + encoding="utf-8", + ) + + result = integrate_power( + csv, + start_unix=1_700_000_000, + end_unix=1_700_000_010, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "no_usable_power_samples" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 2, "1": 1} + + +def test_integrate_power_keeps_zero_power_semantics_inside_window(tmp_path: Path): + """Frozen legacy behavior: an in-window 0 W sample still integrates.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-1, 12): + watts = "0.0" if offset == 5 else "500.0" + lines.append(f"{base + offset},0,{watts},65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=1, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {} + # Trapezoids dip to 0 at t=5: 8 x 500 + 2 x 250 = 4500 J. + assert result.total_gpu_energy_j == pytest.approx(4_500.0) + + +def test_run_writes_boundary_degenerate_rows_to_sidecar(tmp_path: Path): + base = 1_700_000_000.0 + csv = tmp_path / "gpu_metrics.csv" + _write_constant_window_samples( + csv, + start=base, + end=base + 10, + watts_per_gpu=500.0, + num_gpus=2, + ) + bench = tmp_path / "bench.json" + agg = tmp_path / "agg.json" + validation = tmp_path / "power_validation.json" + _write_bench_result( + bench, + start=base, + end=base + 10, + duration=10.0, + total_output=2_000, + total_input=10_000, + ) + agg.write_text(json.dumps({"hw": "mi355x"}), encoding="utf-8") + + exit_code = run(csv, bench, agg, expected_num_gpus=2, validation_result=validation) + + assert exit_code == 0 + audit = json.loads(validation.read_text()) + # Present-and-empty for clean streams: readers can rely on the key. + assert audit["boundary_degenerate_rows"] == {} + + +def test_integrate_power_regression_mi355x_integer_ticks_end_gap(tmp_path: Path): + """Run-32433563482 conc1 regression: amd-smi integer-second ticks stop 4 s + before the fractional aiperf window end (last tick 1787277605 vs end + ...609.157497), so bracketing fails and the producer-side telemetry loss is + attributed as benchmark_window_not_bracketed on every GPU. + + The retrieved artifact's trailing rows all carry valid socket_power + (254-264 W) with N/A activity/voltage cells; the N/A- and 0-power teardown + rows appended past the window end are the documented synthetic degenerate + shapes, asserting they are counted rather than used for bracketing.""" + csv = tmp_path / "gpu_metrics.csv" + start = 1_787_277_560.155891 + end = 1_787_277_609.157497 + last_tick = 1_787_277_605 + powers = [259, 255, 263, 264, 256, 254, 259, 259] + lines = [_MI355X_WATCH_HEADER] + for tick in range(1_787_277_555, last_tick + 1): + for gpu in range(8): + lines.append(_mi355x_watch_row(tick, gpu, str(powers[gpu]))) + # amd-smi watch mode emits a blank line between tick groups. + lines.append("") + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_610, gpu, "N/A")) + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_611, gpu, "0")) + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=start, + end_unix=end, + expected_num_gpus=8, + ) + + assert result.power_valid is False + assert result.invalid_reasons == ("benchmark_window_not_bracketed",) + assert result.device_issues == { + str(gpu): ["benchmark_window_not_bracketed"] for gpu in range(8) + } + assert result.boundary_degenerate_rows == {str(gpu): 2 for gpu in range(8)} + + def test_run_patches_agg_with_power_and_joules(tmp_path: Path): base = 1_700_000_000.0 csv = tmp_path / "gpu_metrics.csv" @@ -708,6 +903,7 @@ def test_power_replacement_removes_stale_metrics(tmp_path, patch_validated_power path.write_text(json.dumps({ "hw": "fixture", "avg_power_w": 99, "total_gpu_energy_j": 50, "power_invalid_reasons": ["stale"], + "power_audit": {"source": "previous-run.json"}, })) patch_validated_power(path, power_valid=valid, metrics={ "avg_power_w": 12.34567, "joules_per_output_token": 0.12345678, @@ -1433,12 +1629,12 @@ def test_power_percentiles_uses_synchronized_total_not_device_percentiles(tmp_pa def test_power_percentiles_weights_time_and_clips_the_validated_window(tmp_path): csv_path = tmp_path / "power.csv" - # Dense readings near the high end must not bias a uniform linear ramp. - _write_amd_csv(csv_path, [(0, 0, 0), (1, 0, 100), (1.9, 0, 190), (2, 0, 200)]) + # Dense readings must not bias the uniform 150-250 W ramp inside the window. + _write_amd_csv(csv_path, [(0, 0, 100), (1, 0, 200), (1.9, 0, 290), (2, 0, 300)]) result = integrate_power(csv_path, start_unix=0.5, end_unix=1.5, expected_num_gpus=1) assert result.power_valid - assert result.p75_power_w == pytest.approx(125) - assert result.p90_power_w == pytest.approx(140) + assert result.p75_power_w == pytest.approx(225) + assert result.p90_power_w == pytest.approx(240) def test_power_percentiles_is_withheld_for_invalid_telemetry(tmp_path): @@ -1462,3 +1658,54 @@ def test_power_percentiles_aligns_asynchronous_gpu_samples(tmp_path): assert result.p75_power_w == pytest.approx(300) assert result.p90_total_gpu_power_w == pytest.approx(600) assert result.p90_power_w == pytest.approx(300) + + +@pytest.mark.parametrize('step_name', ['Upload GPU metrics', 'Upload power audit bundle']) +@pytest.mark.parametrize('directory', ['', 'results']) +def test_uploaded_telemetry_replays_in_a_different_timezone(tmp_path, step_name, directory): + import os + import shutil + import yaml + + repo = Path(__file__).resolve().parents[1] + source = tmp_path / 'source' + telemetry = source / directory + telemetry.mkdir(parents=True) + (telemetry / 'gpu_metrics.csv').write_text( + 'timestamp,index,power.draw [W]\n' + '2024/01/01 00:00:00.000,0,100 W\n' + '2024/01/01 00:00:01.000,0,100 W\n' + '2024/01/01 00:00:02.000,0,100 W\n') + (telemetry / 'gpu_metrics_context.json').write_text('{"timestamp_timezone":"UTC"}') + workflow = yaml.safe_load((repo / '.github/workflows/benchmark-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job['steps'] + if step.get('name') == step_name) + archive = tmp_path / 'downloaded' + for pattern in step['with']['path'].splitlines(): + for file in source.glob(pattern): + destination = archive / file.relative_to(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(file, destination) + result = subprocess.run( + [sys.executable, '-c', """ +import sys, time +from pathlib import Path +from infx.results.power.single_node import integrate_power +time.tzset() +result = integrate_power(Path(sys.argv[1]), start_unix=1704067200, end_unix=1704067202, + expected_num_gpus=1) +assert result.power_valid, result.invalid_reasons +assert result.total_gpu_energy_j == 200 +""", str(archive / directory / 'gpu_metrics.csv')], + cwd=tmp_path, env={**os.environ, 'TZ': 'Etc/GMT+8', 'PYTHONPATH': str(repo)}, + capture_output=True, text=True, timeout=10, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize('context', ['{"timestamp_timezone":"PST"}', '{}', '{invalid']) +def test_legacy_average_rejects_invalid_telemetry_context(tmp_path, context): + csv = tmp_path / 'gpu_metrics.csv' + csv.write_text('timestamp,index,power.draw [W]\n1,0,100\n2,0,100\n') + csv.with_name('gpu_metrics_context.json').write_text(context) + assert aggregate_power(csv, 1, 2) is None diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py new file mode 100644 index 000000000..0aeee2511 --- /dev/null +++ b/utils/test_native_multinode_power.py @@ -0,0 +1,201 @@ +"""Native collector acceptance uses independent, hand-computed two-node traces.""" +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from infx.results.power.native_multinode import record_begin, record_end, run +from infx.results.power.single_node import integrate_power + +REPO = Path(__file__).resolve().parents[1] + + +def _package(tmp_path, vendor="amd"): + root = tmp_path / "native_power" + for rank, role, watts in ((0, "prefill", 100), (1, "decode", 300)): + node = root / f"node-{rank}" + record_begin(node, vendor=vendor, node=f"host-{rank}", rank=rank, role=role, + gpu_indices=[0], num_nodes=2, job_id="job-123", revision="revision-abc", + clock_synchronized=True) + record_end(node, collector_exit_code=0) + manifest = json.loads((node / "manifest.json").read_text()) + manifest.update(collection_start_unix=0, collection_end_unix=5) + (node / "manifest.json").write_text(json.dumps(manifest)) + for ending in ("", "_end"): + if vendor == "amd": + (node / f"gpu_metrics_devices{ending}.json").write_text(json.dumps([ + {"gpu": 0, "uuid": f"uuid-{rank}"}, {"gpu": 1, "uuid": f"unused-{rank}"}])) + else: + (node / f"gpu_metrics_identity{ending}.csv").write_text( + f"index, uuid, pci.bus_id\n0, uuid-{rank}, 0000:01:00.0\n1, unused-{rank}, 0000:02:00.0\n") + # The spare physical GPU is visible but does not belong to the server. + (node / "gpu_metrics.csv").write_text("timestamp,gpu,power\n" + "".join( + f"{tick},0,{watts}\n{tick},1,900\n" for tick in range(5))) + bench = tmp_path / "result.json" + bench.write_text(json.dumps({"benchmark_start_time_unix": 1, "benchmark_end_time_unix": 3, + "duration": 2, "completed": 2, "total_input_tokens": 20, + "total_output_tokens": 10})) + agg = tmp_path / "agg.json" + agg.write_text(json.dumps({"avg_power_w": 999, "prefill_gpu_energy_j": 999})) + return root, bench, agg + + +@pytest.mark.parametrize("vendor", ["amd", "nvidia"]) +def test_native_whole_fleet_and_role_energy_use_only_serving_devices(tmp_path, vendor): + root, bench, agg = _package(tmp_path, vendor) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 1 + assert actual["total_gpu_energy_j"] == 800 + assert actual["avg_power_w"] == 200 + assert actual["p90_power_w"] == 200 + assert actual["p75_power_w"] == 200 + assert actual["joules_per_successful_query"] == 400 + assert actual["prefill_gpu_energy_j"] == 200 + assert actual["decode_gpu_energy_j"] == 600 + assert actual["prefill_joules_per_input_token"] == 10 + assert actual["decode_joules_per_output_token"] == 60 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert audit["observed_gpu_count"] == 2 + assert audit["per_gpu_role"] == {"uuid-0": "prefill", "uuid-1": "decode"} + assert len(audit["nodes"][0]["telemetry_sha256"]) == 64 + from infx.results.power.audit import audit_summary + summary = audit_summary(audit, "power_validation_result.json")["power_audit"] + assert summary["sample_count"] == 10 # Five samples on each of two participating GPUs. + assert summary["producer_sha"] == "revision-abc" + + +def test_native_parse_failure_has_a_public_reason_and_retained_detail(tmp_path): + from infx.results.power.audit import audit_summary + root, bench, agg = _package(tmp_path) + (root / "node-1/manifest.json").write_text("not JSON") + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + summary = audit_summary(audit, "power_validation_result.json") + assert "native_node_invalid" in summary["power_invalid_reasons"] + assert audit["node_errors"][0]["node"] == "node-1" + assert audit["node_errors"][0]["detail"] + + +@pytest.mark.parametrize(("field", "value", "reason"), [ + ("clock_synchronized", False, "native_clock_not_synchronized"), + ("lifecycle", "collecting", "native_collector_incomplete"), + ("collection_end_unix", 2, "native_collection_window_mismatch"), + ("job_id", "other-job", "native_run_identity_mismatch"), + ("role", "prefill", "native_role_gpu_count_mismatch"), + ("expected_num_nodes", 3, "native_node_topology_mismatch"), +]) +def test_native_invalid_evidence_clears_stale_metrics_and_writes_audit(tmp_path, field, value, reason): + root, bench, agg = _package(tmp_path) + path = root / "node-1/manifest.json" + manifest = json.loads(path.read_text()); manifest[field] = value + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 0 + assert "avg_power_w" not in actual + assert "prefill_gpu_energy_j" not in actual + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert reason in audit["reasons"] + + +def test_native_device_replacement_cannot_preserve_validity(tmp_path): + root, bench, agg = _package(tmp_path) + (root / "node-1/gpu_metrics_devices_end.json").write_text('[{"gpu":0,"uuid":"replacement"}]') + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + assert "native_device_identity_changed" in json.loads((tmp_path / "power_validation_result.json").read_text())["reasons"] + + +def test_native_aggregate_nodes_do_not_invent_prefill_decode_metrics(tmp_path): + root, bench, agg = _package(tmp_path) + for path in root.glob("*/manifest.json"): + manifest = json.loads(path.read_text()); manifest["role"] = "aggregate" + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=0, expected_decode_gpus=0, + expected_aggregate_gpus=2, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["avg_power_w"] == 200 + assert "prefill_gpu_energy_j" not in actual + + +def test_utc_context_replays_in_a_different_timezone(tmp_path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text("timestamp,index,power.draw [W]\n2026/01/01 00:00:00,0,100\n2026/01/01 00:00:02,0,100\n") + (tmp_path / "gpu_metrics_context.json").write_text('{"timestamp_timezone":"UTC"}') + script = "from pathlib import Path; from infx.results.power.single_node import integrate_power; " + \ + f"r=integrate_power(Path({str(csv)!r}),start_unix=1767225600,end_unix=1767225602,expected_num_gpus=1); assert r.power_valid; assert r.total_gpu_energy_j == 200" + subprocess.run([sys.executable, "-c", script], cwd=REPO, + env={**os.environ, "TZ": "America/Los_Angeles"}, check=True, timeout=10) + + +@pytest.mark.parametrize("clock_value, synchronized", [ + ("yes", True), ("true", True), ("no", False), ("false", False), ("", False), +]) +def test_native_supervisor_reaps_monitor_and_writes_completion(tmp_path, clock_value, synchronized): + binary = tmp_path / "bin"; binary.mkdir() + (binary / "python3").symlink_to(sys.executable) + fake = binary / "nvidia-smi" + fake.write_text(f'''#!{sys.executable} +import datetime, os, sys, time +if any("index,uuid" in arg for arg in sys.argv): + print("index, uuid, pci.bus_id"); print("0, gpu-0, 0000:01:00.0") +else: + with open({str(tmp_path / 'monitor.pid')!r}, "w") as stream: stream.write(str(os.getpid())) + if "-l" in sys.argv: print("timestamp,index,power.draw [W]", flush=True) + while True: + print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y/%m/%d %H:%M:%S.%f") + ",0,100", flush=True) + if "-l" not in sys.argv: break + time.sleep(0.1) +''') + fake.chmod(0o755) + control = tmp_path / "control"; control.mkdir() + node = tmp_path / "node-0" + process = subprocess.Popen(["bash", str(REPO / "benchmarks/native_power_collect.sh"), + str(node), str(control), "nvidia", "0", "aggregate", "0", "1"], + env={**os.environ, "PATH": f"{binary}:{os.environ['PATH']}", + "SLURM_JOB_ID": "test-job", "POWERX_COLLECTOR_REVISION": "revision", + "POWERX_CLOCK_SYNCHRONIZED": clock_value}, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + try: + deadline = time.monotonic() + 10 + while not (control / "ready-0").exists(): + assert process.poll() is None + assert time.monotonic() < deadline + time.sleep(0.02) + (control / "stop").write_text("stop") + stdout, stderr = process.communicate(timeout=10) + assert process.returncode == 0, (stdout, stderr) + assert (control / "done-0").read_text().strip() == "0" + manifest = json.loads((node / "manifest.json").read_text()) + assert manifest["lifecycle"] == "complete" + assert manifest["clock_synchronized"] is synchronized + assert (node / "gpu_metrics_identity_end.csv").exists() + finally: + if process.poll() is None: + process.terminate(); process.communicate(timeout=10) + + +def test_result_processor_discovers_staged_native_package(tmp_path, monkeypatch): + from infx.results.fixed_sequence import aggregate_power_result + + root, bench, agg = _package(tmp_path) + logs = tmp_path / "LOGS" + logs.mkdir() + root.rename(logs / "native_power") + monkeypatch.chdir(tmp_path) + env = {"IS_MULTINODE": "true", "PREFILL_GPUS": "1", "DECODE_GPUS": "1", + "RESULT_FILENAME": "result", "REQUIRE_POWER": "1"} + assert aggregate_power_result(env, bench, agg) == 0 + assert json.loads(agg.read_text())["total_gpu_energy_j"] == 800 + assert (tmp_path / "power_validation_result.json").is_file() + + # Two competing formats must not silently select one package. + (logs / "power").mkdir() + assert aggregate_power_result(env, bench, agg) == 1 + invalid = json.loads(agg.read_text()) + assert invalid["power_valid"] == 0 + assert "total_gpu_energy_j" not in invalid From af2697e504893e2540cd26309a06068c4036bb0d Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:28:35 -0700 Subject: [PATCH 05/17] feat: collect native power for TileRT 8k1k MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为 TileRT 8k1k 收集原生功耗，保留本任务分配、角色退出状态及采集证据。 --- .../multi_node/tilert_utils/run_node.sh | 42 +++++- benchmarks/multi_node/tilert_utils/submit.sh | 117 ++++++++++++++--- configs/nvidia-master.yaml | 1 + docs/configuration-procedures.md | 4 + docs/configuration-procedures_zh.md | 4 + perf-changelog.yaml | 10 ++ runners/launch_b200-nscale-slurm.sh | 4 + runners/test_tilert_power_lifecycle.py | 122 ++++++++++++++++++ 8 files changed, 280 insertions(+), 24 deletions(-) create mode 100644 runners/test_tilert_power_lifecycle.py diff --git a/benchmarks/multi_node/tilert_utils/run_node.sh b/benchmarks/multi_node/tilert_utils/run_node.sh index 892762415..0b92099c1 100755 --- a/benchmarks/multi_node/tilert_utils/run_node.sh +++ b/benchmarks/multi_node/tilert_utils/run_node.sh @@ -42,6 +42,36 @@ AGENTIC_LOGS_DIR=${AGENTIC_LOGS_DIR:-$RESULT_DIR/LOGS/agentic} mkdir -p "$BENCHMARK_LOGS_DIR" DONE_SENTINEL="$BENCHMARK_LOGS_DIR/.tilert_done.${SLURM_JOB_ID:-local}" +source "$(dirname "$0")/../../native_power_lifecycle.sh" +finish_tilert_node() { + local rc=$? pid + trap - EXIT + if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 && -n "${POWERX_COLLECTOR_PID:-}" ]]; then + if [[ "$TILERT_ROLE" == prefill || "$rc" != 0 ]]; then + powerx_stop_collectors || rc=$? + else + powerx_reap_collector || rc=$? + fi + fi + if [[ "$TILERT_ROLE" == prefill ]]; then + printf '%s\n' "$rc" > "$DONE_SENTINEL.tmp" || rc=1 + if [[ -n "${POWERX_HOST_UID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$DONE_SENTINEL.tmp" || rc=1 + fi + mv -f "$DONE_SENTINEL.tmp" "$DONE_SENTINEL" || rc=1 + fi + for pid in "${ROUTER_PID:-}" "${PREFILL_PID:-}" "${DECODE_PID:-}"; do + [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true + done + exit "$rc" +} +trap finish_tilert_node EXIT +trap 'exit 143' TERM HUP +trap 'exit 130' INT +if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then + powerx_start_collector "/powerx_native/node-$POWERX_RANK" "/powerx_control" \ + nvidia "$POWERX_RANK" "$TILERT_ROLE" "$POWERX_GPU_COUNT" 2 +fi echo "[tilert-run_node] ROLE=$TILERT_ROLE host=$(hostname) DECODE_HOST=$DECODE_HOST PREFILL_HOST=$PREFILL_HOST" log_and_run_bg() { @@ -211,6 +241,9 @@ run_bench_and_eval() { wait_for_server_ready --port "$ROUTER_PORT" \ --server-log "$BENCHMARK_LOGS_DIR/tilert_router.log" --server-pid "$ROUTER_PID" local rc=0 conc np + if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then + powerx_wait_collectors ready || return $? + fi if [[ "${EVAL_ONLY:-false}" != "true" ]]; then for conc in $CONC_LIST; do np=$(( conc * 10 )) @@ -228,7 +261,7 @@ run_bench_and_eval() { || { rc=$?; echo "[bench] WARNING: conc=$conc failed/timed out (rc=$rc)"; } done fi - run_lm_eval + run_lm_eval || rc=$? return $rc } @@ -278,13 +311,13 @@ case "$TILERT_ROLE" in sleep 5 done if [[ -f "$DONE_SENTINEL" ]]; then - echo "[decode] done sentinel received, shutting down"; kill "$DECODE_PID" 2>/dev/null || true; exit 0 + echo "[decode] done sentinel received, shutting down" + exit "$(cat "$DONE_SENTINEL")" fi echo "[decode] decode_server exited early (see $BENCHMARK_LOGS_DIR/tilert_decode.log)"; exit 1 ;; prefill) rdma_preflight || exit 1 - rm -f "$DONE_SENTINEL" if [[ "$TILERT_IS_AGENTIC" == "1" ]]; then resolve_trace_source install_agentic_deps @@ -300,8 +333,7 @@ case "$TILERT_ROLE" in else run_bench_and_eval; BENCH_RC=$? fi - touch "$DONE_SENTINEL" - kill "$ROUTER_PID" "$PREFILL_PID" 2>/dev/null || true + # EXIT drains both collectors before the decode role sees completion. exit $BENCH_RC ;; *) diff --git a/benchmarks/multi_node/tilert_utils/submit.sh b/benchmarks/multi_node/tilert_utils/submit.sh index b63e97bef..c76c33112 100755 --- a/benchmarks/multi_node/tilert_utils/submit.sh +++ b/benchmarks/multi_node/tilert_utils/submit.sh @@ -1,6 +1,11 @@ #!/usr/bin/env bash +set -euo pipefail set -x NODES=$(( ${PREFILL_NODES:-1} + ${DECODE_NODES:-1} )) +if [[ "${PREFILL_NODES:-1}" != 1 || "${DECODE_NODES:-1}" != 1 ]]; then + echo "TileRT native launcher requires one physical node per role" >&2 + exit 1 +fi GPUS_PER_NODE="${GPUS_PER_NODE:-${GPU_COUNT:-$(( ${PREFILL_TP:-8} > ${DECODE_TP:-8} ? ${PREFILL_TP:-8} : ${DECODE_TP:-8} ))}}" SQUASH_DIR="${B200_SQUASH_DIR:-/home/sa-shared/containers}" @@ -15,14 +20,84 @@ squash_path() { echo "$SQUASH_DIR/$(echo "$1" | sed 's/[\/:@#]/_/g').sqsh"; } DECODE_SQUASH="$(squash_path "$DECODE_IMAGE")" PREFILL_SQUASH="$(squash_path "$PREFILL_IMAGE")" -salloc --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \ - --nodes="$NODES" --gres=gpu:"$GPUS_PER_NODE" --exclusive --mem=0 \ - --time="${SALLOC_TIME_LIMIT:-480}" --no-shell --job-name="$RUNNER_NAME" -JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1) - -mapfile -t HOSTS < <(scontrol show hostnames "$(squeue -j "$JOB_ID" -h -o %N)") -[[ "${#HOSTS[@]}" -ge 2 ]] || { echo "expected >=2 nodes, got: ${HOSTS[*]}"; exit 1; } +if [[ "${TILERT_IN_ALLOCATION:-0}" != 1 ]]; then + # Run inside the allocation returned by this request. Looking up a runner + # name can attach to an older job; salloc supplies the authoritative ID. + exec salloc --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \ + --nodes="$NODES" --gres=gpu:"$GPUS_PER_NODE" --exclusive --mem=0 \ + --time="${SALLOC_TIME_LIMIT:-480}" --job-name="$RUNNER_NAME" \ + env TILERT_IN_ALLOCATION=1 bash "$0" "$@" +fi +JOB_ID="${SLURM_JOB_ID:?salloc did not provide its allocation ID}" +mapfile -t HOSTS < <(scontrol show hostnames "${SLURM_JOB_NODELIST:?salloc did not provide its nodes}") +[[ "${#HOSTS[@]}" -eq 2 ]] || { echo "expected 2 nodes, got: ${HOSTS[*]}"; exit 1; } export DECODE_HOST="${HOSTS[0]}" PREFILL_HOST="${HOSTS[1]}" +export POWERX_NATIVE_ENABLED=0 +if [[ "${REQUIRE_POWER:-0}" =~ ^(1|true|TRUE|yes|YES)$ && "$ISL" == 8192 && "$OSL" == 1024 && "${IS_AGENTIC:-0}" != 1 && "${SCENARIO_TYPE:-}" != agentic-coding && "${EVAL_ONLY:-false}" != true ]]; then + export POWERX_NATIVE_ENABLED=1 +fi +export SLURM_JOB_ID="$JOB_ID" +export POWERX_HOST_UID="$(id -u)" POWERX_HOST_GID="$(id -g)" +export POWERX_COLLECTOR_REVISION="$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" +export POWERX_RAW_ROOT="${POWERX_RAW_ROOT:-/tmp/inferencex-native-$JOB_ID}" +export POWERX_CONTROL_ROOT="$GITHUB_WORKSPACE/LOGS/power_control-$JOB_ID" +mkdir -p "$POWERX_CONTROL_ROOT" "$GITHUB_WORKSPACE/LOGS/native_power" +chmod 777 "$POWERX_CONTROL_ROOT" +rm -f "${BENCHMARK_LOGS_DIR:-$GITHUB_WORKSPACE}/.tilert_done.$JOB_ID" +srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 mkdir -p "$POWERX_RAW_ROOT" +srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 chmod 777 "$POWERX_RAW_ROOT" +# Keep node-local receipts inside the allocation until both serving steps drain. +# A caught cancellation exits through the same staging path as normal completion. +wait_owned_step() { + local pid="$1" deadline=$((SECONDS + ${TILERT_DECODE_DRAIN:-60})) rc=0 + while kill -0 "$pid" 2>/dev/null; do + if (( SECONDS >= deadline )); then + echo "[submit] step $pid did not drain before timeout" >&2 + kill -TERM "$pid" 2>/dev/null || true + sleep 2 + kill -KILL "$pid" 2>/dev/null || true + rc=1 + break + fi + sleep 1 + done + wait "$pid" || rc=$? + return "$rc" +} + +finish_tilert_submit() { + local rc=$? step_rc pid role_rank + trap - EXIT + trap '' TERM HUP INT + if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + printf 'stop\n' > "$POWERX_CONTROL_ROOT/stop" || { [[ "$rc" != 0 ]] || rc=1; } + fi + if [[ "$rc" != 0 ]]; then + for pid in "${PREFILL_SRUN_PID:-}" "${DECODE_SRUN_PID:-}"; do + [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true + done + fi + for pid in "${PREFILL_SRUN_PID:-}" "${DECODE_SRUN_PID:-}"; do + [[ -n "$pid" ]] || continue + step_rc=0 + wait_owned_step "$pid" || step_rc=$? + [[ "$rc" != 0 ]] || rc=$step_rc + done + if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + for role_rank in 0 1; do + # A failed or missing rank must not suppress the other rank's audit. + srun --jobid="$JOB_ID" --nodelist="${HOSTS[$role_rank]}" --ntasks=1 \ + bash -c 'cp -R "$POWERX_RAW_ROOT/node-$1" "$GITHUB_WORKSPACE/LOGS/native_power/"' bash "$role_rank" & + step_rc=0 + wait_owned_step "$!" || step_rc=$? + [[ "$rc" != 0 ]] || rc=$step_rc + done + fi + exit "$rc" +} +trap finish_tilert_submit EXIT +trap 'exit 143' TERM HUP +trap 'exit 130' INT import_image() { local image_ref="$1" squash_file="$2" host="$3" @@ -43,6 +118,13 @@ mkdir -p "$TILERT_WEIGHTS_DIR" run_role() { local role="$1" host="$2" squash_file="$3" + local rank=0 gpu_count="${DECODE_TP:-8}" + if [[ "$role" == prefill ]]; then rank=1; gpu_count="${PREFILL_TP:-8}"; fi + export POWERX_RANK="$rank" POWERX_GPU_COUNT="$gpu_count" POWERX_NODE_NAME="$host" + export CUDA_VISIBLE_DEVICES="$(seq -s, 0 "$((gpu_count - 1))")" + export POWERX_CLOCK_SYNCHRONIZED + POWERX_CLOCK_SYNCHRONIZED=$(srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \ + bash -c 'timedatectl show -p NTPSynchronized --value 2>/dev/null || echo false') # The tilert image bakes no NVIDIA_VISIBLE_DEVICES (unlike vllm-openai), and # enroot's nvidia hook only injects the driver when it is set — without it the # decode container has no libcuda and torch dies with "Found no NVIDIA driver". @@ -50,10 +132,11 @@ run_role() { # Exported here (not in --export) because the capabilities value contains a # comma, which srun's --export parsing would split on. export NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=compute,utility - srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \ + exec srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \ --container-image="$squash_file" \ - --container-mounts="$GITHUB_WORKSPACE:/workspace,$MODEL_PATH:$MODEL_PATH,$TILERT_WEIGHTS_DIR:$TILERT_WEIGHTS_DIR" \ + --container-mounts="$GITHUB_WORKSPACE:/workspace,$MODEL_PATH:$MODEL_PATH,$TILERT_WEIGHTS_DIR:$TILERT_WEIGHTS_DIR,$POWERX_RAW_ROOT:/powerx_native,$POWERX_CONTROL_ROOT:/powerx_control" \ --container-workdir=/workspace --no-container-entrypoint \ + --container-env=CUDA_VISIBLE_DEVICES,POWERX_NATIVE_ENABLED,POWERX_HOST_UID,POWERX_HOST_GID,POWERX_COLLECTOR_REVISION,POWERX_NODE_NAME,POWERX_CLOCK_SYNCHRONIZED,POWERX_RANK,POWERX_GPU_COUNT \ --export=ALL,TILERT_ROLE="$role",DECODE_HOST="$DECODE_HOST",PREFILL_HOST="$PREFILL_HOST",PORT="${PORT:-8888}" \ bash "/workspace/benchmarks/multi_node/tilert_utils/run_node.sh" } @@ -61,14 +144,10 @@ run_role() { run_role decode "$DECODE_HOST" "$DECODE_SQUASH" & DECODE_SRUN_PID=$! -run_role prefill "$PREFILL_HOST" "$PREFILL_SQUASH" -PREFILL_RC=$? - -for _ in $(seq 1 "${TILERT_DECODE_DRAIN:-60}"); do - kill -0 "$DECODE_SRUN_PID" 2>/dev/null || break - sleep 1 -done -kill -0 "$DECODE_SRUN_PID" 2>/dev/null && { echo "[submit] decode srun did not exit on its own, killing it"; kill "$DECODE_SRUN_PID" 2>/dev/null; } -wait "$DECODE_SRUN_PID" 2>/dev/null || true - +# Both roles run as owned children so a signal interrupts the shell's wait and +# reaches EXIT cleanup immediately; run_role execs srun to preserve that PID. +run_role prefill "$PREFILL_HOST" "$PREFILL_SQUASH" & +PREFILL_SRUN_PID=$! +PREFILL_RC=0 +wait "$PREFILL_SRUN_PID" || PREFILL_RC=$? exit "$PREFILL_RC" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index db0784ef1..e88856d2b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -10273,6 +10273,7 @@ glm5.1-fp8-b200-tilert: - "DECODE_NODES=1" - isl: 8192 osl: 1024 + require-power: true search-space: - spec-decoding: "mtp" conc-list: [1] diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 581e5d09a..328447652 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -114,6 +114,10 @@ The runner-name prefix is load-bearing: workflow routing uses `launch_${RUNNER_N 6. Verify every runner is **Idle** in [repository runner settings](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners) before adding it to sweep traffic. 7. Verify launcher mounts for `_work`, HF cache, staged weights, and squash images from a compute node. Root containers must not leave root-owned files in the shared workspace. +## Native TileRT power + +Only fixed 8192/1024 `glm5.1-fp8-b200-tilert` requires native power. TileRT runs inside its returned `salloc` allocation, retains both role exit codes and drains collectors before staging audits. Exactly one physical node per role is supported. Other sequence lengths, AgentX and eval-only do not enable this collector. Hardware qualification and publication remain pending. + ## Register an srt-slurm recipe Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md). Checked-in recipes: [`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index aeff271a8..6c5a2f0de 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -114,6 +114,10 @@ runner 名称前缀是关键契约：workflow 通过 `launch_${RUNNER_NAME%%_*}. 6. 将 runner 加入 sweep 流量前，在[仓库 runner 设置页](https://github.com/SemiAnalysisAI/InferenceX/settings/actions/runners)确认每个 runner 都是 **Idle**。 7. 从计算节点验证 launcher 对 `_work`、HF cache、预置权重和 squash 镜像的挂载。root 容器不得在共享 workspace 留下 root 所有的文件。 +## TileRT 原生功耗 + +仅固定 8192/1024 的 `glm5.1-fp8-b200-tilert` 要求原生功耗。TileRT 在 `salloc` 返回的分配内运行，保留两个角色的退出码，并在保存审计数据前等待采集器排空。每个角色仅支持一个物理节点。其他序列长度、AgentX 和 eval-only 不启用此采集器。硬件资格验证与发布仍待完成。 + ## 注册 srt-slurm 配方 映射来源：[`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md)。检入的配方：[`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/)。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 10db254b5..405f3578a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7421,3 +7421,13 @@ - "Retain UTC SMI telemetry context and validate native multinode collector artifacts without enabling new recipes." - "保留 UTC SMI 遥测上下文并验证原生多节点采集产物，不启用新 recipe。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - glm5.1-fp8-b200-tilert + scenario-type: + - fixed-seq-len + description: + - Require native power only for TileRT 8k1k; keep native launches inside the owned + allocation and retain both role exit statuses. + - 仅对 TileRT 8k1k 要求原生功耗；原生启动始终使用本任务分配，并保留两个角色退出状态。 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3027 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index b5e0ef627..e86491152 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -40,6 +40,10 @@ if [[ "$IS_MULTINODE" != "true" ]]; then run_compat_launcher fi +if [[ "$FRAMEWORK" == "tilert" && "${IS_AGENTIC:-0}" != "1" ]]; then + run_compat_launcher +fi + if [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then export MODEL_PATH="${MODEL_PATH:-$NSCALE_MODEL_ROOT/DeepSeek-V4-Pro}" export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" diff --git a/runners/test_tilert_power_lifecycle.py b/runners/test_tilert_power_lifecycle.py new file mode 100644 index 000000000..972be6a6a --- /dev/null +++ b/runners/test_tilert_power_lifecycle.py @@ -0,0 +1,122 @@ +"""CPU-only lifecycle checks with controlled external collector/Slurm processes.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +LIFECYCLE = ROOT / 'benchmarks/native_power_lifecycle.sh' +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' + + +@pytest.mark.parametrize(('decode_rc', 'interruption'), [(0, None), (9, None), + (0, 'TERM'), (0, 'HUP'), + (0, 'INT'), (0, 'hang')]) +def test_tilert_submit_keeps_decode_status_and_stages_both_roles(tmp_path, decode_rc, interruption): + repo, bindir = tmp_path / 'repo', tmp_path / 'bin' + for path in (repo, bindir): + path.mkdir() + commands = { + 'salloc':'#!' + sys.executable + '\n' + r''' +import json,os,pathlib,subprocess,sys +args=sys.argv[1:] +try: + result=subprocess.run(args[args.index('env'):],env={**os.environ,'SLURM_JOB_ID':'123','SLURM_JOB_NODELIST':'node-[a-b]'}) +finally: + staged=pathlib.Path(os.environ['GITHUB_WORKSPACE'])/'LOGS/native_power' + pathlib.Path(os.environ['RELEASE_RECEIPT']).write_text(json.dumps( + [json.loads(p.read_text()) for p in sorted(staged.glob('node-*/manifest.json'))])) +sys.exit(result.returncode) +''', + 'squeue':'#!/bin/sh\necho unexpected job-name lookup >&2\nexit 98\n', + 'scontrol':'#!/bin/sh\nprintf "node-a\\nnode-b\\n"\n', + 'scancel':'#!/bin/sh\necho unexpected cancellation >&2\nexit 98\n', + 'git':'#!/bin/sh\necho 0123456789012345678901234567890123456789\n', + 'srun':'#!' + sys.executable + '\n' + r''' +import json, os, pathlib, signal, subprocess, sys, time +args=sys.argv[1:] +if any(a.startswith('--container-image=') for a in args): + rank=os.environ['POWERX_RANK'] + out=pathlib.Path(os.environ['POWERX_RAW_ROOT']) / f'node-{rank}' + out.mkdir(parents=True,exist_ok=True) + mode=os.environ.get('INTERRUPTION') + manifest={'rank':int(rank),'synthetic':True} + def finish(signum, frame): + assert pathlib.Path(os.environ['POWERX_CONTROL_ROOT'],'stop').exists() + manifest['terminated']=True + (out/'manifest.json').write_text(json.dumps(manifest)) + sys.exit(143) + signal.signal(signal.SIGTERM, signal.SIG_IGN if mode=='hang' and rank=='0' else finish) + (out/'manifest.json').write_text(json.dumps(manifest)) + if mode: + if rank=='1': + deadline=time.monotonic()+5 + while not (out.parent/'node-0/manifest.json').exists(): + assert time.monotonic() Date: Sat, 12 Sep 2026 02:30:16 -0700 Subject: [PATCH 06/17] fix: retain per-window SMI context artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留每个测量窗口的 SMI 上下文工件，覆盖原 #3027 的共享上传规则。 --- .github/workflows/benchmark-tmpl.yml | 8 ++++---- utils/test_process_result.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 1da933f63..632401822 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -445,13 +445,13 @@ jobs: name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }} path: | gpu_metrics.csv - gpu_metrics_context.json + gpu_metrics*_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv results/gpu_metrics*.csv - results/gpu_metrics_context.json + results/gpu_metrics*_context.json results/gpu_metrics_identity.json if-no-files-found: ignore @@ -464,14 +464,14 @@ jobs: ${{ env.RESULT_FILENAME }}.json agg_${{ env.RESULT_FILENAME }}.json gpu_metrics.csv - gpu_metrics_context.json + gpu_metrics*_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json results/gpu_metrics*.csv - results/gpu_metrics_context.json + results/gpu_metrics*_context.json results/gpu_metrics_identity.json results/agentic_power_window.json results/agentic_power_timezone_offset.txt diff --git a/utils/test_process_result.py b/utils/test_process_result.py index d14d6a87a..7bc8dc4d9 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -1623,3 +1623,19 @@ def test_public_power_audit_bounds_text_and_device_identifiers(): assert audit['exporter_image_sha256'] == 'a' * 64 assert len(audit['observed_gpu_ids']) == 1024 assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] + + +@pytest.mark.parametrize('step_name', ['Upload GPU metrics', 'Upload power audit bundle']) +def test_workflow_retains_context_for_each_metrics_csv(tmp_path, step_name): + import yaml + + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == step_name) + patterns = step['with']['path'].splitlines() + for relative in ['gpu_metrics_concurrency_4_context.json', + 'results/gpu_metrics_concurrency_4_context.json']: + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('{"timestamp_timezone":"UTC"}') + assert any(path in tmp_path.glob(pattern.strip()) for pattern in patterns) From f68ba33bfc8a2e1aa6d03dd28c679e5b68200dfb Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:35:39 -0700 Subject: [PATCH 07/17] fix: skip TileRT power preparation when disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 未启用功耗时跳过 TileRT 采集目录、设备设置和挂载。 --- benchmarks/multi_node/tilert_utils/submit.sh | 38 ++++++++++++-------- runners/test_tilert_power_lifecycle.py | 21 +++++++---- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/benchmarks/multi_node/tilert_utils/submit.sh b/benchmarks/multi_node/tilert_utils/submit.sh index c76c33112..fd8bc2b7b 100755 --- a/benchmarks/multi_node/tilert_utils/submit.sh +++ b/benchmarks/multi_node/tilert_utils/submit.sh @@ -37,15 +37,21 @@ if [[ "${REQUIRE_POWER:-0}" =~ ^(1|true|TRUE|yes|YES)$ && "$ISL" == 8192 && "$OS export POWERX_NATIVE_ENABLED=1 fi export SLURM_JOB_ID="$JOB_ID" -export POWERX_HOST_UID="$(id -u)" POWERX_HOST_GID="$(id -g)" -export POWERX_COLLECTOR_REVISION="$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" -export POWERX_RAW_ROOT="${POWERX_RAW_ROOT:-/tmp/inferencex-native-$JOB_ID}" -export POWERX_CONTROL_ROOT="$GITHUB_WORKSPACE/LOGS/power_control-$JOB_ID" -mkdir -p "$POWERX_CONTROL_ROOT" "$GITHUB_WORKSPACE/LOGS/native_power" -chmod 777 "$POWERX_CONTROL_ROOT" +POWERX_MOUNTS="" +POWERX_ENV="POWERX_NATIVE_ENABLED" +if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + export POWERX_HOST_UID="$(id -u)" POWERX_HOST_GID="$(id -g)" + export POWERX_COLLECTOR_REVISION="$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" + export POWERX_RAW_ROOT="${POWERX_RAW_ROOT:-/tmp/inferencex-native-$JOB_ID}" + export POWERX_CONTROL_ROOT="$GITHUB_WORKSPACE/LOGS/power_control-$JOB_ID" + mkdir -p "$POWERX_CONTROL_ROOT" "$GITHUB_WORKSPACE/LOGS/native_power" + chmod 777 "$POWERX_CONTROL_ROOT" + srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 mkdir -p "$POWERX_RAW_ROOT" + srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 chmod 777 "$POWERX_RAW_ROOT" + POWERX_MOUNTS=",$POWERX_RAW_ROOT:/powerx_native,$POWERX_CONTROL_ROOT:/powerx_control" + POWERX_ENV+=",CUDA_VISIBLE_DEVICES,POWERX_HOST_UID,POWERX_HOST_GID,POWERX_COLLECTOR_REVISION,POWERX_NODE_NAME,POWERX_CLOCK_SYNCHRONIZED,POWERX_RANK,POWERX_GPU_COUNT" +fi rm -f "${BENCHMARK_LOGS_DIR:-$GITHUB_WORKSPACE}/.tilert_done.$JOB_ID" -srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 mkdir -p "$POWERX_RAW_ROOT" -srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 chmod 777 "$POWERX_RAW_ROOT" # Keep node-local receipts inside the allocation until both serving steps drain. # A caught cancellation exits through the same staging path as normal completion. wait_owned_step() { @@ -120,11 +126,13 @@ run_role() { local role="$1" host="$2" squash_file="$3" local rank=0 gpu_count="${DECODE_TP:-8}" if [[ "$role" == prefill ]]; then rank=1; gpu_count="${PREFILL_TP:-8}"; fi - export POWERX_RANK="$rank" POWERX_GPU_COUNT="$gpu_count" POWERX_NODE_NAME="$host" - export CUDA_VISIBLE_DEVICES="$(seq -s, 0 "$((gpu_count - 1))")" - export POWERX_CLOCK_SYNCHRONIZED - POWERX_CLOCK_SYNCHRONIZED=$(srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \ - bash -c 'timedatectl show -p NTPSynchronized --value 2>/dev/null || echo false') + if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + export POWERX_RANK="$rank" POWERX_GPU_COUNT="$gpu_count" POWERX_NODE_NAME="$host" + export CUDA_VISIBLE_DEVICES="$(seq -s, 0 "$((gpu_count - 1))")" + export POWERX_CLOCK_SYNCHRONIZED + POWERX_CLOCK_SYNCHRONIZED=$(srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \ + bash -c 'timedatectl show -p NTPSynchronized --value 2>/dev/null || echo false') + fi # The tilert image bakes no NVIDIA_VISIBLE_DEVICES (unlike vllm-openai), and # enroot's nvidia hook only injects the driver when it is set — without it the # decode container has no libcuda and torch dies with "Found no NVIDIA driver". @@ -134,9 +142,9 @@ run_role() { export NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=compute,utility exec srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \ --container-image="$squash_file" \ - --container-mounts="$GITHUB_WORKSPACE:/workspace,$MODEL_PATH:$MODEL_PATH,$TILERT_WEIGHTS_DIR:$TILERT_WEIGHTS_DIR,$POWERX_RAW_ROOT:/powerx_native,$POWERX_CONTROL_ROOT:/powerx_control" \ + --container-mounts="$GITHUB_WORKSPACE:/workspace,$MODEL_PATH:$MODEL_PATH,$TILERT_WEIGHTS_DIR:$TILERT_WEIGHTS_DIR$POWERX_MOUNTS" \ --container-workdir=/workspace --no-container-entrypoint \ - --container-env=CUDA_VISIBLE_DEVICES,POWERX_NATIVE_ENABLED,POWERX_HOST_UID,POWERX_HOST_GID,POWERX_COLLECTOR_REVISION,POWERX_NODE_NAME,POWERX_CLOCK_SYNCHRONIZED,POWERX_RANK,POWERX_GPU_COUNT \ + --container-env="$POWERX_ENV" \ --export=ALL,TILERT_ROLE="$role",DECODE_HOST="$DECODE_HOST",PREFILL_HOST="$PREFILL_HOST",PORT="${PORT:-8888}" \ bash "/workspace/benchmarks/multi_node/tilert_utils/run_node.sh" } diff --git a/runners/test_tilert_power_lifecycle.py b/runners/test_tilert_power_lifecycle.py index 972be6a6a..265c51567 100644 --- a/runners/test_tilert_power_lifecycle.py +++ b/runners/test_tilert_power_lifecycle.py @@ -8,14 +8,13 @@ import pytest ROOT = Path(__file__).resolve().parents[1] -LIFECYCLE = ROOT / 'benchmarks/native_power_lifecycle.sh' -JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' -@pytest.mark.parametrize(('decode_rc', 'interruption'), [(0, None), (9, None), - (0, 'TERM'), (0, 'HUP'), - (0, 'INT'), (0, 'hang')]) -def test_tilert_submit_keeps_decode_status_and_stages_both_roles(tmp_path, decode_rc, interruption): +@pytest.mark.parametrize(('decode_rc', 'interruption', 'enabled'), + [(0, None, True), (9, None, True), (0, 'TERM', True), + (0, 'HUP', True), (0, 'INT', True), (0, 'hang', True), + (0, None, False)]) +def test_tilert_submit_keeps_decode_status_and_stages_both_roles(tmp_path, decode_rc, interruption, enabled): repo, bindir = tmp_path / 'repo', tmp_path / 'bin' for path in (repo, bindir): path.mkdir() @@ -39,6 +38,9 @@ def test_tilert_submit_keeps_decode_status_and_stages_both_roles(tmp_path, decod import json, os, pathlib, signal, subprocess, sys, time args=sys.argv[1:] if any(a.startswith('--container-image=') for a in args): + if os.environ['POWERX_NATIVE_ENABLED'] != '1': + assert not any('/powerx_native' in a for a in args) + sys.exit(0) rank=os.environ['POWERX_RANK'] out=pathlib.Path(os.environ['POWERX_RAW_ROOT']) / f'node-{rank}' out.mkdir(parents=True,exist_ok=True) @@ -81,7 +83,7 @@ def finish(signum, frame): 'IMAGE':'synthetic-decode','PREFILL_IMAGE':'synthetic-prefill', 'MODEL_PATH':str(repo),'MODEL_PREFIX':'fixture','PRECISION':'fp8', 'PREFILL_TP':'2','DECODE_TP':'2','SLURM_ACCOUNT':'fixture', - 'SLURM_PARTITION':'fixture','RUNNER_NAME':'fixture','REQUIRE_POWER':'1', 'ISL':'8192','OSL':'1024', + 'SLURM_PARTITION':'fixture','RUNNER_NAME':'fixture','REQUIRE_POWER':'1' if enabled else '0', 'ISL':'8192','OSL':'1024', 'TILERT_WEIGHTS_DIR':str(tmp_path/'weights'),'TILERT_DECODE_DRAIN':'1', 'POWERX_RAW_ROOT':str(tmp_path/'raw'),'DECODE_RC':str(decode_rc), 'RELEASE_RECEIPT':str(tmp_path/'released'), 'INTERRUPTION':interruption or ''} @@ -90,6 +92,11 @@ def finish(signum, frame): expected_rc={'TERM':143,'HUP':143,'INT':130,'hang':7}.get(interruption,decode_rc) assert result.returncode == expected_rc, result.stderr + result.stdout release_evidence=json.loads((tmp_path/'released').read_text()) + if not enabled: + assert release_evidence == [] + assert not (repo / 'LOGS/native_power').exists() + assert not (tmp_path / 'raw').exists() + return assert [item['rank'] for item in release_evidence] == [0,1] if interruption in {'TERM','HUP','INT'}: assert all(item.get('terminated') for item in release_evidence) From b230e7077dc24a58b4773457de911946b4a6bb2f Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:41:07 -0700 Subject: [PATCH 08/17] fix: preserve native role metrics and abort receipts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 独立生成 prefill 与 decode 角色指标，并在 AMD 采集器被回收前完成取消状态记录。 --- benchmarks/benchmark_lib.sh | 2 +- infx/results/power/native_multinode.py | 10 ++-- utils/test_native_multinode_power.py | 76 ++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 9d99a4cc9..ab6699c9f 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -225,7 +225,7 @@ stop_gpu_monitor() { # tick in the same second as the window end still fails bracketing — # the stream needs a tick at the NEXT whole second (measured on MI355X: # end=...153.325 vs last sample ...153.0). - if [[ "$GPU_MONITOR_VENDOR" == "amd" ]]; then + if [[ "$GPU_MONITOR_VENDOR" == "amd" && "${AMD_MONITOR_STOP_TIMEOUT_S:-}" != "0" ]]; then sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 )) fi kill "$GPU_MONITOR_PID" 2>/dev/null diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py index 1ffae5bd9..86b5d683b 100644 --- a/infx/results/power/native_multinode.py +++ b/infx/results/power/native_multinode.py @@ -221,13 +221,15 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, metrics = {} if not reasons and integration is not None and benchmark is not None: metrics = _derived_metrics(integration, benchmark) - if expected_decode_gpus: + if roles["prefill"]: prefill = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["prefill"]) - decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) - metrics.update(prefill_gpu_energy_j=prefill, decode_gpu_energy_j=decode, + metrics.update(prefill_gpu_energy_j=prefill, prefill_avg_power_w=prefill / benchmark.integration_duration_s / expected_prefill_gpus, + prefill_joules_per_input_token=prefill / benchmark.total_input_tokens) + if roles["decode"]: + decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) + metrics.update(decode_gpu_energy_j=decode, decode_avg_power_w=decode / benchmark.integration_duration_s / expected_decode_gpus, - prefill_joules_per_input_token=prefill / benchmark.total_input_tokens, decode_joules_per_output_token=decode / benchmark.total_output_tokens) valid = not reasons try: diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py index 0aeee2511..c2945cb75 100644 --- a/utils/test_native_multinode_power.py +++ b/utils/test_native_multinode_power.py @@ -199,3 +199,79 @@ def test_result_processor_discovers_staged_native_package(tmp_path, monkeypatch) invalid = json.loads(agg.read_text()) assert invalid["power_valid"] == 0 assert "total_gpu_energy_j" not in invalid + + +@pytest.mark.parametrize('role', ['prefill', 'decode']) +def test_native_single_role_preserves_whole_fleet_and_role_metrics(tmp_path, role): + root, bench, agg = _package(tmp_path) + for path in root.glob('*/manifest.json'): + manifest = json.loads(path.read_text()) + manifest['role'] = role + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=2 if role == 'prefill' else 0, + expected_decode_gpus=2 if role == 'decode' else 0, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual['total_gpu_energy_j'] == 800 + assert actual[f'{role}_gpu_energy_j'] == 800 + assert actual[f'{role}_avg_power_w'] == 200 + opposite = 'decode' if role == 'prefill' else 'prefill' + assert f'{opposite}_gpu_energy_j' not in actual + assert json.loads((tmp_path / 'power_validation_result.json').read_text())['power_valid'] + + +def test_native_amd_abort_skips_legacy_tail_wait(tmp_path): + result = subprocess.run(['bash', '-c', '''source "$1" +kill() { return 0; }; wait() { return 0; } +sleep() { echo unexpected-tail-wait >&2; } +_write_amd_smi_sidecar() { return 0; } +GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd AMD_MONITOR_STOP_TIMEOUT_S=0 +GPU_METRICS_CSV="$2/missing.csv" +stop_gpu_monitor +''', 'bash', str(REPO / 'benchmarks/benchmark_lib.sh'), str(tmp_path)], + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert 'unexpected-tail-wait' not in result.stderr + + +def test_native_amd_abort_publishes_receipt_before_reaper_deadline(tmp_path): + binary = tmp_path / 'bin' + binary.mkdir() + (binary / 'python3').symlink_to(sys.executable) + fake = binary / 'amd-smi' + fake.write_text(f'''#!{sys.executable} +import sys, time +if "-w" in sys.argv: + print("timestamp,gpu,socket_power", flush=True) + while True: + print(str(int(time.time())) + ",0,100", flush=True) + time.sleep(0.1) +else: + print("[]") +''') + fake.chmod(0o755) + control = tmp_path / 'control' + control.mkdir() + node = tmp_path / 'node-0' + process = subprocess.Popen(['bash', '-c', '\n'.join([ + 'source "$1"', + 'bash "$4" "$2" "$3" amd 0 aggregate 0 1 &', + 'POWERX_COLLECTOR_PID=$! POWERX_CONTROL_DIR=$3 POWERX_NUM_NODES=1', + 'POWERX_BARRIER_TIMEOUT_S=5', + 'powerx_wait_collectors ready || exit 1', + 'POWERX_BARRIER_TIMEOUT_S=0', + 'powerx_reap_collector', + ]), 'bash', str(REPO / 'benchmarks/native_power_lifecycle.sh'), str(node), str(control), str(REPO / 'benchmarks/native_power_collect.sh')], + env={**os.environ, 'PATH': f'{binary}:/usr/bin:/bin', 'SLURM_JOB_ID': 'test-job'}, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) + try: + stdout, stderr = process.communicate(timeout=8) + assert process.returncode == 143, (stdout, stderr) + assert (control / 'done-0').read_text().strip() == '143' + assert json.loads((node / 'manifest.json').read_text())['lifecycle'] == 'failed' + finally: + import signal + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() From 880f1153b489da0965261ededb090932d5a374dd Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:43:51 -0700 Subject: [PATCH 09/17] ci: run native collector contract regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在现有 CPU 工作流中执行原生采集契约回归，并跟踪对应文件变更。 --- .github/workflows/test-process-result.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 4d4cf0adf..7d5071770 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -10,6 +10,11 @@ on: - '.github/workflows/e2e-tests.yml' - '.github/workflows/test-process-result.yml' - 'benchmarks/benchmark_lib.sh' + - 'benchmarks/native_power_collect.sh' + - 'benchmarks/native_power_lifecycle.sh' + - 'runners/test_native_collector_barriers.py' + - 'runners/test_native_collector_receipts.py' + - 'utils/test_native_multinode_power.py' - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/**/*.yaml' - 'benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/**/*.yaml' - 'benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/**/*.yaml' @@ -57,4 +62,4 @@ jobs: run: | cd utils uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \ - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py -v From 69eb721dfe4c61304ec5984d68dae8908169592c Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:45:28 -0700 Subject: [PATCH 10/17] ci: run fixed-sequence power regression tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将新增功耗回归测试与相关路径接入现有 CPU 工作流。 --- .github/workflows/test-process-result.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 4d4cf0adf..2be5590ac 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -36,6 +36,9 @@ on: - 'utils/test_gb300_power_official_contract.py' - 'utils/test_inject_srt_power_concurrencies.py' - 'utils/test_process_result.py' + - 'benchmarks/multi_node/tilert_utils/**' + - 'benchmarks/multi_node/glm5.1_fp8_b200_tilert-disagg.sh' + - 'runners/test_tilert_power_lifecycle.py' permissions: contents: read @@ -57,4 +60,4 @@ jobs: run: | cd utils uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \ - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py ../runners/test_tilert_power_lifecycle.py -v From 671b0c1e534787749ebc7e7c56070953a963b569 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:48:07 -0700 Subject: [PATCH 11/17] fix: honor accepted power requirement field names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 矩阵生成同时识别校验允许的字段名与别名，避免静默丢失功耗门槛。 --- infx/matrix/generate.py | 2 +- utils/matrix_logic/test_generate_sweep_configs.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/infx/matrix/generate.py b/infx/matrix/generate.py index ee6166045..abadad2ce 100644 --- a/infx/matrix/generate.py +++ b/infx/matrix/generate.py @@ -809,7 +809,7 @@ def _fixed_sequence_entries( is_multinode = config.get(Fields.MULTINODE.value, False) disagg = config.get(Fields.DISAGG.value, False) isl, osl = sequence[Fields.ISL.value], sequence[Fields.OSL.value] - require_power = sequence.get(Fields.REQUIRE_POWER.value, False) + require_power = sequence.get(Fields.REQUIRE_POWER.value, sequence.get("require_power", False)) if require_power and (isl, osl) != (8192, 1024): raise ValueError("require-power rollout supports only fixed-sequence 8192/1024") model_code = config[Fields.MODEL_PREFIX.value] diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index eeae0b83d..d88db4e9e 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -3244,7 +3244,8 @@ def test_empty_matrix_has_no_jobs(self, split_e2e_configs): @pytest.mark.parametrize("multinode", [False, True]) -def test_require_power_is_scoped_to_one_fixed_sequence(multinode, sample_single_node_config, +@pytest.mark.parametrize("power_key", ["require-power", "require_power"]) +def test_require_power_is_scoped_to_one_fixed_sequence(multinode, power_key, sample_single_node_config, sample_multinode_config, sample_runner_config): from infx.matrix.generate import expand_full_sweep, select_matrix_evals from infx.matrix.validation import MultiNodeSeqLenConfig, SingleNodeSeqLenConfig @@ -3257,7 +3258,7 @@ def test_require_power_is_scoped_to_one_fixed_sequence(multinode, sample_single_ sequences[-1]["isl"] = 8192 before = expand_full_sweep(config, sample_runner_config) assert all("require-power" not in row for row in before) - sequences[-1]["require-power"] = True + sequences[-1][power_key] = True schema = MultiNodeSeqLenConfig if multinode else SingleNodeSeqLenConfig schema.model_validate(sequences[-1]) after = expand_full_sweep(config, sample_runner_config) @@ -3267,6 +3268,6 @@ def test_require_power_is_scoped_to_one_fixed_sequence(multinode, sample_single_ evals = select_matrix_evals(copy.deepcopy(after), mode="subset") assert evals assert all("require-power" not in row for row in evals) - sequences[0]["require-power"] = True + sequences[0][power_key] = True with pytest.raises(ValueError, match="only fixed-sequence 8192/1024"): expand_full_sweep(config, sample_runner_config) From 654c33b2f71584f81a4c11a65e97a3dbd0fc41a9 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:48:30 -0700 Subject: [PATCH 12/17] test: synchronize shared replay signal readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将已验证的信号就绪屏障移至共享原生采集前置，避免重放 exec 前的竞态。 --- utils/agentic/aggregation/test_power_lifecycle.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/utils/agentic/aggregation/test_power_lifecycle.py b/utils/agentic/aggregation/test_power_lifecycle.py index 20710e68e..6d955af12 100644 --- a/utils/agentic/aggregation/test_power_lifecycle.py +++ b/utils/agentic/aggregation/test_power_lifecycle.py @@ -280,8 +280,13 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( }} stop_gpu_monitor() {{ printf 'monitor-stop\n' >> {str(event_log)!r}; }} fake_replay() {{ - printf 'replay-ready\n' >> {str(event_log)!r} - exec sleep 30 + exec {sys.executable!r} -c ' +import signal, sys, time +signal.signal(signal.SIGINT, signal.SIG_DFL) +signal.signal(signal.SIGTERM, signal.SIG_DFL) +print("replay-ready", file=open(sys.argv[1], "a"), flush=True) +time.sleep(30) +' {str(event_log)!r} }} trap 'printf "parent-exit\\n" >> {str(event_log)!r}' EXIT trap 'printf "parent-int\\n" >> {str(event_log)!r}; exit 130' INT @@ -301,7 +306,8 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( ) try: # The monitor starts before the production signal traps are installed. - # Wait for replay so the signal actually exercises those traps. + # Publish readiness from the execed process after restoring signal handling; + # a shell marker before exec races with the group SIGINT. deadline = time.monotonic() + 5 while time.monotonic() < deadline: if event_log.exists() and "replay-ready" in event_log.read_text().splitlines(): From d141e71c8be2bb6c57c37a2a623bfb6c274e5921 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:02:16 -0700 Subject: [PATCH 13/17] fix: preserve diagnostic sidecars and legacy result processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 只忽略已定义的诊断 sidecar，保留损坏与缺失点门禁；旧处理器逐点处理并累计失败，新版导入错误不回退。 --- .../workflows/benchmark-multinode-tmpl.yml | 42 ++++++++- infx/results/fixed_sequence.py | 9 +- utils/test_process_result.py | 87 +++++++++++++++++++ 3 files changed, 136 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 80b0842f8..d26d4e0bb 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -480,7 +480,47 @@ jobs: export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA" fi - python3 utils/process_result.py --all + # Older measured refs predate the batch CLI. Import failures in a + # present processor are errors, not a reason to downgrade validation. + has_batch=$(python3 - <<'PY' + import importlib + from pathlib import Path + if Path('infx/results/fixed_sequence.py').is_file(): + module = importlib.import_module('infx.results.fixed_sequence') + print(int(hasattr(module, 'process_multinode_results'))) + else: + print(0) + PY + ) + if [ "$has_batch" = 1 ]; then + python3 utils/process_result.py --all + else + batch_rc=0 + observed=() + for result_file in "${RESULT_FILENAME}"_*.json; do + [ -f "$result_file" ] || continue + case "$result_file" in + *.pytorch.json|"${RESULT_FILENAME}_gpu_metrics_context.json"|"${RESULT_FILENAME}_gpu_metrics_identity.json") continue ;; + esac + if [[ "$result_file" =~ _(c|conc|concurrency_)([0-9]+)(_req_rate_[^_]+)?_gpus_([0-9]+)(_ctx_([0-9]+)_gen_([0-9]+))?\.json$ ]]; then + observed+=("${BASH_REMATCH[2]}") + gpus=${BASH_REMATCH[4]} + prefill_gpus=${BASH_REMATCH[6]:-$gpus} + decode_gpus=${BASH_REMATCH[7]:-0} + RESULT_FILENAME=${result_file%.json} IS_MULTINODE=true PREFILL_GPUS="$prefill_gpus" DECODE_GPUS="$decode_gpus" \ + python3 utils/process_result.py || batch_rc=1 + else + echo "Unrecognized result filename: $result_file" >&2 + batch_rc=1 + fi + done + read -r -a expected <<< "$CONC_LIST" + if [[ "$(printf '%s\n' "${observed[@]}" | sort -n)" != "$(printf '%s\n' "${expected[@]}" | sort -n)" ]]; then + echo 'Legacy result batch has missing, duplicate or unexpected concurrencies' >&2 + batch_rc=1 + fi + exit "$batch_rc" + fi - name: Upload result if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index f6fa76102..3db53a3b7 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -318,9 +318,16 @@ def process_multinode_results(env: Mapping[str, str]) -> int: if not expected or min(expected) <= 0: raise ValueError('CONC_LIST must contain positive concurrencies') points: list[dict[str, Any]] = [] + ignored_sidecars: list[str] = [] observed: set[int] = set() status = 0 for path in sorted(Path('.').glob(f"{env['RESULT_FILENAME']}_*.json")): + if path.name.endswith('.pytorch.json') or path.name in { + f"{env['RESULT_FILENAME']}_gpu_metrics_context.json", + f"{env['RESULT_FILENAME']}_gpu_metrics_identity.json", + }: + ignored_sidecars.append(path.name) + continue point: dict[str, Any] = {'source': path.name} try: match = re.search( @@ -364,7 +371,7 @@ def process_multinode_results(env: Mapping[str, str]) -> int: status = max(status, int(bool(missing or unexpected))) summary = {'expected_concurrencies': sorted(expected), 'missing_concurrencies': missing, 'unexpected_concurrencies': unexpected, - 'points': points, 'exit_code': status} + 'points': points, 'ignored_sidecars': ignored_sidecars, 'exit_code': status} Path(f"result_processing_{env['RESULT_FILENAME']}.json").write_text(json.dumps(summary, indent=2)) if missing or unexpected: print(f'[process_result] incomplete sweep: missing={missing}, unexpected={unexpected}', file=sys.stderr) diff --git a/utils/test_process_result.py b/utils/test_process_result.py index d14d6a87a..2b394e3bb 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -1623,3 +1623,90 @@ def test_public_power_audit_bounds_text_and_device_identifiers(): assert audit['exporter_image_sha256'] == 'a' * 64 assert len(audit['observed_gpu_ids']) == 1024 assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] + + +@pytest.mark.parametrize('sidecar', ['run_recipe_conc4_gpus_4_ctx_2_gen_2.pytorch.json', + 'run_gpu_metrics_context.json', 'run_gpu_metrics_identity.json']) +@pytest.mark.parametrize('point_state', ['valid', 'missing', 'malformed']) +def test_multinode_batch_retains_sidecars_without_counting_them_as_points( + tmp_path, multinode_env_vars, sample_benchmark_result, sidecar, point_state, +): + (tmp_path / sidecar).write_text('{"diagnostic": true}') + if point_state != 'missing': + (tmp_path / 'run_recipe_conc4_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': 4}) + if point_state == 'valid' else '{broken') + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4', + 'PYTHONPATH': str(REPO_ROOT)} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, env=env, + capture_output=True, text=True) + assert result.returncode == int(point_state != 'valid'), result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['ignored_sidecars'] == [sidecar] + assert receipt['missing_concurrencies'] == ([] if point_state == 'valid' else [4]) + assert (tmp_path / sidecar).read_text() == '{"diagnostic": true}' + + +def test_multinode_batch_rejects_unknown_point_filename( + tmp_path, multinode_env_vars, sample_benchmark_result, +): + for name in ['run_conc4_gpus_4_ctx_2_gen_2.json', 'run_conc16_gpus_bad.json']: + (tmp_path / name).write_text(json.dumps({**sample_benchmark_result, 'max_concurrency': 4})) + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4 16', 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1 + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [16] + assert any('filename lacks' in point.get('error', '') for point in receipt['points']) + + +@pytest.mark.parametrize('first_rc', [0, 7]) +@pytest.mark.parametrize('expected_concs', ['4 16', '4 8 16', '4']) +def test_multinode_workflow_processes_every_point_with_legacy_processor(tmp_path, first_rc, expected_concs): + (tmp_path / 'utils').mkdir() + (tmp_path / 'utils/process_result.py').write_text('''import json, os, sys +from pathlib import Path +stem = os.environ['RESULT_FILENAME'] +data = json.loads(Path(stem + '.json').read_text()) +Path('agg_' + stem + '.json').write_text(json.dumps(data)) +sys.exit(data['exit_code']) +''') + for conc, rc in [(4, first_rc), (16, 0)]: + (tmp_path / f'run_conc{conc}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({'exit_code': rc})) + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == 'Process result') + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step['run']], cwd=tmp_path, + env={**os.environ, 'RESULT_FILENAME': 'run', 'POWER_PRODUCER_SHA': '', + 'CONC_LIST': expected_concs, 'PYTHONPATH': '', + 'PATH': f"{Path(sys.executable).parent}:{os.environ['PATH']}"}, + capture_output=True, text=True, timeout=10) + assert result.returncode == int(bool(first_rc) or expected_concs != '4 16'), result.stderr + for conc in [4, 16]: + assert (tmp_path / f'agg_run_conc{conc}_gpus_4_ctx_2_gen_2.json').is_file() + + +def test_multinode_workflow_does_not_downgrade_processor_import_errors(tmp_path): + (tmp_path / 'infx/results').mkdir(parents=True) + (tmp_path / 'infx/__init__.py').touch() + (tmp_path / 'infx/results/__init__.py').touch() + (tmp_path / 'infx/results/fixed_sequence.py').write_text( + 'raise RuntimeError("broken processor import")\n') + (tmp_path / 'utils').mkdir() + (tmp_path / 'utils/process_result.py').write_text( + 'from pathlib import Path\nPath("legacy-called").touch()\n') + (tmp_path / 'run_conc4_gpus_4.json').write_text('{}') + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == 'Process result') + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step['run']], cwd=tmp_path, + env={**os.environ, 'RESULT_FILENAME': 'run', 'POWER_PRODUCER_SHA': '', + 'CONC_LIST': '4', 'PYTHONPATH': '', + 'PATH': f"{Path(sys.executable).parent}:{os.environ['PATH']}"}, + capture_output=True, text=True, timeout=10) + assert result.returncode != 0 + assert 'broken processor import' in result.stderr + assert not (tmp_path / 'legacy-called').exists() From 8408779c22817e681238ac323f7b4c4ad34977b0 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:02:55 -0700 Subject: [PATCH 14/17] test: preserve existing result test section formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 恢复合并时意外改动的测试注释分隔线与空行。 --- utils/test_process_result.py | 42 ++++++++++++------------------------ 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 350856ea7..9d3ba8ddf 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -155,11 +155,9 @@ def test_fixed_topology_rejects_empty_parallelism( build_result(sample_benchmark_result, {**single_node_env_vars, name: ""}) -# ====================================================================== - +# ============================================================================= # Test Fixtures - Based on real benchmark output structure -# ====================================================================== - +# ============================================================================= @pytest.fixture def sample_benchmark_result(): @@ -298,11 +296,9 @@ def failing_import(name, *args, **kwargs): ) -# ====================================================================== - +# ============================================================================= # Test script execution via subprocess -# ====================================================================== - +# ============================================================================= class TestProcessResultScript: """Tests for process_result.py script execution.""" @@ -532,11 +528,9 @@ def test_missing_result_file(self, tmp_path, single_node_env_vars): assert result.returncode != 0 -# ====================================================================== - +# ============================================================================= # Test latency and throughput calculations -# ====================================================================== - +# ============================================================================= class TestCalculations: """Tests for throughput and latency calculations.""" @@ -665,11 +659,9 @@ def test_multinode_zero_total_gpus_fails(self, tmp_path, sample_benchmark_result assert "Multinode results require at least one GPU" in result.stderr -# ====================================================================== - +# ============================================================================= # Test output file generation -# ====================================================================== - +# ============================================================================= class TestOutputFile: """Tests for output file generation.""" @@ -698,11 +690,9 @@ def test_output_file_has_correct_prefix(self, tmp_path, sample_benchmark_result, assert output_file.exists() -# ====================================================================== - +# ============================================================================= # Test edge cases -# ====================================================================== - +# ============================================================================= class TestEdgeCases: """Tests for edge cases and special scenarios.""" @@ -751,11 +741,9 @@ def test_integer_conversion(self, tmp_path, single_node_env_vars): assert isinstance(output_data["isl"], int) assert isinstance(output_data["osl"], int) -# ====================================================================== - +# ============================================================================= # Integration: power aggregation patches the agg JSON -# ====================================================================== - +# ============================================================================= class TestPowerAggregationIntegration: """End-to-end wiring: process_result.py invokes aggregate_power.py and @@ -1335,11 +1323,9 @@ def test_start_stop_gpu_monitor_amd_lifecycle(self, tmp_path): assert identity == {"gpu_data": []} -# ====================================================================== - +# ============================================================================= # Integration: multinode power aggregation patches the agg JSON -# ====================================================================== - +# ============================================================================= class TestMultinodePower: From 6ed8f8b5ed69e71fdd6972ceaeaa8d97a160f347 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:05:34 -0700 Subject: [PATCH 15/17] fix: validate native role counts and retain boundary sample audits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 独立核对每种角色的 GPU 数量，并记录窗口外异常样本，窗口内异常仍阻止功耗有效性。 --- infx/results/power/native_multinode.py | 7 +++--- utils/test_native_multinode_power.py | 30 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py index 86b5d683b..78ff89889 100644 --- a/infx/results/power/native_multinode.py +++ b/infx/results/power/native_multinode.py @@ -202,10 +202,8 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, if expected_aggregate_gpus: if roles["prefill"] or roles["decode"] or len(roles["aggregate"]) != expected_aggregate_gpus: reasons.append("native_role_gpu_count_mismatch") - elif expected_decode_gpus: - if roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or len(roles["decode"]) != expected_decode_gpus: - reasons.append("native_role_gpu_count_mismatch") - elif roles["decode"] or len(roles["aggregate"] + roles["prefill"]) != expected_prefill_gpus: + elif (roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or + len(roles["decode"]) != expected_decode_gpus): reasons.append("native_role_gpu_count_mismatch") combined_path = validation_result.with_name(f"{validation_result.stem}_native.csv") combined_path.parent.mkdir(parents=True, exist_ok=True) @@ -247,6 +245,7 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, "observed_gpu_count": integration.observed_num_gpus if integration else 0, "per_gpu_role": {uuid: role for role, uuids in roles.items() for uuid in uuids}, "per_gpu_sample_counts": integration.per_gpu_sample_counts if integration else {}, + "boundary_degenerate_rows": integration.boundary_degenerate_rows if integration else {}, "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s if integration else {}, "producer": {"name": "inferencex-native-smi", "revisions": sorted(revisions), "producer_git_commit": next(iter(revisions)) if len(revisions) == 1 else None}, diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py index c2945cb75..e4234facc 100644 --- a/utils/test_native_multinode_power.py +++ b/utils/test_native_multinode_power.py @@ -275,3 +275,33 @@ def test_native_amd_abort_publishes_receipt_before_reaper_deadline(tmp_path): except ProcessLookupError: pass process.communicate() + + +def test_native_prefill_only_rejects_aggregate_role_devices(tmp_path): + root, bench, agg = _package(tmp_path) + manifest_path = root / 'node-1/manifest.json' + manifest = json.loads(manifest_path.read_text()) + manifest['role'] = 'aggregate' + manifest_path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=2, expected_decode_gpus=0, + require_power=True) == 1 + audit = json.loads((tmp_path / 'power_validation_result.json').read_text()) + assert 'native_role_gpu_count_mismatch' in audit['reasons'] + assert json.loads(agg.read_text())['power_valid'] == 0 + assert 'prefill_avg_power_w' not in json.loads(agg.read_text()) + + +@pytest.mark.parametrize('tick', [2, 4]) +def test_native_audit_retains_boundary_noise_without_relaxing_in_window_errors(tmp_path, tick): + root, bench, agg = _package(tmp_path) + for rank in [0, 1]: + with (root / f'node-{rank}/gpu_metrics.csv').open('a') as stream: + stream.write(f'{tick},0,N/A\n') + if rank == 0: + stream.write(f'{tick},0,0\n') + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, + require_power=True) == int(tick == 2) + audit = json.loads((tmp_path / 'power_validation_result.json').read_text()) + assert audit['boundary_degenerate_rows'] == ({'uuid-0': 2, 'uuid-1': 1} if tick == 4 else {}) + if tick == 4: + assert json.loads(agg.read_text())['total_gpu_energy_j'] == 800 From 85384dd3ba00223513ec5ef65a2f27b422b8c7ae Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:32:14 -0700 Subject: [PATCH 16/17] fix: validate native collector readiness and identity completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 发布就绪收据前检查采集进程仍存活，并将 AMD 结束身份采集失败写入失败收据。补充真实进程与清单回归测试。 --- benchmarks/native_power_collect.sh | 3 +- runners/test_native_collector_receipts.py | 77 +++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/benchmarks/native_power_collect.sh b/benchmarks/native_power_collect.sh index 69df07951..445d00ac3 100644 --- a/benchmarks/native_power_collect.sh +++ b/benchmarks/native_power_collect.sh @@ -35,7 +35,7 @@ finish() { if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi stop_gpu_monitor if [[ "$vendor" == amd ]]; then - _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json + amd-smi list --json > "$power_dir/gpu_metrics_devices_end.json" || collector_rc=1 else nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \ > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1 @@ -65,6 +65,7 @@ start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 if [[ "$vendor" == amd ]]; then _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json fi +_background_process_is_running "$GPU_MONITOR_PID" || exit 1 write_control "ready-$rank" ready while [[ ! -f "$control_dir/stop" ]]; do _background_process_is_running "$GPU_MONITOR_PID" || exit 1 diff --git a/runners/test_native_collector_receipts.py b/runners/test_native_collector_receipts.py index b981f57ce..63d9de677 100644 --- a/runners/test_native_collector_receipts.py +++ b/runners/test_native_collector_receipts.py @@ -14,3 +14,80 @@ def test_native_control_receipt_is_owned_before_publication(tmp_path): assert result.returncode == 0, result.stderr assert (tmp_path / 'done-0').read_text() == '7\n' assert not list(tmp_path.glob('*.tmp')) + + +def _collector_with_monitor(tmp_path, *, alive, end_identity_rc=0): + import shutil + import sys + + scripts = tmp_path / 'repo/benchmarks' + scripts.mkdir(parents=True) + shutil.copyfile(ROOT / 'benchmarks/native_power_collect.sh', scripts / 'native_power_collect.sh') + library = (ROOT / 'benchmarks/benchmark_lib.sh').read_text() + functions = [] + for name in ['_background_process_is_running', '_write_amd_smi_sidecar']: + start = library.index(name + '() {') + functions.append(library[start:library.index('\n}', start) + 2]) + (scripts / 'benchmark_lib.sh').write_text('\n'.join(functions) + ''' +GPU_MONITOR_PID="" GPU_MONITOR_VENDOR="" +start_gpu_monitor() { + GPU_MONITOR_VENDOR=amd + if [[ "$MONITOR_ALIVE" == 1 ]]; then + sleep 30 & + GPU_MONITOR_PID=$! + else + false & + GPU_MONITOR_PID=$! + wait "$GPU_MONITOR_PID" || true + fi +} +stop_gpu_monitor() { + kill "$GPU_MONITOR_PID" 2>/dev/null || true + wait "$GPU_MONITOR_PID" 2>/dev/null || true +} +amd-smi() { + if [[ -f "$IDENTITY_CALLED" && "$END_IDENTITY_RC" != 0 ]]; then return "$END_IDENTITY_RC"; fi + touch "$IDENTITY_CALLED" + printf '[{"gpu":0,"uuid":"test-gpu"}]\\n' +} +''') + control = tmp_path / 'control' + control.mkdir() + (control / 'stop').touch() + power = tmp_path / 'power' + result = subprocess.run(['bash', str(scripts / 'native_power_collect.sh'), str(power), + str(control), 'amd', '0', 'prefill', '0', '1'], + env={'PATH': f'{Path(sys.executable).parent}:/usr/bin:/bin', + 'PYTHONPATH': str(ROOT), 'MONITOR_ALIVE': str(int(alive)), + 'END_IDENTITY_RC': str(end_identity_rc), + 'IDENTITY_CALLED': str(tmp_path / 'identity-called')}, + capture_output=True, text=True, timeout=5) + return result, control, power + + +def test_dead_monitor_never_publishes_ready(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=False) + assert not (control / 'ready-0').exists(), result.stderr + assert (control / 'done-0').read_text().strip() != '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'failed' + + +def test_amd_end_identity_failure_is_not_a_successful_done_receipt(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=True, end_identity_rc=7) + assert (control / 'ready-0').exists(), result.stderr + assert (control / 'done-0').read_text().strip() != '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'failed' + + +def test_live_monitor_with_successful_identity_completes(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=True) + assert result.returncode == 0, result.stderr + assert (control / 'ready-0').exists() + assert (control / 'done-0').read_text().strip() == '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'complete' From 358cbc8328903f2084a36e9f6fffab9eacb64eef Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 19:02:34 -0700 Subject: [PATCH 17/17] fix: mount prepared GLM snapshots for TileRT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留显式 GLM-5.1 模型路径，并挂载完整 HF 缓存以解析 snapshot 软链接；复用已通过的 11 项生命周期回归测试。 --- benchmarks/multi_node/tilert_utils/submit.sh | 7 ++++++- docs/configuration-procedures.md | 2 ++ docs/configuration-procedures_zh.md | 2 ++ perf-changelog.yaml | 7 +++++++ runners/launch_b200-nscale-compat.sh | 2 +- runners/test_tilert_power_lifecycle.py | 21 ++++++++++++++++++++ 6 files changed, 39 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/tilert_utils/submit.sh b/benchmarks/multi_node/tilert_utils/submit.sh index 9f47869f6..e29ae1398 100755 --- a/benchmarks/multi_node/tilert_utils/submit.sh +++ b/benchmarks/multi_node/tilert_utils/submit.sh @@ -19,6 +19,11 @@ DECODE_IMAGE="${DECODE_IMAGE:-$IMAGE}" squash_path() { echo "$SQUASH_DIR/$(echo "$1" | sed 's/[\/:@#]/_/g').sqsh"; } DECODE_SQUASH="$(squash_path "$DECODE_IMAGE")" PREFILL_SQUASH="$(squash_path "$PREFILL_IMAGE")" +MODEL_MOUNTS="$MODEL_PATH:$MODEL_PATH" +if [[ -n "${HF_HUB_CACHE_HOST_PATH:-}" ]]; then + # HF snapshots link to sibling blobs outside the snapshot directory. + MODEL_MOUNTS="$HF_HUB_CACHE_HOST_PATH:$HF_HUB_CACHE_HOST_PATH,$MODEL_MOUNTS" +fi if [[ "${TILERT_IN_ALLOCATION:-0}" != 1 ]]; then # Run inside the allocation returned by this request. Looking up a runner @@ -143,7 +148,7 @@ run_role() { export NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=compute,utility exec srun --jobid="$JOB_ID" --nodelist="$host" --ntasks=1 \ --container-image="$squash_file" \ - --container-mounts="$GITHUB_WORKSPACE:/workspace,$MODEL_PATH:$MODEL_PATH,$TILERT_WEIGHTS_DIR:$TILERT_WEIGHTS_DIR$POWERX_MOUNTS" \ + --container-mounts="$GITHUB_WORKSPACE:/workspace,$MODEL_MOUNTS,$TILERT_WEIGHTS_DIR:$TILERT_WEIGHTS_DIR$POWERX_MOUNTS" \ --container-workdir=/workspace --no-container-entrypoint \ --container-env="$POWERX_ENV" \ --export=ALL,TILERT_ROLE="$role",DECODE_HOST="$DECODE_HOST",PREFILL_HOST="$PREFILL_HOST",PORT="${PORT:-8888}" \ diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 328447652..21475c30a 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -116,6 +116,8 @@ The runner-name prefix is load-bearing: workflow routing uses `launch_${RUNNER_N ## Native TileRT power +For GLM-5.1 on B200 Nscale, `MODEL_PATH` can select an existing shared checkpoint instead of the default `/scratch/models/GLM-5.1-FP8`. When it selects an HF snapshot, also set `HF_HUB_CACHE_HOST_PATH` to the existing cache root; TileRT mounts that root at the same absolute path so snapshot links to sibling blobs remain readable. Keep `TILERT_WEIGHTS_DIR` pointed at the separately converted decode weights. + Only fixed 8192/1024 `glm5.1-fp8-b200-tilert` requires native power. TileRT runs inside its returned `salloc` allocation, retains both role exit codes and drains collectors before staging audits. Exactly one physical node per role is supported. Other sequence lengths, AgentX and eval-only do not enable this collector. Hardware qualification and publication remain pending. ## Register an srt-slurm recipe diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 6c5a2f0de..638807434 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -116,6 +116,8 @@ runner 名称前缀是关键契约：workflow 通过 `launch_${RUNNER_NAME%%_*}. ## TileRT 原生功耗 +B200 Nscale 的 GLM-5.1 可用 `MODEL_PATH` 指定已有共享权重，覆盖默认的 `/scratch/models/GLM-5.1-FP8`。若指定 HF snapshot，还需把 `HF_HUB_CACHE_HOST_PATH` 设为现有缓存根目录；TileRT 按相同绝对路径挂载整个缓存，使 snapshot 指向同级 blobs 的软链接可读。`TILERT_WEIGHTS_DIR` 仍指向单独转换的 decode 权重。 + 仅固定 8192/1024 的 `glm5.1-fp8-b200-tilert` 要求原生功耗。TileRT 在 `salloc` 返回的分配内运行，保留两个角色的退出码，并在保存审计数据前等待采集器排空。每个角色仅支持一个物理节点。其他序列长度、AgentX 和 eval-only 不启用此采集器。硬件资格验证与发布仍待完成。 ## 注册 srt-slurm 配方 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 45c984e2f..5423fd531 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7438,3 +7438,10 @@ allocation and retain both role exit statuses. - 仅对 TileRT 8k1k 要求原生功耗；原生启动始终使用本任务分配，并保留两个角色退出状态。 pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3057 + +- config-keys: + - glm5.1-fp8-b200-tilert + description: + - "Preserve an explicit prepared GLM-5.1 model path and mount shared HF cache blobs in native TileRT containers." + - "保留显式指定的 GLM-5.1 预备模型路径，并在原生 TileRT 容器中挂载共享 HF 缓存 blobs。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3057 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 960e33e9d..5f822ec53 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -56,7 +56,7 @@ elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/scratch/models/GLM-5-FP8" export SRT_SLURM_MODEL_PREFIX="glm5-fp8" elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp8" ]]; then - export MODEL_PATH="/scratch/models/GLM-5.1-FP8" + export MODEL_PATH="${MODEL_PATH:-/scratch/models/GLM-5.1-FP8}" export SRT_SLURM_MODEL_PREFIX="glm5.1-fp8" elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/scratch/models/GLM-5-NVFP4" diff --git a/runners/test_tilert_power_lifecycle.py b/runners/test_tilert_power_lifecycle.py index 8a20da6c6..973e7595e 100644 --- a/runners/test_tilert_power_lifecycle.py +++ b/runners/test_tilert_power_lifecycle.py @@ -10,6 +10,21 @@ ROOT = Path(__file__).resolve().parents[1] +@pytest.mark.parametrize('prepared_path', ['', '/shared/hf/hub/snapshots/revision']) +def test_b200_tilert_preserves_prepared_model_path(prepared_path): + env = {**os.environ, 'MODEL_PREFIX': 'glm5.1', 'PRECISION': 'fp8', + 'FRAMEWORK': 'tilert', 'IS_MULTINODE': 'true', 'SCENARIO_SUBDIR': '', + 'EXP_NAME': 'glm5.1_8k1k', 'GITHUB_WORKSPACE': str(ROOT), + 'MODEL_PATH': prepared_path} + result = subprocess.run( + ['bash', '-c', 'exec() { printf "%s\\n" "$MODEL_PATH"; exit; }; source "$1"', + 'bash', str(ROOT / 'runners/launch_b200-nscale-compat.sh')], + env=env, capture_output=True, text=True, timeout=10, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == (prepared_path or '/scratch/models/GLM-5.1-FP8') + + @pytest.mark.parametrize(('decode_rc', 'interruption', 'enabled'), [(0, None, True), (9, None, True), (0, 'TERM', True), (0, 'HUP', True), (0, 'INT', True), (0, 'hang', True), @@ -43,6 +58,11 @@ def test_tilert_submit_keeps_decode_status_and_stages_both_roles(tmp_path, decod import json, os, pathlib, signal, subprocess, sys, time args=sys.argv[1:] if any(a.startswith('--container-image=') for a in args): + cache=os.environ['HF_HUB_CACHE_HOST_PATH'] + mounts=next(a.split('=',1)[1] for a in args if a.startswith('--container-mounts=')) + if cache: + assert f'{cache}:{cache}' in mounts.split(',') + assert f"{os.environ['MODEL_PATH']}:{os.environ['MODEL_PATH']}" in mounts.split(',') if os.environ['POWERX_NATIVE_ENABLED'] != '1': assert not any('/powerx_native' in a for a in args) sys.exit(0) @@ -90,6 +110,7 @@ def finish(signum, frame): 'GITHUB_WORKSPACE':str(repo),'B200_SQUASH_DIR':str(tmp_path/'squash'), 'IMAGE':'synthetic-decode','PREFILL_IMAGE':'synthetic-prefill', 'MODEL_PATH':str(repo),'MODEL_PREFIX':'fixture','PRECISION':'fp8', + 'HF_HUB_CACHE_HOST_PATH':str(tmp_path/'hf-cache') if enabled else '', 'PREFILL_TP':'2','DECODE_TP':'2','SLURM_ACCOUNT':'fixture', 'SLURM_PARTITION':'fixture','RUNNER_NAME':'fixture','REQUIRE_POWER':'1' if enabled else '0', 'ISL':'8192','OSL':'1024', 'TILERT_WEIGHTS_DIR':str(tmp_path/'weights'),'TILERT_DECODE_DRAIN':'1',