From c3040728c629feac18cdd26aca7cdbc246b03b73 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:23:08 -0700 Subject: [PATCH 01/11] fix: preserve complete request outcomes and diagnostic batches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留完整请求结果状态与诊断批次，拆分 Slurm 生命周期和原生功耗采集。 --- .../workflows/benchmark-multinode-tmpl.yml | 26 +-- .github/workflows/benchmark-tmpl.yml | 4 +- docs/results-and-ingestion.md | 12 +- docs/results-and-ingestion_zh.md | 12 +- infx/bench_serving/benchmark_outcome.py | 31 +++ infx/bench_serving/benchmark_serving.py | 17 +- infx/results/fixed_sequence.py | 109 ++++++++- infx/results/power/__init__.py | 1 + infx/results/power/audit.py | 50 ++++ perf-changelog.yaml | 11 + utils/test_process_result.py | 217 +++++++++++++++++- 11 files changed, 444 insertions(+), 46 deletions(-) create mode 100644 infx/bench_serving/benchmark_outcome.py create mode 100644 infx/results/power/audit.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index bae33211b..80b0842f8 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -470,7 +470,7 @@ jobs: fi - name: Process result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ always() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} env: RUNNER_TYPE: ${{ inputs.runner }} run: | @@ -480,29 +480,10 @@ jobs: export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA" fi - # Process each result file - for result_file in ${RESULT_FILENAME}_*.json; do - if [ -f "$result_file" ]; then - echo "Processing $result_file" - # Extract GPU count, prefill_gpus and decode_gpus from filename for tp_size calculation - gpus=$(echo "$result_file" | sed -n "s/.*_gpus_\([0-9][0-9]*\).*\.json/\1/p") - prefill_gpus=$(echo "$result_file" | sed -n "s/.*_ctx_\([0-9]*\).*\.json/\1/p") - decode_gpus=$(echo "$result_file" | sed -n "s/.*_gen_\([0-9]*\).*\.json/\1/p") - - if [ -n "$gpus" ]; then - # Non-disagg results lack _ctx_/_gen_ fields; all GPUs belong to the single worker - if [ -z "$prefill_gpus" ] || [ -z "$decode_gpus" ]; then - prefill_gpus=$gpus - decode_gpus=0 - fi - echo "Extracted: gpus=$gpus, prefill_gpus=$prefill_gpus, decode_gpus=$decode_gpus" - RESULT_FILENAME=${result_file%.json} IS_MULTINODE=true PREFILL_GPUS="$prefill_gpus" DECODE_GPUS="$decode_gpus" python3 utils/process_result.py - fi - fi - done + python3 utils/process_result.py --all - name: Upload result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bmk_${{ env.RESULT_FILENAME }} @@ -518,6 +499,7 @@ jobs: agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json LOGS/power/** + result_processing_${{ env.RESULT_FILENAME }}.json LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json LOGS/agentic/**/agentic_power_timezone_offset.txt diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index b08ef0775..4d1b817ea 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -395,14 +395,14 @@ jobs: fi - name: Process result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ always() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} env: RUNNER_TYPE: ${{ inputs.runner }} run: | python3 utils/process_result.py - name: Upload result - if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} + if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bmk_${{ env.RESULT_FILENAME }} diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index daba2715b..a1e357b30 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -92,10 +92,20 @@ The fixed-sequence transformer requires runner, framework, precision, speculativ | Latency and interactivity | Each benchmark input key ending in `ms` is converted from milliseconds to seconds with `_ms` removed. Keys containing `tpot` also produce an `intvty` reciprocal. | | Optional runtime metadata | `router` as exactly `{name, version}`, `kv_p2p_transfer`, and measured power patched from `gpu_metrics.csv` when available | -Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physical GPU count. Multinode per-GPU denominators use the declared prefill and decode GPU counts. Invalid or missing required metadata fails transformation. Power aggregation is explicitly best effort and cannot fail the benchmark aggregate. +Single-node GPU count is `tp * pp * pcp_size`. DCP does not multiply the physical GPU count. Multinode per-GPU denominators use the declared prefill and decode GPU counts. Invalid or missing required metadata fails transformation. Power aggregation is best effort by default; `REQUIRE_POWER=1` fails the job after preserving available results and audits when power validation fails. InferenceX-app treats routing fields as columns or config dimensions and stores numeric measurements in `benchmark_results.metrics` JSONB. The mapper supports v1 shared topology, v2 split prefill/decode topology, and nested v3 AgentX metrics. Unknown numeric metrics are retained and warned about, which permits schema growth without silently losing numeric data. +### Fixed-sequence outcomes and PowerX audits + +The serving client records `benchmark_outcome` before saving its raw result. It retains the existing maximum request-failure rate of 5%, including the requested/completed/failed counts. The processor verifies this record, copies it to the aggregate, and returns failure even when telemetry is valid. Zero successful requests retain a diagnostic aggregate without fabricated reciprocal latency. Legacy results without outcome metadata remain distinguishable; power validity alone never establishes benchmark success or answer quality. + +`power_invalid_reasons` and `power_audit` carry a bounded summary alongside numeric metrics. The summary includes the available measurement window, expected/observed GPU counts, sampling diagnostics, observed device identifiers and producer pin. Its `source` names the retained `power_validation_*.json` sidecar. Device identifiers retain the collector's semantics; local SMI indices are not physical UUID proof. + +For multinode fixed-sequence jobs, `utils/process_result.py --all` processes every available result before returning failure. It accepts `_c_gpus_...`, `_conc_gpus_...`, and AMD `_concurrency__req_rate__gpus_...` filenames, including `inf` request rates. It compares result concurrencies with `CONC_LIST`, rejects duplicate or contradictory point identities, and records omissions/errors in `result_processing_.json`. Aggregate workers pass `AGGREGATE_GPUS` with zero role GPU counts to telemetry validation; separate prefill/decode energy remains absent. Existing layout fields remain compatible, and `num_aggregate_gpu` marks the shared allocation. + +Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index cc5c9e302..6a054d5f0 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -92,10 +92,20 @@ shape: array of benchmark row objects | 延迟和交互性 | 基准输入中每个以 `ms` 结尾的键都会从毫秒换算为秒，并移除 `_ms`。包含 `tpot` 的键还会产生其倒数 `intvty`。 | | 可选运行时元数据 | 形式必须精确为 `{name, version}` 的 `router`、`kv_p2p_transfer`，以及在可用时由 `gpu_metrics.csv` 补入的实测功耗 | -单节点 GPU 数为 `tp * pp * pcp_size`。DCP 不会增加物理 GPU 数。多节点每 GPU 指标的分母使用声明的 prefill 和 decode GPU 数。无效或缺失的必需元数据会使转换失败。功耗聚合明确采用尽力而为模式，不能导致基准聚合失败。 +单节点 GPU 数为 `tp * pp * pcp_size`。DCP 不会增加物理 GPU 数。多节点每 GPU 指标的分母使用声明的 prefill 和 decode GPU 数。无效或缺失的必需元数据会使转换失败。功耗聚合默认尽力而为；当设置 `REQUIRE_POWER=1` 时，功耗验证失败会在保留已有结果和审计后使任务失败。 InferenceX-app 将路由字段作为列或配置维度，并把数值测量存入 `benchmark_results.metrics` JSONB。映射器支持共享拓扑的 v1、拆分 prefill/decode 拓扑的 v2，以及嵌套 AgentX 指标的 v3。未知数值指标会被保留并产生警告，因此架构可以扩展，同时不会无提示地丢失数值数据。 +### 固定序列基准结果状态与 PowerX 审计 + +服务客户端在保存原始结果前写入 `benchmark_outcome`，保留现有的 5% 最大请求失败率，以及请求总数、完成数和失败数。处理器检查该记录并复制到聚合结果中；即使遥测有效，请求失败率超限仍返回失败。零成功请求会保留诊断聚合结果，但不会生成不存在的延迟倒数。没有状态元数据的历史结果仍可区分；功耗有效不能证明基准成功或答案质量。 + +`power_invalid_reasons` 和 `power_audit` 在数值指标旁携带有界摘要，包括可用的测量窗口、预期与观测 GPU 数、采样诊断、观测设备标识和生产者版本。`source` 指向保留的 `power_validation_*.json` 工件名称。设备标识保留采集器原有语义，本地 SMI 序号不是物理 UUID 的证明。 + +对于多节点固定序列任务，`utils/process_result.py --all` 先处理所有已有结果，再返回失败。它接受 `_c_gpus_...`、`_conc_gpus_...` 和 AMD 的 `_concurrency__req_rate__gpus_...` 文件名，也支持 `inf` 请求速率。它将结果并发度与 `CONC_LIST` 比较，拒绝重复或矛盾的点身份，并将遗漏和错误记录到 `result_processing_.json`。共享工作池通过 `AGGREGATE_GPUS` 及零值角色 GPU 数进行遥测验证；独立的 prefill/decode 能耗保持缺失。现有布局字段保持兼容，`num_aggregate_gpu` 明确标记共享分配。 + +启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 + ## 评测工件 ### 单配置身份和收集 diff --git a/infx/bench_serving/benchmark_outcome.py b/infx/bench_serving/benchmark_outcome.py new file mode 100644 index 000000000..e14dca470 --- /dev/null +++ b/infx/bench_serving/benchmark_outcome.py @@ -0,0 +1,31 @@ +"""The serving client's existing request-failure gate, preserved with results.""" + +from __future__ import annotations + +from typing import Literal, TypedDict + + +MAX_FAILURE_RATE = 0.05 + + +class BenchmarkOutcome(TypedDict): + status: Literal["passed", "failed"] + requested: int + completed: int + failed: int + max_failure_rate: float + + +def benchmark_outcome(requested: int, completed: int) -> BenchmarkOutcome: + """Record the same five-percent gate before cleanup or artifact upload.""" + if (type(requested) is not int or type(completed) is not int + or requested <= 0 or not 0 <= completed <= requested): + raise ValueError("Benchmark request counts must satisfy 0 <= completed <= requested > 0") + failed = requested - completed + return { + "status": "failed" if failed / requested > MAX_FAILURE_RATE else "passed", + "requested": requested, + "completed": completed, + "failed": failed, + "max_failure_rate": MAX_FAILURE_RATE, + } diff --git a/infx/bench_serving/benchmark_serving.py b/infx/bench_serving/benchmark_serving.py index 82dfec973..1619d28ae 100644 --- a/infx/bench_serving/benchmark_serving.py +++ b/infx/bench_serving/benchmark_serving.py @@ -62,6 +62,7 @@ from argparse import ArgumentParser as FlexibleArgumentParser from .benchmark_utils import convert_to_pytorch_benchmark_format +from .benchmark_outcome import benchmark_outcome from .encoding_dsv4 import encode_messages as dsv4_encode_messages MILLISECONDS_TO_SECONDS_CONVERSION = 1000 @@ -898,6 +899,11 @@ def main(args: argparse.Namespace): lora_modules=args.lora_modules, )) + # Preserve the request gate before writing results: cleanup or a surrounding + # launcher must not turn an error-containing benchmark into a passed point. + outcome = benchmark_outcome(args.num_prompts, benchmark_result["completed"]) + benchmark_result["benchmark_outcome"] = outcome + # Save config and results to json if args.save_result: result_json: Dict[str, Any] = {} @@ -957,14 +963,11 @@ def main(args: argparse.Namespace): json.dump(result_json, outfile) save_to_pytorch_benchmark_format(args, result_json, file_name) - max_failure_rate = 0.05 - completed = benchmark_result["completed"] - failure_rate = 1 - completed / args.num_prompts - if failure_rate > max_failure_rate: + if outcome["status"] == "failed": raise SystemExit( - f"FAIL: request failure rate {failure_rate:.1%} exceeds " - f"{max_failure_rate:.0%} threshold " - f"({completed}/{args.num_prompts} completed)" + f"FAIL: request failure rate {outcome['failed'] / outcome['requested']:.1%} exceeds " + f"{outcome['max_failure_rate']:.0%} threshold " + f"({outcome['completed']}/{outcome['requested']} completed)" ) diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index ef6ad2b8e..f6fa76102 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -3,7 +3,9 @@ from __future__ import annotations import json +import math import os +import re import sys from collections.abc import Iterable, Mapping from pathlib import Path @@ -12,6 +14,7 @@ from .metadata import parse_component_metadata from .power import ALL_POWER_METRIC_KEYS, POWER_METRIC_SCHEMA_VERSION, with_power_metrics from .topology import Parallelism, validate_parallelism +from infx.bench_serving.benchmark_outcome import benchmark_outcome _BASE_ENV_VARS = ( @@ -49,6 +52,14 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s 'isl': int(env['ISL']), 'osl': int(env['OSL']), } + if 'benchmark_outcome' in benchmark: + outcome = benchmark['benchmark_outcome'] + expected = benchmark_outcome(outcome['requested'], outcome['completed']) + if (outcome != expected + or benchmark.get('completed', expected['completed']) != expected['completed'] + or benchmark.get('num_prompts', expected['requested']) != expected['requested']): + raise ValueError("Benchmark outcome does not match the recorded request counts and gate") + data['benchmark_outcome'] = expected router = parse_component_metadata(env.get('ROUTER_METADATA'), 'ROUTER_METADATA') if router is not None: @@ -61,9 +72,6 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s is_multinode = env.get('IS_MULTINODE', 'false').lower() == 'true' if is_multinode: - # TODO: Eventually will have to have a separate condition in here for multinode disagg and - # multinode agg. For now, just assume that multinode implies disagg. - multinode_vars = ['PREFILL_GPUS', 'DECODE_GPUS', 'PREFILL_NUM_WORKERS', 'PREFILL_TP', 'PREFILL_EP', 'PREFILL_DP_ATTN', 'DECODE_NUM_WORKERS', 'DECODE_TP', 'DECODE_EP', 'DECODE_DP_ATTN'] @@ -76,6 +84,14 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s ) prefill_gpus = int(env['PREFILL_GPUS']) decode_gpus = int(env['DECODE_GPUS']) + aggregate_gpus = int(env.get('AGGREGATE_GPUS', '0')) + if aggregate_gpus: + if aggregate_gpus < 0 or prefill_gpus or decode_gpus or disagg: + raise ValueError("Aggregate GPUs require non-disaggregated topology without role GPUs") + # Preserve the existing multinode layout fields for consumers, but + # expose the actual shared role and validate telemetry as `agg`. + prefill_gpus = aggregate_gpus + data['num_aggregate_gpu'] = aggregate_gpus prefill_num_workers = int(env['PREFILL_NUM_WORKERS']) prefill = Parallelism( tp=int(env['PREFILL_TP']), @@ -154,9 +170,9 @@ def build_result(benchmark: Mapping[str, Any], env: Mapping[str, str]) -> dict[s data = data | single_node_data for key, value in benchmark.items(): - if key.endswith('ms'): + if key.endswith('ms') and math.isfinite(float(value)): data[key.replace('_ms', '')] = float(value) / 1000.0 - if 'tpot' in key: + if 'tpot' in key and math.isfinite(float(value)) and float(value) > 0: data[key.replace('_ms', '').replace( 'tpot', 'intvty')] = 1000.0 / float(value) return data @@ -221,7 +237,8 @@ def aggregate_power_result( source = Path(env.get('POWER_ARTIFACT_DIR', 'LOGS/power')) prefill_gpus = int(env['PREFILL_GPUS']) decode_gpus = int(env['DECODE_GPUS']) - expected_num_gpus = prefill_gpus + decode_gpus + aggregate_gpus = int(env.get('AGGREGATE_GPUS', '0')) + expected_num_gpus = prefill_gpus + decode_gpus + aggregate_gpus else: candidates = [env.get('GPU_METRICS_CSV'), 'gpu_metrics.csv', '/workspace/gpu_metrics.csv'] source = next( @@ -237,6 +254,7 @@ def aggregate_power_result( source, bench_path, agg_path, prefill_gpus=prefill_gpus, decode_gpus=decode_gpus, + aggregate_gpus=aggregate_gpus, expected_producer_sha=env.get('POWER_PRODUCER_SHA') or None, logs_root=Path(env.get('POWER_RESULT_ROOT', 'LOGS')), validation_result=validation_path, @@ -265,8 +283,7 @@ def aggregate_power_result( return int(require_power) -def main() -> int: - env = os.environ +def process_result(env: Mapping[str, str]) -> int: require_environment(env, _BASE_ENV_VARS) result_filename = env['RESULT_FILENAME'] bench_path = Path(f'{result_filename}.json') @@ -277,10 +294,86 @@ def main() -> int: with open(agg_path, 'w') as f: json.dump(data, f, indent=2) status = aggregate_power_result(env, bench_path, agg_path) + validation_path = Path(f'power_validation_{result_filename}.json') + from .power.audit import audit_summary + + result = json.loads(agg_path.read_text()) + try: + validation = json.loads(validation_path.read_text()) + result.update(audit_summary(validation, validation_path.name)) + except (OSError, ValueError, TypeError) as exc: + print(f'[process_result] audit summary unavailable: {exc}', file=sys.stderr) + result['power_invalid_reasons'] = ['validation_artifact_unavailable'] + # A required run must preserve its audit as well as numeric metrics. + status = max(status, int(env.get('REQUIRE_POWER', '').lower() in {'1', 'true', 'yes'})) + agg_path.write_text(json.dumps(result, indent=2)) with open(agg_path) as f: print(json.dumps(json.load(f), indent=2)) + return max(status, int(data.get('benchmark_outcome', {}).get('status') == 'failed')) + + +def process_multinode_results(env: Mapping[str, str]) -> int: + """Process every available point and preserve sweep omissions before failing.""" + expected = {int(value) for value in env['CONC_LIST'].split()} + if not expected or min(expected) <= 0: + raise ValueError('CONC_LIST must contain positive concurrencies') + points: list[dict[str, Any]] = [] + observed: set[int] = set() + status = 0 + for path in sorted(Path('.').glob(f"{env['RESULT_FILENAME']}_*.json")): + point: dict[str, Any] = {'source': path.name} + try: + match = re.search( + r'_(?:c|conc|concurrency_)(\d+)(?:_req_rate_[^_]+)?_gpus_(\d+)' + r'(?:_ctx_(\d+)_gen_(\d+))?$', path.stem) + if match is None: + raise ValueError('Result filename lacks concurrency and physical GPU counts') + concurrency, total = int(match[1]), int(match[2]) + raw = json.loads(path.read_text()) + if raw['max_concurrency'] != concurrency: + raise ValueError('Result concurrency does not match its filename') + if concurrency in observed: + raise ValueError('Duplicate concurrency result within one recipe') + observed.add(concurrency) + point['concurrency'] = concurrency + point_env = {**env, 'RESULT_FILENAME': path.stem, 'IS_MULTINODE': 'true'} + disagg = env['DISAGG'].lower() == 'true' + # Some disaggregated recipe groups include a shared-worker point. + # Its zero decode workers and filename must both describe aggregate + # execution; the group-level flag cannot manufacture role energy. + aggregate = not disagg or int(env['DECODE_NUM_WORKERS']) == 0 + if aggregate: + if match[3] is not None and (int(match[3]) != total or int(match[4]) != 0): + raise ValueError('Aggregate result contains separate role GPU counts') + point_env.update(DISAGG='false', PREFILL_GPUS='0', DECODE_GPUS='0', + AGGREGATE_GPUS=str(total)) + else: + if match[3] is None: + raise ValueError('Disaggregated result lacks prefill/decode GPU counts') + prefill, decode = int(match[3]), int(match[4]) + if prefill + decode != total: + raise ValueError('Role GPU counts do not equal total GPU count') + point_env.update(PREFILL_GPUS=str(prefill), DECODE_GPUS=str(decode), AGGREGATE_GPUS='0') + point['exit_code'] = process_result(point_env) + except (OSError, ValueError, KeyError, TypeError) as exc: + point.update(exit_code=1, error=str(exc)) + print(f'[process_result] {path}: {exc}', file=sys.stderr) + status = max(status, point['exit_code']) + points.append(point) + missing, unexpected = sorted(expected - observed), sorted(observed - expected) + status = max(status, int(bool(missing or unexpected))) + summary = {'expected_concurrencies': sorted(expected), + 'missing_concurrencies': missing, 'unexpected_concurrencies': unexpected, + 'points': points, 'exit_code': status} + Path(f"result_processing_{env['RESULT_FILENAME']}.json").write_text(json.dumps(summary, indent=2)) + if missing or unexpected: + print(f'[process_result] incomplete sweep: missing={missing}, unexpected={unexpected}', file=sys.stderr) return status +def main() -> int: + return process_multinode_results(os.environ) if sys.argv[1:] == ['--all'] else process_result(os.environ) + + if __name__ == '__main__': sys.exit(main()) diff --git a/infx/results/power/__init__.py b/infx/results/power/__init__.py index f0e1da95c..8007f5a91 100644 --- a/infx/results/power/__init__.py +++ b/infx/results/power/__init__.py @@ -59,6 +59,7 @@ def with_power_metrics( data["power_metric_schema_version"] = schema_version data["power_valid"] = int(power_valid) data.pop("power_invalid_reasons", None) + data.pop("power_audit", None) if power_valid: for key, value in metrics.items(): if value is None or not math.isfinite(value): diff --git a/infx/results/power/audit.py b/infx/results/power/audit.py new file mode 100644 index 000000000..6463976cf --- /dev/null +++ b/infx/results/power/audit.py @@ -0,0 +1,50 @@ +"""Bounded public audit metadata from retained power-validation sidecars.""" + +from __future__ import annotations + +import math +import re +from collections.abc import Mapping +from typing import Any + + +def audit_summary(validation: Mapping[str, Any], source: str) -> dict[str, Any]: + """Project the shared app audit contract without publishing raw telemetry.""" + audit: dict[str, Any] = {"source": source} + window = validation.get("benchmark_window") or {} + producer = validation.get("producer") or {} + fields = { + "window_start_unix": window.get("start_time_unix"), + "window_end_unix": window.get("end_time_unix"), + "expected_gpu_count": validation.get("expected_gpu_count"), + "observed_gpu_count": validation.get("observed_gpu_count"), + } + counts = validation.get("per_gpu_sample_counts") or {} + if counts: + fields["sample_count"] = sum(counts.values()) + gaps = validation.get("per_gpu_max_sample_gap_s") or {} + finite_gaps = [value for value in gaps.values() + if type(value) in (int, float) and math.isfinite(value)] + if finite_gaps: + fields["max_sample_gap_s"] = max(finite_gaps) + for key, value in fields.items(): + if type(value) in (int, float) and math.isfinite(value) and value >= 0: + audit[key] = value + for target, original in (("producer_sha", "producer_git_commit"), + ("exporter_image_sha256", "exporter_image_sha256")): + value = producer.get(original) + if isinstance(value, str) and 0 < len(value) <= 128: + audit[target] = value + ids = validation.get("observed_gpu_ids") + if ids is None: + ids = list((validation.get("per_gpu_role") or {}).keys()) + if ids: + audit["observed_gpu_ids"] = list(dict.fromkeys( + str(value) for value in ids if 0 < len(str(value)) <= 128 + ))[:1024] + return { + "power_invalid_reasons": [reason for reason in validation.get("reasons", []) + if isinstance(reason, str) + and re.fullmatch(r"[a-z][a-z0-9_]{0,63}", reason)][:32], + "power_audit": audit, + } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 88c163461..f3b85de24 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7399,3 +7399,14 @@ - "Disable adaptive verification in the EVAL_ONLY DSpark config as well. It trims verification requests on device, which the ROCm DeepseekV4IndexerBackend does not support, so the eval-only engine refused to start (run 34651830283, c32). Evals keep real block rejection; throughput settings are unchanged." - "EVAL_ONLY 的 DSpark 配置同样关闭自适应验证：它会在设备端裁剪验证请求，而 ROCm 的 DeepseekV4IndexerBackend 不支持该操作，导致仅评测引擎拒绝启动（运行 34651830283，c32）。评测仍保留真实块拒绝采样；吞吐设置不变。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Preserve request failure outcomes, retain all available fixed-sequence diagnostics, and reject incomplete concurrency batches before publishing benchmark artifacts." + - "保留请求失败状态及全部可用的固定序列诊断信息；发布基准产物前拒绝缺失并发点的批次。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 485021b36..d14d6a87a 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -28,12 +28,13 @@ def test_workflow_processes_results_through_compatibility_entrypoint( step = next(step for job in workflow["jobs"].values() for step in job.get("steps", []) if step.get("name", "").startswith("Process result")) multinode = workflow_name == "benchmark-multinode-tmpl.yml" - stem = "fixture_gpus_28_ctx_20_gen_8" if multinode else "fixture" + stem = "fixture_conc64_gpus_28_ctx_20_gen_8" if multinode else "fixture" (tmp_path / f"{stem}.json").write_text(json.dumps({ **sample_benchmark_result, "total_token_throughput": 56, "output_throughput": 28, })) env = {**(multinode_env_vars if multinode else single_node_env_vars), "RESULT_FILENAME": "fixture" if multinode else stem, "POWER_PRODUCER_SHA": "", + "CONC_LIST": "64", "PATH": f"{Path(sys.executable).parent}:{os.environ['PATH']}"} result = subprocess.run(["bash", "-euo", "pipefail", "-c", step["run"]], cwd=tmp_path, env=env, capture_output=True, text=True, timeout=10) @@ -845,7 +846,7 @@ def test_missing_csv_does_not_break_process_result(self, tmp_path, single_node_e assert "avg_power_w" not in patched assert "joules_per_output_token" not in patched assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] validation = json.loads( (tmp_path / "power_validation_benchmark_result.json").read_text() @@ -876,7 +877,7 @@ def test_missing_bench_timestamps_does_not_patch(self, tmp_path, single_node_env assert "avg_power_w" not in patched assert "joules_per_output_token" not in patched assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] def test_expected_gpu_count_mismatch_is_invalid(self, tmp_path, single_node_env_vars): """TP/PP/PCP topology is checked against the observed device IDs.""" @@ -902,7 +903,7 @@ def test_expected_gpu_count_mismatch_is_invalid(self, tmp_path, single_node_env_ assert result.returncode == 0, f"Script failed: {result.stderr}" patched = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert patched["power_valid"] == 0 - assert "power_invalid_reasons" not in patched + assert patched["power_invalid_reasons"] assert "total_gpu_energy_j" not in patched def test_require_power_propagates_validation_failure( @@ -1009,7 +1010,7 @@ def test_internal_aggregation_error_is_always_auditable( agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert agg["power_metric_schema_version"] == 2 assert agg["power_valid"] == 0 - assert "power_invalid_reasons" not in agg + assert agg["power_invalid_reasons"] == ["aggregation_internal_error"] validation = json.loads( (tmp_path / "power_validation_benchmark_result.json").read_text() ) @@ -1416,3 +1417,209 @@ def test_strict_mode_passes_on_valid_package(self, tmp_path, power_env): assert result.returncode == 0, f"Script failed: {result.stderr}" agg = json.loads((tmp_path / "agg_benchmark_result.json").read_text()) assert agg["power_valid"] == 1 + + +@pytest.mark.parametrize('completed,status', [(100, 'passed'), (95, 'passed'), (94, 'failed')]) +def test_request_outcome_preserves_existing_failure_threshold(completed, status): + from infx.bench_serving.benchmark_outcome import benchmark_outcome + + outcome = benchmark_outcome(100, completed) + assert outcome['status'] == status + assert outcome['failed'] == 100 - completed + assert outcome['max_failure_rate'] == 0.05 + + +def test_failed_client_is_preserved_with_valid_power(tmp_path, single_node_env_vars): + start, end = 1_700_000_100.0, 1_700_000_110.0 + TestPowerAggregationIntegration._write_nvidia_csv(tmp_path / 'gpu_metrics.csv', start, end, num_gpus=2) + outcome = {'status': 'failed', 'requested': 100, 'completed': 94, 'failed': 6, + 'max_failure_rate': 0.05} + raw = {'model_id': 'fixture', 'max_concurrency': 4, 'total_token_throughput': 500, + 'output_throughput': 100, 'benchmark_start_time_unix': start, + 'benchmark_end_time_unix': end, 'duration': 10, 'completed': 94, + 'total_input_tokens': 8192, 'total_output_tokens': 1024, + 'benchmark_outcome': outcome} + result = run_script(tmp_path, {**single_node_env_vars, 'TP': '2'}, raw) + assert result.returncode == 1, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['benchmark_outcome'] == outcome + assert aggregate['power_valid'] == 1 + assert aggregate['power_invalid_reasons'] == [] + assert aggregate['power_audit']['window_start_unix'] == start + assert aggregate['power_audit']['expected_gpu_count'] == 2 + assert aggregate['power_audit']['observed_gpu_count'] == 2 + assert aggregate['power_audit']['source'] == 'power_validation_benchmark_result.json' + + +def test_request_outcome_cannot_disagree_with_raw_counts(single_node_env_vars, sample_benchmark_result): + from infx.results.fixed_sequence import build_result + + raw = {**sample_benchmark_result, 'completed': 94, + 'benchmark_outcome': {'status': 'passed', 'requested': 100, 'completed': 100, + 'failed': 0, 'max_failure_rate': 0.05}} + with pytest.raises(ValueError, match='request counts and gate'): + build_result(raw, single_node_env_vars) + + +def test_multinode_aggregate_role_through_result_processor(tmp_path, multinode_env_vars): + pkg = build_package(tmp_path, bench_extra=TestMultinodePower.BENCH_EXTRA) + manifest_path = pkg.power_dir / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + for device in manifest['expected_devices']: + for assignment in device['assignments']: + assignment.update(worker_role='agg', het_group=None) + manifest_path.write_text(json.dumps(manifest)) + env = {**multinode_env_vars, 'DISAGG': 'false', 'PREFILL_GPUS': '0', + 'DECODE_GPUS': '0', 'AGGREGATE_GPUS': '4', 'POWER_PRODUCER_SHA': PRODUCER_SHA, + 'REQUIRE_POWER': '1'} + result = run_script(tmp_path, env, json.loads(pkg.original_result.read_text())) + assert result.returncode == 0, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['power_valid'] == 1 + assert aggregate['num_aggregate_gpu'] == 4 + assert aggregate['avg_power_w'] == 350 + assert aggregate['power_audit']['producer_sha'] == PRODUCER_SHA + assert set(ROLE_METRIC_KEYS).isdisjoint(aggregate) + + +@pytest.mark.parametrize('conc_token,rate_suffix', [ + ('c', ''), ('conc', ''), + ('concurrency_', '_req_rate_1'), ('concurrency_', '_req_rate_inf'), +]) +@pytest.mark.parametrize('expected_concs,missing', [('4 8 16', [8]), ('4 16', [])]) +def test_multinode_batch_preserves_points_and_checks_completeness( + tmp_path, multinode_env_vars, sample_benchmark_result, conc_token, rate_suffix, + expected_concs, missing, +): + for conc in (4, 16): + (tmp_path / f'run_recipe_{conc_token}{conc}{rate_suffix}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': conc})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': expected_concs, 'REQUIRE_POWER': '0'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == int(bool(missing)), result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == missing + assert receipt['unexpected_concurrencies'] == [] + assert len(receipt['points']) == 2 + for conc in (4, 16): + stem = f'run_recipe_{conc_token}{conc}{rate_suffix}_gpus_4_ctx_2_gen_2' + aggregate = json.loads((tmp_path / f'agg_{stem}.json').read_text()) + assert aggregate['power_valid'] == 0 + assert aggregate['power_invalid_reasons'] + assert (tmp_path / f'power_validation_{stem}.json').is_file() + + +@pytest.mark.parametrize('role_suffix', ['', '_ctx_4_gen_0']) +def test_multinode_batch_normalizes_legacy_zero_decode_aggregate( + tmp_path, multinode_env_vars, role_suffix, +): + pkg = build_package(tmp_path, bench_extra=TestMultinodePower.BENCH_EXTRA) + manifest_path = pkg.power_dir / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + for device in manifest['expected_devices']: + for assignment in device['assignments']: + assignment.update(worker_role='agg', het_group=None) + manifest_path.write_text(json.dumps(manifest)) + stem = f'run_recipe_c4_gpus_4{role_suffix}' + (tmp_path / f'{stem}.json').write_text(pkg.original_result.read_text()) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4', 'DECODE_NUM_WORKERS': '0', 'REQUIRE_POWER': '1', + 'POWER_PRODUCER_SHA': PRODUCER_SHA} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 0, result.stderr + aggregate = json.loads((tmp_path / f'agg_{stem}.json').read_text()) + assert aggregate['disagg'] is False + assert aggregate['num_aggregate_gpu'] == 4 + assert aggregate['num_decode_gpu'] == 0 + assert aggregate['decode_num_workers'] == 0 + assert aggregate['power_valid'] == 1 + assert aggregate['avg_power_w'] == 350 + assert set(ROLE_METRIC_KEYS).isdisjoint(aggregate) + + +@pytest.mark.parametrize('decode_workers,role_suffix', [ + ('0', '_ctx_2_gen_2'), ('0', '_ctx_3_gen_0'), ('1', ''), +]) +def test_multinode_batch_rejects_misdeclared_aggregate_role_counts( + tmp_path, multinode_env_vars, sample_benchmark_result, decode_workers, role_suffix, +): + stem = f'run_recipe_c4_gpus_4{role_suffix}' + (tmp_path / f'{stem}.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': 4})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4', 'DECODE_NUM_WORKERS': decode_workers} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['points'][0]['exit_code'] == 1 + assert not (tmp_path / f'agg_{stem}.json').exists() + + +def test_zero_successful_requests_preserves_diagnostic_result(tmp_path, single_node_env_vars, sample_benchmark_result): + raw = {**sample_benchmark_result, 'tpot_p50_ms': 0, 'tpot_p99_ms': float('nan'), + 'completed': 0, 'num_prompts': 4, + 'benchmark_outcome': {'status': 'failed', 'requested': 4, 'completed': 0, + 'failed': 4, 'max_failure_rate': 0.05}} + result = run_script(tmp_path, single_node_env_vars, raw) + assert result.returncode == 1, result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['benchmark_outcome']['failed'] == 4 + assert aggregate['power_valid'] == 0 + assert 'intvty_p50' not in aggregate + assert 'tpot_p99' not in aggregate + + +@pytest.mark.parametrize('extra_conc,error', [(4, 'Duplicate concurrency'), (8, None)]) +def test_multinode_batch_rejects_extra_results_without_losing_other_points( + tmp_path, multinode_env_vars, sample_benchmark_result, extra_conc, error, +): + for label, conc in [('a', 4), ('b', extra_conc), ('c', 16)]: + (tmp_path / f'run_{label}_conc{conc}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': conc})) + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4 16'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [] + assert receipt['unexpected_concurrencies'] == ([] if error else [8]) + if error: + assert error in receipt['points'][1]['error'] + assert (tmp_path / 'agg_run_a_conc4_gpus_4_ctx_2_gen_2.json').is_file() + assert (tmp_path / 'agg_run_c_conc16_gpus_4_ctx_2_gen_2.json').is_file() + + +def test_multinode_empty_sweep_records_every_missing_point(tmp_path, multinode_env_vars): + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4 8'} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**env, 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1, result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [4, 8] + assert receipt['points'] == [] + + +def test_public_power_audit_bounds_text_and_device_identifiers(): + from infx.results.power.audit import audit_summary + + summary = audit_summary({ + 'producer': {'producer_git_commit': 'x' * 129, 'exporter_image_sha256': 'a' * 64}, + 'observed_gpu_ids': ['gpu0', 'gpu0', 'x' * 129] + [f'gpu{i}' for i in range(1, 1025)], + 'reasons': ['window_missing', ''] + [f'reason_{i}' for i in range(40)], + }, 'power_validation_run.json') + assert len(summary['power_invalid_reasons']) == 32 + assert '' not in summary['power_invalid_reasons'] + audit = summary['power_audit'] + assert 'producer_sha' not in audit + assert audit['exporter_image_sha256'] == 'a' * 64 + assert len(audit['observed_gpu_ids']) == 1024 + assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] From ba0275c4769dc42e606d6fec2e89a3c910b9eb5a Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:24:13 -0700 Subject: [PATCH 02/11] feat: separate native multinode power collection contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 拆分原生多节点功耗采集契约、UTC 上下文和生命周期屏障，不启用新路径。 --- .../workflows/benchmark-multinode-tmpl.yml | 1 + .github/workflows/benchmark-tmpl.yml | 4 + benchmarks/benchmark_lib.sh | 7 +- benchmarks/native_power_collect.sh | 73 +++++ benchmarks/native_power_lifecycle.sh | 77 +++++ docs/results-and-ingestion.md | 6 + docs/results-and-ingestion_zh.md | 6 + infx/results/fixed_sequence.py | 15 + infx/results/power/native_multinode.py | 286 ++++++++++++++++++ infx/results/power/single_node.py | 59 +++- perf-changelog.yaml | 11 + runners/test_native_collector_barriers.py | 58 ++++ runners/test_native_collector_receipts.py | 16 + utils/test_aggregate_power.py | 255 +++++++++++++++- utils/test_native_multinode_power.py | 201 ++++++++++++ 15 files changed, 1057 insertions(+), 18 deletions(-) create mode 100644 benchmarks/native_power_collect.sh create mode 100644 benchmarks/native_power_lifecycle.sh create mode 100644 infx/results/power/native_multinode.py create mode 100644 runners/test_native_collector_barriers.py create mode 100644 runners/test_native_collector_receipts.py create mode 100644 utils/test_native_multinode_power.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 80b0842f8..e9f3a3c90 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -499,6 +499,7 @@ jobs: agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json LOGS/power/** + LOGS/native_power/** result_processing_${{ env.RESULT_FILENAME }}.json LOGS/*/results_*.json LOGS/agentic/**/agentic_power_concurrency_*.json diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 4d1b817ea..1da933f63 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -445,11 +445,13 @@ jobs: name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }} path: | gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json if-no-files-found: ignore @@ -462,12 +464,14 @@ jobs: ${{ env.RESULT_FILENAME }}.json agg_${{ env.RESULT_FILENAME }}.json gpu_metrics.csv + gpu_metrics_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json results/gpu_metrics*.csv + results/gpu_metrics_context.json results/gpu_metrics_identity.json results/agentic_power_window.json results/agentic_power_timezone_offset.txt diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 305a118ef..9d99a4cc9 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -177,6 +177,7 @@ start_gpu_monitor() { GPU_METRICS_CSV="$output" GPU_MONITOR_INTERVAL="$interval" export GPU_METRICS_CSV + printf '{"timestamp_timezone":"UTC"}\n' > "${output%.csv}_context.json" if command -v nvidia-smi &>/dev/null; then GPU_MONITOR_VENDOR="nvidia" @@ -185,7 +186,7 @@ start_gpu_monitor() { rm -f "${output%.csv}_identity.csv" echo "[GPU Monitor] Warning: NVIDIA identity sidecar failed" >&2 fi - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv -l "$interval" > "$output" 2>/dev/null & GPU_MONITOR_PID=$! echo "[GPU Monitor] Started NVIDIA (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)" @@ -196,7 +197,7 @@ start_gpu_monitor() { # Python; measured on MI355X: trailing ticks were lost at kill without it). # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated # headers, and flush every row so killing the pipe cannot discard buffered samples. - PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ + TZ=UTC PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ | awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" & GPU_MONITOR_PID=$! # Hardware energy-accumulator + identity snapshots; the end-side twin in @@ -232,7 +233,7 @@ stop_gpu_monitor() { case "$GPU_MONITOR_VENDOR" in nvidia) if _repair_truncated_gpu_metrics_tail; then - nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv,noheader >> "$GPU_METRICS_CSV" 2>/dev/null || echo "[GPU Monitor] Warning: final NVIDIA sample failed" >&2 fi diff --git a/benchmarks/native_power_collect.sh b/benchmarks/native_power_collect.sh new file mode 100644 index 000000000..69df07951 --- /dev/null +++ b/benchmarks/native_power_collect.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# One native SMI collector per serving node. Raw files stay on node-local scratch; +# the launcher stages them as its host user after containers stop. +set -uo pipefail +power_dir=$1 +control_dir=$2 +vendor=$3 +rank=$4 +role=$5 +gpu_indices=$6 +num_nodes=$7 +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" +source "$repo_root/benchmarks/benchmark_lib.sh" +mkdir -p "$power_dir" +collector_rc=0 +finished=0 + +write_control() { + local path="$control_dir/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + # The control directory is created by the host user before Docker starts. + # Containers must not strand root-owned files in the shared runner tree. + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +finish() { + local incoming_rc=$? + [[ "$finished" == 0 ]] || return + if [[ "$incoming_rc" != 0 ]]; then collector_rc=$incoming_rc; fi + if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi + stop_gpu_monitor + if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json + else + nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \ + > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1 + fi + python3 -m infx.results.power.native_multinode end --directory "$power_dir" \ + --collector-exit-code "$collector_rc" || collector_rc=1 + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown -R "$POWERX_HOST_UID:$POWERX_HOST_GID" "$power_dir" || collector_rc=1 + fi + write_control "done-$rank" "$collector_rc" + finished=1 +} +trap finish EXIT +trap 'collector_rc=130; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 130' INT +trap 'collector_rc=143; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 143' TERM HUP + +case "${POWERX_CLOCK_SYNCHRONIZED:-false}" in + yes|true) clock_synchronized=true ;; + *) clock_synchronized=false ;; +esac + +python3 -m infx.results.power.native_multinode begin --directory "$power_dir" \ + --vendor "$vendor" --rank "$rank" --role "$role" --gpu-indices "$gpu_indices" \ + --num-nodes "$num_nodes" --clock-synchronized "$clock_synchronized" || exit 1 +start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 +[[ "$GPU_MONITOR_VENDOR" == "$vendor" ]] || exit 1 +if [[ "$vendor" == amd ]]; then + _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json +fi +write_control "ready-$rank" ready +while [[ ! -f "$control_dir/stop" ]]; do + _background_process_is_running "$GPU_MONITOR_PID" || exit 1 + sleep 1 & + wait $! || true +done diff --git a/benchmarks/native_power_lifecycle.sh b/benchmarks/native_power_lifecycle.sh new file mode 100644 index 000000000..b24d1dada --- /dev/null +++ b/benchmarks/native_power_lifecycle.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Shared barriers for native collectors. Launchers own host scratch and mounts. + +powerx_start_collector() { + local power_dir="$1" control_dir="$2" vendor="$3" rank="$4" role="$5" gpus="$6" nodes="$7" + local indices + indices=$(seq -s, 0 "$((gpus - 1))") || return 1 + POWERX_CONTROL_DIR="$control_dir" + POWERX_NUM_NODES="$nodes" + bash "$(dirname "${BASH_SOURCE[0]}")/native_power_collect.sh" \ + "$power_dir" "$control_dir" "$vendor" "$rank" "$role" "$indices" "$nodes" & + POWERX_COLLECTOR_PID=$! +} + +powerx_write_control() { + local path="$POWERX_CONTROL_DIR/$1" + local pending="$path.tmp" + printf '%s\n' "$2" > "$pending" || return + if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return + fi + mv -f "$pending" "$path" +} + +powerx_wait_collectors() { + local phase="$1" deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rank pending failed + while :; do + pending=0 + failed=0 + for ((rank=0; rank&2 + return 1 + fi + if [[ ! -f "$POWERX_CONTROL_DIR/$phase-$rank" ]]; then + pending=1 + elif [[ "$phase" == done && "$(cat "$POWERX_CONTROL_DIR/done-$rank")" != 0 ]]; then + failed=1 + fi + done + if [[ "$pending" == 0 ]]; then + [[ "$failed" == 0 ]] || echo "One or more PowerX collectors failed" >&2 + return "$failed" + fi + if (( SECONDS >= deadline )); then + echo "Timed out waiting for PowerX $phase receipts" >&2 + return 1 + fi + sleep 1 + done +} + +powerx_stop_collectors() { + local rc=0 + powerx_write_control stop stop || rc=$? + powerx_wait_collectors done || rc=$? + powerx_reap_collector || rc=$? + return "$rc" +} + +powerx_reap_collector() { + [[ -n "${POWERX_COLLECTOR_PID:-}" ]] || return 0 + local deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rc=0 + while kill -0 "$POWERX_COLLECTOR_PID" 2>/dev/null; do + if (( SECONDS >= deadline )); then + kill -TERM "$POWERX_COLLECTOR_PID" 2>/dev/null || true + sleep 2 + kill -KILL "$POWERX_COLLECTOR_PID" 2>/dev/null || true + rc=1 + break + fi + sleep 1 + done + wait "$POWERX_COLLECTOR_PID" || rc=$? + POWERX_COLLECTOR_PID="" + return "$rc" +} diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index a1e357b30..6da11624d 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -106,6 +106,12 @@ For multinode fixed-sequence jobs, `utils/process_result.py --all` processes eve Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks. +### Native multinode telemetry + +`native_power_collect.sh` and `native_power_lifecycle.sh` provide per-node collection and bounded ready/stop receipts. Launchers opt into the native package under `LOGS/native_power`; this prerequisite enables no new recipe. The adapter validates serving GPU identity, synchronized clocks, collector completion, and complete formal-window coverage. It preserves per-node failures, sample counts, and collector revision in the audit. + +SMI collection records UTC context beside each CSV for portable replay. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index 6a054d5f0..fd3fa2a2d 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -106,6 +106,12 @@ InferenceX-app 将路由字段作为列或配置维度，并把数值测量存 启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。 +### 原生多节点遥测 + +`native_power_collect.sh` 和 `native_power_lifecycle.sh` 提供每节点采集及有时限的就绪/停止状态文件。启动器可使用 `LOGS/native_power` 下的原生产物；此前置改动不会启用新 recipe。适配器验证服务 GPU 身份、时钟同步、采集完成及正式窗口完整覆盖，并在审计中保留节点故障、样本数和采集器版本。 + +SMI 采集在 CSV 旁记录 UTC 上下文以支持跨环境回放。正式窗口外的无效样本不能构成覆盖；`boundary_degenerate_rows` 保留其逐 GPU 计数。 + ## 评测工件 ### 单配置身份和收集 diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index f6fa76102..5ae8a65ee 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -248,6 +248,21 @@ def aggregate_power_result( expected_num_gpus = int(env['TP']) * int(env.get('PP_SIZE', '1')) * int(env.get('PCP_SIZE', '1')) try: if is_multinode: + native_dir = Path(env.get('POWERX_NATIVE_DIR', 'LOGS/native_power')) + if env.get('POWERX_NATIVE_DIR') or native_dir.is_dir(): + if source.is_dir() and source != native_dir: + raise ValueError('Both native and SRT power packages are present') + source = native_dir + from .power.native_multinode import run + + return run( + native_dir, bench_path, agg_path, + expected_prefill_gpus=prefill_gpus, + expected_decode_gpus=decode_gpus, + expected_aggregate_gpus=aggregate_gpus, + validation_result=validation_path, + require_power=require_power, + ) from .power.multinode import run return run( diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py new file mode 100644 index 000000000..1ffae5bd9 --- /dev/null +++ b/infx/results/power/native_multinode.py @@ -0,0 +1,286 @@ +"""Validate native SMI traces from each node of one fixed-sequence deployment. + +This format is owned by InferenceX. It does not claim the srt-slurm/DCGM wire +contract. The launcher records real serving-device membership and synchronized +host clocks; the normal result processor binds every trace to the client window. +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import os +import socket +import time +from datetime import timezone +from pathlib import Path + +from . import ALL_POWER_METRIC_KEYS +from .common import ( + _load_benchmark_data, _write_json_atomic, benchmark_window_payload, + patch_power_metrics, +) +from .single_node import _derived_metrics, _detect_columns, _parse_timestamp, integrate_power + + +def _identity(path: Path, vendor: str) -> dict[str, str]: + """Map the vendor enumeration index to a physical UUID; never use PCI alone.""" + if vendor == "nvidia": + with path.open(newline="") as stream: + rows = [] + for row in csv.DictReader(stream, skipinitialspace=True): + if any(not isinstance(k, str) or not isinstance(v, str) for k, v in row.items()): + raise ValueError("invalid_device_identity") + rows.append({k.strip().lower(): v.strip() for k, v in row.items()}) + elif vendor == "amd": + payload = json.loads(path.read_text()) + rows = [] + + def visit(value: object) -> None: + if isinstance(value, list): + for item in value: + visit(item) + elif isinstance(value, dict): + row = {str(k).lower(): v for k, v in value.items()} + if "gpu" in row and "uuid" in row: + rows.append(row) + else: + for item in value.values(): + visit(item) + + visit(payload) + else: + raise ValueError("unsupported_native_vendor") + identities = {} + for row in rows: + index = str(row.get("index", row.get("gpu", ""))).strip() + uuid = str(row.get("uuid", "")).strip() + if not index.isdigit() or not uuid or uuid.lower() in {"n/a", "none", "null"}: + raise ValueError("invalid_device_identity") + if index in identities or uuid in identities.values(): + raise ValueError("duplicate_device_identity") + identities[index] = uuid + if not identities: + raise ValueError("device_identity_missing") + return identities + + +def record_begin(directory: Path, *, vendor: str, node: str, rank: int, + role: str, gpu_indices: list[int], num_nodes: int, job_id: str, + clock_synchronized: bool, revision: str) -> None: + if (role not in {"prefill", "decode", "aggregate"} or rank < 0 or + num_nodes <= rank or not gpu_indices or min(gpu_indices) < 0 or + len(set(gpu_indices)) != len(gpu_indices)): + raise ValueError("invalid_native_topology") + directory.mkdir(parents=True, exist_ok=True) + _write_json_atomic(directory / "manifest.json", { + "schema_version": 1, "collector": "inferencex-native-smi", + "vendor": vendor, "node": node, "rank": rank, "role": role, + "selected_gpu_indices": gpu_indices, "expected_num_nodes": num_nodes, + "job_id": job_id, "collector_revision": revision, + "clock_source": "utc_ntp", "clock_synchronized": clock_synchronized, + "clock_observation": "timedatectl NTPSynchronized on serving host; no measured clock offset", + "collection_start_unix": time.time(), "lifecycle": "collecting", + }) + + +def record_end(directory: Path, *, collector_exit_code: int) -> None: + manifest = json.loads((directory / "manifest.json").read_text()) + manifest.update(collection_end_unix=time.time(), + collector_exit_code=collector_exit_code, + lifecycle="complete" if collector_exit_code == 0 else "failed") + _write_json_atomic(directory / "manifest.json", manifest) + + +def run(power_dir: Path, bench_result: Path, agg_result: Path, *, + expected_prefill_gpus: int, expected_decode_gpus: int, expected_aggregate_gpus: int = 0, + validation_result: Path | None = None, require_power: bool = False) -> int: + validation_result = validation_result or bench_result.with_name( + f"power_validation_{bench_result.stem}.json") + benchmark, reasons = _load_benchmark_data(bench_result) + expected_gpus = expected_prefill_gpus + expected_decode_gpus + expected_aggregate_gpus + if expected_aggregate_gpus and (expected_prefill_gpus or expected_decode_gpus): + reasons.append("native_mixed_aggregate_role_topology") + roles: dict[str, list[str]] = {"prefill": [], "decode": [], "aggregate": []} + receipts = [] + node_errors = [] + samples = [] + ranks = [] + nodes = [] + jobs = set() + revisions = set() + expected_nodes = set() + paths = sorted(power_dir.glob("node-*/manifest.json")) + if not paths: + reasons.append("native_manifests_missing") + if expected_gpus <= 0 or min(expected_prefill_gpus, expected_decode_gpus, expected_aggregate_gpus) < 0: + reasons.append("invalid_expected_gpu_count") + for path in paths: + try: + manifest = json.loads(path.read_text()) + rank = manifest["rank"] + role = manifest["role"] + if (manifest.get("schema_version") != 1 or + manifest.get("collector") != "inferencex-native-smi" or + type(rank) is not int or rank < 0 or role not in roles or + path.parent.name != f"node-{rank}"): + raise ValueError("invalid_native_manifest") + if (not isinstance(manifest.get("node"), str) or not manifest["node"] or + not isinstance(manifest.get("job_id"), str) or + not isinstance(manifest.get("collector_revision"), str) or + type(manifest.get("expected_num_nodes")) is not int or + manifest["expected_num_nodes"] <= 0): + raise ValueError("invalid_native_run_identity") + ranks.append(rank) + nodes.append(manifest["node"]) + expected_nodes.add(manifest["expected_num_nodes"]) + jobs.add(manifest["job_id"]) + revisions.add(manifest["collector_revision"]) + if (manifest.get("lifecycle") != "complete" or + manifest.get("collector_exit_code") != 0): + reasons.append("native_collector_incomplete") + if (manifest.get("clock_source") != "utc_ntp" or + manifest.get("clock_synchronized") is not True): + reasons.append("native_clock_not_synchronized") + start, end = manifest["collection_start_unix"], manifest["collection_end_unix"] + if (not all(type(x) in (int, float) and math.isfinite(x) for x in (start, end)) + or end <= start or (benchmark is not None and + (start > benchmark.start_unix or end < benchmark.end_unix))): + reasons.append("native_collection_window_mismatch") + selected = manifest["selected_gpu_indices"] + if (not isinstance(selected, list) or not selected or + any(type(i) is not int or i < 0 for i in selected) or + len(set(selected)) != len(selected)): + raise ValueError("invalid_native_gpu_selection") + suffix = "csv" if manifest["vendor"] == "nvidia" else "json" + stem = "gpu_metrics_identity" if suffix == "csv" else "gpu_metrics_devices" + first_path = path.parent / f"{stem}.{suffix}" + last_path = path.parent / f"{stem}_end.{suffix}" + first = _identity(first_path, manifest["vendor"]) + last = _identity(last_path, manifest["vendor"]) + selected_ids = {str(i): first[str(i)] for i in selected} + if any(last.get(i) != uuid for i, uuid in selected_ids.items()): + reasons.append("native_device_identity_changed") + previous = {uuid for members in roles.values() for uuid in members} + if previous.intersection(selected_ids.values()): + reasons.append("native_duplicate_physical_gpu") + roles[role].extend(selected_ids.values()) + csv_path = path.parent / "gpu_metrics.csv" + with csv_path.open(newline="") as stream: + reader = csv.DictReader(stream, skipinitialspace=True) + reader.fieldnames = [c.strip() for c in (reader.fieldnames or [])] + t_col, p_col, g_col = _detect_columns(reader.fieldnames) + if not all((t_col, p_col, g_col)): + raise ValueError("native_telemetry_columns_missing") + for row in reader: + gpu = (row.get(g_col) or "").strip() + if gpu not in first: + reasons.append("native_unknown_device_index") + continue + if gpu not in selected_ids: + continue + timestamp = _parse_timestamp((row.get(t_col) or ""), naive_timezone=timezone.utc) + # Invalid rows are retained for the common validator to reject. + samples.append((timestamp, selected_ids[gpu], row.get(p_col) or "")) + receipts.append({**manifest, "physical_gpu_ids": selected_ids, + "manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "telemetry_sha256": hashlib.sha256(csv_path.read_bytes()).hexdigest(), + "identity_sha256": hashlib.sha256(first_path.read_bytes()).hexdigest(), + "identity_end_sha256": hashlib.sha256(last_path.read_bytes()).hexdigest()}) + except (OSError, ValueError, KeyError, TypeError, csv.Error) as exc: + reasons.append("native_node_invalid") + node_errors.append({"node": path.parent.name, "type": type(exc).__name__, "detail": str(exc)}) + if (len(expected_nodes) != 1 or not expected_nodes or + type(next(iter(expected_nodes), None)) is not int or + set(ranks) != set(range(next(iter(expected_nodes), 0))) or + len(set(nodes)) != len(nodes) or len(ranks) != len(set(ranks))): + reasons.append("native_node_topology_mismatch") + if len(jobs) != 1 or "" in jobs or len(revisions) != 1 or "" in revisions: + reasons.append("native_run_identity_mismatch") + if expected_aggregate_gpus: + if roles["prefill"] or roles["decode"] or len(roles["aggregate"]) != expected_aggregate_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif expected_decode_gpus: + if roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or len(roles["decode"]) != expected_decode_gpus: + reasons.append("native_role_gpu_count_mismatch") + elif roles["decode"] or len(roles["aggregate"] + roles["prefill"]) != expected_prefill_gpus: + reasons.append("native_role_gpu_count_mismatch") + combined_path = validation_result.with_name(f"{validation_result.stem}_native.csv") + combined_path.parent.mkdir(parents=True, exist_ok=True) + with combined_path.open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["timestamp", "gpu", "power"]) + writer.writerows(samples) + integration = None + if benchmark is not None: + integration = integrate_power(combined_path, start_unix=benchmark.start_unix, + end_unix=benchmark.end_unix, expected_num_gpus=expected_gpus) + reasons.extend(integration.invalid_reasons) + metrics = {} + if not reasons and integration is not None and benchmark is not None: + metrics = _derived_metrics(integration, benchmark) + if expected_decode_gpus: + prefill = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["prefill"]) + decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) + metrics.update(prefill_gpu_energy_j=prefill, decode_gpu_energy_j=decode, + prefill_avg_power_w=prefill / benchmark.integration_duration_s / expected_prefill_gpus, + decode_avg_power_w=decode / benchmark.integration_duration_s / expected_decode_gpus, + prefill_joules_per_input_token=prefill / benchmark.total_input_tokens, + decode_joules_per_output_token=decode / benchmark.total_output_tokens) + valid = not reasons + try: + patch_power_metrics(agg_result, metric_keys=ALL_POWER_METRIC_KEYS, power_valid=valid, metrics=metrics) + except (OSError, ValueError): + reasons.append("aggregate_result_unwritable") + valid, metrics = False, {} + audit = {"schema_version": 1, "telemetry_kind": "native_multinode_smi", + "power_valid": valid, "reasons": list(dict.fromkeys(reasons)), + "benchmark_result": str(bench_result), + "benchmark_result_sha256": hashlib.sha256(bench_result.read_bytes()).hexdigest() if bench_result.is_file() else None, + "benchmark_window": benchmark_window_payload(benchmark), + "expected_gpu_count": expected_gpus, "nodes": receipts, + "node_errors": node_errors, + "observed_gpu_count": integration.observed_num_gpus if integration else 0, + "per_gpu_role": {uuid: role for role, uuids in roles.items() for uuid in uuids}, + "per_gpu_sample_counts": integration.per_gpu_sample_counts if integration else {}, + "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s if integration else {}, + "producer": {"name": "inferencex-native-smi", "revisions": sorted(revisions), + "producer_git_commit": next(iter(revisions)) if len(revisions) == 1 else None}, + "integration_method": "per_device_trapezoidal_with_linear_boundary_interpolation", + "power_percentile_method": "time_weighted_synchronized_total_piecewise_linear", + "metrics": metrics} + _write_json_atomic(validation_result, audit) + return int(require_power and not valid) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + begin = sub.add_parser("begin") + begin.add_argument("--directory", type=Path, required=True) + begin.add_argument("--vendor", choices=("amd", "nvidia"), required=True) + begin.add_argument("--node", default=os.environ.get("POWERX_NODE_NAME", socket.gethostname())) + begin.add_argument("--rank", type=int, required=True) + begin.add_argument("--role", choices=("prefill", "decode", "aggregate"), required=True) + begin.add_argument("--gpu-indices", required=True) + begin.add_argument("--num-nodes", type=int, required=True) + begin.add_argument("--job-id", default=os.environ.get("SLURM_JOB_ID", "")) + begin.add_argument("--revision", default=os.environ.get("POWERX_COLLECTOR_REVISION", "")) + begin.add_argument("--clock-synchronized", choices=("true", "false"), default="false") + end = sub.add_parser("end") + end.add_argument("--directory", type=Path, required=True) + end.add_argument("--collector-exit-code", type=int, required=True) + args = vars(parser.parse_args()) + action = args.pop("action") + if action == "begin": + args["gpu_indices"] = [int(i) for i in args["gpu_indices"].split(",")] + args["clock_synchronized"] = args["clock_synchronized"] == "true" + record_begin(**args) + else: + record_end(**args) + + +if __name__ == "__main__": + main() diff --git a/infx/results/power/single_node.py b/infx/results/power/single_node.py index 54669a1a8..4ce1bcaaf 100644 --- a/infx/results/power/single_node.py +++ b/infx/results/power/single_node.py @@ -9,7 +9,11 @@ aggregate and a validation sidecar, but does not fail the benchmark. Power studies can set ``REQUIRE_POWER=1`` to fail after those audit artifacts exist. The aggregate carries numeric ``power_valid`` (1/0) for metric ingestion; the -sidecar is the canonical source for boolean validity and reason codes. +sidecar is the canonical source for boolean validity and reason codes. Rows in +the ingest band but outside the formal window whose power is missing, +non-finite, or <= 0 are teardown noise: they are skipped and counted in the +sidecar's ``boundary_degenerate_rows`` instead of poisoning validity or faking +window bracketing. """ from __future__ import annotations @@ -22,7 +26,7 @@ import os import re import sys -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from statistics import mean @@ -65,6 +69,10 @@ class PowerIntegration: per_gpu_max_sample_gap_s: dict[str, float] per_gpu_energy_j: dict[str, float] device_issues: dict[str, list[str]] + # Rows in the ingest band but outside the formal window whose power was + # missing/N-A/non-finite/<=0, skipped and counted per GPU; "unknown" + # buckets rows without a GPU identity. + boundary_degenerate_rows: dict[str, int] = field(default_factory=dict) avg_power_w: float | None = None p75_power_w: float | None = None p75_total_gpu_power_w: float | None = None @@ -78,7 +86,7 @@ def observed_num_gpus(self) -> int: return len(self.observed_gpu_ids) -def _parse_timestamp(value: str) -> float | None: +def _parse_timestamp(value: str, *, naive_timezone: timezone | None = None) -> float | None: """Best-effort timestamp parse to Unix epoch seconds (local wall clock). Handles the formats observed in practice: @@ -96,7 +104,7 @@ def _parse_timestamp(value: str) -> float | None: # nvidia-smi: "YYYY/MM/DD HH:MM:SS.ffffff" for fmt in ("%Y/%m/%d %H:%M:%S.%f", "%Y/%m/%d %H:%M:%S"): try: - return datetime.strptime(value, fmt).timestamp() + return datetime.strptime(value, fmt).replace(tzinfo=naive_timezone).timestamp() except ValueError: pass # ISO 8601 (amd-smi variants). fromisoformat tolerates 'T' or space separator @@ -108,7 +116,7 @@ def _parse_timestamp(value: str) -> float | None: return None if dt.tzinfo is None: # Treat naive timestamps as local time (matches nvidia-smi convention). - return dt.timestamp() + return dt.replace(tzinfo=naive_timezone).timestamp() return dt.astimezone(timezone.utc).timestamp() @@ -146,6 +154,16 @@ def _detect_columns(header: list[str]) -> tuple[str | None, str | None, str | No return timestamp_col, power_col, gpu_col +def _telemetry_timezone(csv_path: Path) -> timezone | None: + context = csv_path.with_name(f"{csv_path.stem}_context.json") + if not context.is_file(): + return None + payload = json.loads(context.read_text()) + if not isinstance(payload, dict) or payload.get("timestamp_timezone") != "UTC": + raise ValueError("unsupported_telemetry_timezone") + return timezone.utc + + def aggregate_power( csv_path: Path, start_unix: float, @@ -162,6 +180,7 @@ def aggregate_power( return None try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [c.strip() for c in (reader.fieldnames or [])] @@ -189,7 +208,7 @@ def aggregate_power( for row in reader: ts_raw = (row.get(timestamp_col) or "").strip() pw_raw = (row.get(power_col) or "").strip() - ts = _parse_timestamp(ts_raw) + ts = _parse_timestamp(ts_raw, naive_timezone=timestamp_timezone) pw = _parse_power(pw_raw) if ts is None or pw is None: continue @@ -204,7 +223,7 @@ def aggregate_power( if gpu_id: per_sample_gpus.setdefault(bucket, set()).add(gpu_id) gpu_keys.add(gpu_id) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): return None if not per_sample_total: @@ -237,6 +256,7 @@ def _empty_integration( *, expected_num_gpus: int | None, reasons: list[str], + boundary_degenerate_rows: dict[str, int] | None = None, ) -> PowerIntegration: """Build an invalid integration result when no device data is available.""" return PowerIntegration( @@ -248,6 +268,7 @@ def _empty_integration( per_gpu_max_sample_gap_s={}, per_gpu_energy_j={}, device_issues={}, + boundary_degenerate_rows=boundary_degenerate_rows or {}, ) @@ -308,8 +329,10 @@ def integrate_power( # expose timestamps at lower resolution than their sampling cadence, so # duplicate-timestamp readings are averaged rather than treated as corrupt. raw_samples: dict[str, dict[float, list[float]]] = {} + boundary_degenerate: dict[str, int] = {} saw_missing_gpu_identity = False try: + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -332,7 +355,7 @@ def integrate_power( ) for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) if timestamp is None or not math.isfinite(timestamp): _append_reason(reasons, "invalid_timestamp_sample") continue @@ -348,6 +371,15 @@ def integrate_power( power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() + if (power is None or not math.isfinite(power) or power <= 0.0) and ( + timestamp < start_unix or timestamp > end_unix + ): + # SMI teardown rows can carry N/A or 0 W cells: outside the + # formal window they are counted, never used to satisfy + # bracketing or to poison in-window validity. + key = gpu_id or "unknown" + boundary_degenerate[key] = boundary_degenerate.get(key, 0) + 1 + continue if power is None: _append_reason(reasons, "invalid_power_sample") continue @@ -359,11 +391,12 @@ def integrate_power( continue values = raw_samples.setdefault(gpu_id, {}).setdefault(timestamp, []) values.append(power) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): _append_reason(reasons, "telemetry_file_unreadable") return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) if saw_missing_gpu_identity: @@ -373,6 +406,7 @@ def integrate_power( return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) observed_gpu_ids = tuple(sorted(raw_samples, key=_gpu_sort_key)) @@ -450,6 +484,7 @@ def integrate_power( per_gpu_max_sample_gap_s=per_gpu_max_sample_gap_s, per_gpu_energy_j=per_gpu_energy_j, device_issues=device_issues, + boundary_degenerate_rows=boundary_degenerate, avg_power_w=avg_power_w, p75_power_w=p75_total / len(observed_gpu_ids) if p75_total is not None else None, p75_total_gpu_power_w=p75_total, @@ -485,6 +520,7 @@ def _read_energy_snapshot(path: Path) -> dict[str, float] | None: def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float]]]: """Group every parseable (timestamp, watt) sample per GPU, sorted in time.""" samples: dict[str, list[tuple[float, float]]] = {} + timestamp_timezone = _telemetry_timezone(csv_path) with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: reader = csv.DictReader(f, skipinitialspace=True) header = [column.strip() for column in (reader.fieldnames or [])] @@ -493,7 +529,7 @@ def _stream_samples_by_gpu(csv_path: Path) -> dict[str, list[tuple[float, float] if not timestamp_col or not power_col or not gpu_col: return {} for row in reader: - timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip()) + timestamp = _parse_timestamp((row.get(timestamp_col) or "").strip(), naive_timezone=timestamp_timezone) power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() if timestamp is None or power is None or not gpu_id: @@ -679,6 +715,7 @@ def _validation_payload( "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s, "per_gpu_energy_j": integration.per_gpu_energy_j, "device_issues": integration.device_issues, + "boundary_degenerate_rows": integration.boundary_degenerate_rows, "accumulator_check": accumulator_check, "metrics": audit_metrics(metrics), } @@ -754,7 +791,7 @@ def run( try: accumulator_check = cross_check_accumulator(csv_path) - except (OSError, csv.Error): + except (OSError, csv.Error, ValueError): accumulator_check = {"available": False, "reason": "cross_check_error"} try: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f3b85de24..6d321aec8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7410,3 +7410,14 @@ - "Preserve request failure outcomes, retain all available fixed-sequence diagnostics, and reject incomplete concurrency batches before publishing benchmark artifacts." - "保留请求失败状态及全部可用的固定序列诊断信息；发布基准产物前拒绝缺失并发点的批次。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - qwen3.5-fp8-b200-sglang + - qwen3.5-fp8-h100-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Retain UTC SMI telemetry context and validate native multinode collector artifacts without enabling new recipes." + - "保留 UTC SMI 遥测上下文并验证原生多节点采集产物，不启用新 recipe。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3051 diff --git a/runners/test_native_collector_barriers.py b/runners/test_native_collector_barriers.py new file mode 100644 index 000000000..14c8fe2ce --- /dev/null +++ b/runners/test_native_collector_barriers.py @@ -0,0 +1,58 @@ +"""CPU-only lifecycle checks with controlled external collector/Slurm processes.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +LIFECYCLE = ROOT / 'benchmarks/native_power_lifecycle.sh' +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' + + +@pytest.mark.parametrize('failed_rank', [None, 1]) +def test_stop_waits_for_every_collector_and_retains_failure(tmp_path, failed_rank): + command = ''' +source "$1" +export POWERX_CONTROL_DIR="$2" POWERX_NUM_NODES=2 POWERX_BARRIER_TIMEOUT_S=5 +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.1; echo 0 > "$2/done-0") & +POWERX_COLLECTOR_PID=$! +(while [[ ! -f "$2/stop" ]]; do sleep 0.01; done; sleep 0.2; echo "$3" > "$2/done-1") & +remote_pid=$! +powerx_stop_collectors +rc=$? +wait "$remote_pid" +exit "$rc" +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(LIFECYCLE), + str(tmp_path), '1' if failed_rank else '0'], + capture_output=True, text=True, timeout=10) + assert result.returncode == (1 if failed_rank else 0), result.stderr + assert (tmp_path / 'done-0').read_text().strip() == '0' + assert (tmp_path / 'done-1').read_text().strip() == ('1' if failed_rank else '0') + + +def test_ready_barrier_rejects_collector_that_already_stopped(tmp_path): + (tmp_path / 'ready-0').write_text('ready') + (tmp_path / 'done-0').write_text('1') + result = subprocess.run(['bash', '-c', 'source "$1"; POWERX_CONTROL_DIR="$2"; ' + 'POWERX_NUM_NODES=1; powerx_wait_collectors ready', + 'bash', str(LIFECYCLE), str(tmp_path)], capture_output=True, text=True) + assert result.returncode == 1 + assert 'stopped before benchmark readiness' in result.stderr + + +def test_control_receipt_is_owned_before_publication(tmp_path): + command = '''source "$1" +POWERX_CONTROL_DIR=$2 +POWERX_HOST_UID=1000 POWERX_HOST_GID=1000 +chown() { [[ ! -e "$POWERX_CONTROL_DIR/stop" ]]; } +powerx_write_control stop stop +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(LIFECYCLE), str(tmp_path)], + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert (tmp_path / 'stop').read_text() == 'stop\n' + assert not list(tmp_path.glob('*.tmp')) diff --git a/runners/test_native_collector_receipts.py b/runners/test_native_collector_receipts.py new file mode 100644 index 000000000..b981f57ce --- /dev/null +++ b/runners/test_native_collector_receipts.py @@ -0,0 +1,16 @@ +import subprocess +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] + +def test_native_control_receipt_is_owned_before_publication(tmp_path): + source = (ROOT / 'benchmarks/native_power_collect.sh').read_text() + function = source[source.index('write_control() {'):source.index('\nfinish() {')] + result = subprocess.run(['bash', '-c', function + ''' +control_dir=$1 +POWERX_HOST_UID=1000 POWERX_HOST_GID=1000 +chown() { [[ ! -e "$control_dir/done-0" ]]; } +write_control done-0 7 +''', 'bash', str(tmp_path)], capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert (tmp_path / 'done-0').read_text() == '7\n' + assert not list(tmp_path.glob('*.tmp')) diff --git a/utils/test_aggregate_power.py b/utils/test_aggregate_power.py index aa3bf23f9..5b8b76121 100644 --- a/utils/test_aggregate_power.py +++ b/utils/test_aggregate_power.py @@ -574,6 +574,201 @@ def test_run_rejects_malformed_telemetry_inside_window( assert audit["reasons"] == [expected_reason] + +# AMDSMI 26.2.0 `metric -p -c -t -u -w 1 --csv` header (order-faithful subset, +# measured on MI355X; see test_detect_columns_amd_watch_mode_real_header). +_MI355X_WATCH_HEADER = ( + "timestamp,gpu,gfx_activity,umc_activity,mm_activity,vcn_activity," + "jpeg_activity,gfx_busy_inst_xcp_0,jpeg_busy_xcp_0,vcn_busy_xcp_0," + "socket_power,gfx_voltage,soc_voltage,mem_voltage,throttle_status," + "power_management,gfx_0_clk,mem_0_clk,edge,hotspot,mem" +) + + +def _mi355x_watch_row(timestamp: int, gpu: int, socket_power: str) -> str: + """One data row in the shape captured from run 32433563482 (conc1): + integer-second epoch, quoted list cells with embedded commas, N/A cells, + and a trailing carriage return.""" + return ( + f"{timestamp},{gpu},0,0,N/A,\"['N/A', 'N/A', 'N/A', 'N/A']\"," + "\"['N/A', 'N/A']\",\"[0, 0, 0, 0, 0, 0, 0, 0]\",\"[0, 0]\",\"[0, 0]\"," + f"{socket_power},N/A,N/A,N/A,N/A,ENABLED,1404,2000,N/A,40,25\r" + ) + + +def test_integrate_power_skips_na_power_rows_outside_window(tmp_path: Path): + """N/A-power teardown rows past the window end are counted, not poisonous.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-2, 13): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},N/A,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + assert result.total_gpu_energy_j == pytest.approx(10_000.0) + + +def test_integrate_power_does_not_bracket_with_zero_power_tail(tmp_path: Path): + """A 0 W teardown row past the window end must not fake end bracketing. + + Legacy behavior accepted the 0 W row as a valid boundary sample, corrupting + the end interpolation with a bogus value; this intentionally flips that + case to an explicit benchmark_window_not_bracketed failure.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + # Good rows stop at end-4; the only post-end sample per GPU has power 0. + for offset in range(-1, 7): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},0,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "benchmark_window_not_bracketed" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + + +def test_integrate_power_preserves_boundary_counts_without_usable_samples(tmp_path: Path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text( + "timestamp,gpu,socket_power\n" + "1699999999,0,N/A\n" + "1700000011,0,0\n" + "1700000011,1,N/A\n", + encoding="utf-8", + ) + + result = integrate_power( + csv, + start_unix=1_700_000_000, + end_unix=1_700_000_010, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "no_usable_power_samples" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 2, "1": 1} + + +def test_integrate_power_keeps_zero_power_semantics_inside_window(tmp_path: Path): + """Frozen legacy behavior: an in-window 0 W sample still integrates.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-1, 12): + watts = "0.0" if offset == 5 else "500.0" + lines.append(f"{base + offset},0,{watts},65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=1, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {} + # Trapezoids dip to 0 at t=5: 8 x 500 + 2 x 250 = 4500 J. + assert result.total_gpu_energy_j == pytest.approx(4_500.0) + + +def test_run_writes_boundary_degenerate_rows_to_sidecar(tmp_path: Path): + base = 1_700_000_000.0 + csv = tmp_path / "gpu_metrics.csv" + _write_constant_window_samples( + csv, + start=base, + end=base + 10, + watts_per_gpu=500.0, + num_gpus=2, + ) + bench = tmp_path / "bench.json" + agg = tmp_path / "agg.json" + validation = tmp_path / "power_validation.json" + _write_bench_result( + bench, + start=base, + end=base + 10, + duration=10.0, + total_output=2_000, + total_input=10_000, + ) + agg.write_text(json.dumps({"hw": "mi355x"}), encoding="utf-8") + + exit_code = run(csv, bench, agg, expected_num_gpus=2, validation_result=validation) + + assert exit_code == 0 + audit = json.loads(validation.read_text()) + # Present-and-empty for clean streams: readers can rely on the key. + assert audit["boundary_degenerate_rows"] == {} + + +def test_integrate_power_regression_mi355x_integer_ticks_end_gap(tmp_path: Path): + """Run-32433563482 conc1 regression: amd-smi integer-second ticks stop 4 s + before the fractional aiperf window end (last tick 1787277605 vs end + ...609.157497), so bracketing fails and the producer-side telemetry loss is + attributed as benchmark_window_not_bracketed on every GPU. + + The retrieved artifact's trailing rows all carry valid socket_power + (254-264 W) with N/A activity/voltage cells; the N/A- and 0-power teardown + rows appended past the window end are the documented synthetic degenerate + shapes, asserting they are counted rather than used for bracketing.""" + csv = tmp_path / "gpu_metrics.csv" + start = 1_787_277_560.155891 + end = 1_787_277_609.157497 + last_tick = 1_787_277_605 + powers = [259, 255, 263, 264, 256, 254, 259, 259] + lines = [_MI355X_WATCH_HEADER] + for tick in range(1_787_277_555, last_tick + 1): + for gpu in range(8): + lines.append(_mi355x_watch_row(tick, gpu, str(powers[gpu]))) + # amd-smi watch mode emits a blank line between tick groups. + lines.append("") + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_610, gpu, "N/A")) + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_611, gpu, "0")) + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=start, + end_unix=end, + expected_num_gpus=8, + ) + + assert result.power_valid is False + assert result.invalid_reasons == ("benchmark_window_not_bracketed",) + assert result.device_issues == { + str(gpu): ["benchmark_window_not_bracketed"] for gpu in range(8) + } + assert result.boundary_degenerate_rows == {str(gpu): 2 for gpu in range(8)} + + def test_run_patches_agg_with_power_and_joules(tmp_path: Path): base = 1_700_000_000.0 csv = tmp_path / "gpu_metrics.csv" @@ -708,6 +903,7 @@ def test_power_replacement_removes_stale_metrics(tmp_path, patch_validated_power path.write_text(json.dumps({ "hw": "fixture", "avg_power_w": 99, "total_gpu_energy_j": 50, "power_invalid_reasons": ["stale"], + "power_audit": {"source": "previous-run.json"}, })) patch_validated_power(path, power_valid=valid, metrics={ "avg_power_w": 12.34567, "joules_per_output_token": 0.12345678, @@ -1433,12 +1629,12 @@ def test_power_percentiles_uses_synchronized_total_not_device_percentiles(tmp_pa def test_power_percentiles_weights_time_and_clips_the_validated_window(tmp_path): csv_path = tmp_path / "power.csv" - # Dense readings near the high end must not bias a uniform linear ramp. - _write_amd_csv(csv_path, [(0, 0, 0), (1, 0, 100), (1.9, 0, 190), (2, 0, 200)]) + # Dense readings must not bias the uniform 150-250 W ramp inside the window. + _write_amd_csv(csv_path, [(0, 0, 100), (1, 0, 200), (1.9, 0, 290), (2, 0, 300)]) result = integrate_power(csv_path, start_unix=0.5, end_unix=1.5, expected_num_gpus=1) assert result.power_valid - assert result.p75_power_w == pytest.approx(125) - assert result.p90_power_w == pytest.approx(140) + assert result.p75_power_w == pytest.approx(225) + assert result.p90_power_w == pytest.approx(240) def test_power_percentiles_is_withheld_for_invalid_telemetry(tmp_path): @@ -1462,3 +1658,54 @@ def test_power_percentiles_aligns_asynchronous_gpu_samples(tmp_path): assert result.p75_power_w == pytest.approx(300) assert result.p90_total_gpu_power_w == pytest.approx(600) assert result.p90_power_w == pytest.approx(300) + + +@pytest.mark.parametrize('step_name', ['Upload GPU metrics', 'Upload power audit bundle']) +@pytest.mark.parametrize('directory', ['', 'results']) +def test_uploaded_telemetry_replays_in_a_different_timezone(tmp_path, step_name, directory): + import os + import shutil + import yaml + + repo = Path(__file__).resolve().parents[1] + source = tmp_path / 'source' + telemetry = source / directory + telemetry.mkdir(parents=True) + (telemetry / 'gpu_metrics.csv').write_text( + 'timestamp,index,power.draw [W]\n' + '2024/01/01 00:00:00.000,0,100 W\n' + '2024/01/01 00:00:01.000,0,100 W\n' + '2024/01/01 00:00:02.000,0,100 W\n') + (telemetry / 'gpu_metrics_context.json').write_text('{"timestamp_timezone":"UTC"}') + workflow = yaml.safe_load((repo / '.github/workflows/benchmark-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job['steps'] + if step.get('name') == step_name) + archive = tmp_path / 'downloaded' + for pattern in step['with']['path'].splitlines(): + for file in source.glob(pattern): + destination = archive / file.relative_to(source) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(file, destination) + result = subprocess.run( + [sys.executable, '-c', """ +import sys, time +from pathlib import Path +from infx.results.power.single_node import integrate_power +time.tzset() +result = integrate_power(Path(sys.argv[1]), start_unix=1704067200, end_unix=1704067202, + expected_num_gpus=1) +assert result.power_valid, result.invalid_reasons +assert result.total_gpu_energy_j == 200 +""", str(archive / directory / 'gpu_metrics.csv')], + cwd=tmp_path, env={**os.environ, 'TZ': 'Etc/GMT+8', 'PYTHONPATH': str(repo)}, + capture_output=True, text=True, timeout=10, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize('context', ['{"timestamp_timezone":"PST"}', '{}', '{invalid']) +def test_legacy_average_rejects_invalid_telemetry_context(tmp_path, context): + csv = tmp_path / 'gpu_metrics.csv' + csv.write_text('timestamp,index,power.draw [W]\n1,0,100\n2,0,100\n') + csv.with_name('gpu_metrics_context.json').write_text(context) + assert aggregate_power(csv, 1, 2) is None diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py new file mode 100644 index 000000000..0aeee2511 --- /dev/null +++ b/utils/test_native_multinode_power.py @@ -0,0 +1,201 @@ +"""Native collector acceptance uses independent, hand-computed two-node traces.""" +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from infx.results.power.native_multinode import record_begin, record_end, run +from infx.results.power.single_node import integrate_power + +REPO = Path(__file__).resolve().parents[1] + + +def _package(tmp_path, vendor="amd"): + root = tmp_path / "native_power" + for rank, role, watts in ((0, "prefill", 100), (1, "decode", 300)): + node = root / f"node-{rank}" + record_begin(node, vendor=vendor, node=f"host-{rank}", rank=rank, role=role, + gpu_indices=[0], num_nodes=2, job_id="job-123", revision="revision-abc", + clock_synchronized=True) + record_end(node, collector_exit_code=0) + manifest = json.loads((node / "manifest.json").read_text()) + manifest.update(collection_start_unix=0, collection_end_unix=5) + (node / "manifest.json").write_text(json.dumps(manifest)) + for ending in ("", "_end"): + if vendor == "amd": + (node / f"gpu_metrics_devices{ending}.json").write_text(json.dumps([ + {"gpu": 0, "uuid": f"uuid-{rank}"}, {"gpu": 1, "uuid": f"unused-{rank}"}])) + else: + (node / f"gpu_metrics_identity{ending}.csv").write_text( + f"index, uuid, pci.bus_id\n0, uuid-{rank}, 0000:01:00.0\n1, unused-{rank}, 0000:02:00.0\n") + # The spare physical GPU is visible but does not belong to the server. + (node / "gpu_metrics.csv").write_text("timestamp,gpu,power\n" + "".join( + f"{tick},0,{watts}\n{tick},1,900\n" for tick in range(5))) + bench = tmp_path / "result.json" + bench.write_text(json.dumps({"benchmark_start_time_unix": 1, "benchmark_end_time_unix": 3, + "duration": 2, "completed": 2, "total_input_tokens": 20, + "total_output_tokens": 10})) + agg = tmp_path / "agg.json" + agg.write_text(json.dumps({"avg_power_w": 999, "prefill_gpu_energy_j": 999})) + return root, bench, agg + + +@pytest.mark.parametrize("vendor", ["amd", "nvidia"]) +def test_native_whole_fleet_and_role_energy_use_only_serving_devices(tmp_path, vendor): + root, bench, agg = _package(tmp_path, vendor) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 1 + assert actual["total_gpu_energy_j"] == 800 + assert actual["avg_power_w"] == 200 + assert actual["p90_power_w"] == 200 + assert actual["p75_power_w"] == 200 + assert actual["joules_per_successful_query"] == 400 + assert actual["prefill_gpu_energy_j"] == 200 + assert actual["decode_gpu_energy_j"] == 600 + assert actual["prefill_joules_per_input_token"] == 10 + assert actual["decode_joules_per_output_token"] == 60 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert audit["observed_gpu_count"] == 2 + assert audit["per_gpu_role"] == {"uuid-0": "prefill", "uuid-1": "decode"} + assert len(audit["nodes"][0]["telemetry_sha256"]) == 64 + from infx.results.power.audit import audit_summary + summary = audit_summary(audit, "power_validation_result.json")["power_audit"] + assert summary["sample_count"] == 10 # Five samples on each of two participating GPUs. + assert summary["producer_sha"] == "revision-abc" + + +def test_native_parse_failure_has_a_public_reason_and_retained_detail(tmp_path): + from infx.results.power.audit import audit_summary + root, bench, agg = _package(tmp_path) + (root / "node-1/manifest.json").write_text("not JSON") + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + summary = audit_summary(audit, "power_validation_result.json") + assert "native_node_invalid" in summary["power_invalid_reasons"] + assert audit["node_errors"][0]["node"] == "node-1" + assert audit["node_errors"][0]["detail"] + + +@pytest.mark.parametrize(("field", "value", "reason"), [ + ("clock_synchronized", False, "native_clock_not_synchronized"), + ("lifecycle", "collecting", "native_collector_incomplete"), + ("collection_end_unix", 2, "native_collection_window_mismatch"), + ("job_id", "other-job", "native_run_identity_mismatch"), + ("role", "prefill", "native_role_gpu_count_mismatch"), + ("expected_num_nodes", 3, "native_node_topology_mismatch"), +]) +def test_native_invalid_evidence_clears_stale_metrics_and_writes_audit(tmp_path, field, value, reason): + root, bench, agg = _package(tmp_path) + path = root / "node-1/manifest.json" + manifest = json.loads(path.read_text()); manifest[field] = value + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + actual = json.loads(agg.read_text()) + assert actual["power_valid"] == 0 + assert "avg_power_w" not in actual + assert "prefill_gpu_energy_j" not in actual + audit = json.loads((tmp_path / "power_validation_result.json").read_text()) + assert reason in audit["reasons"] + + +def test_native_device_replacement_cannot_preserve_validity(tmp_path): + root, bench, agg = _package(tmp_path) + (root / "node-1/gpu_metrics_devices_end.json").write_text('[{"gpu":0,"uuid":"replacement"}]') + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, require_power=True) == 1 + assert "native_device_identity_changed" in json.loads((tmp_path / "power_validation_result.json").read_text())["reasons"] + + +def test_native_aggregate_nodes_do_not_invent_prefill_decode_metrics(tmp_path): + root, bench, agg = _package(tmp_path) + for path in root.glob("*/manifest.json"): + manifest = json.loads(path.read_text()); manifest["role"] = "aggregate" + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=0, expected_decode_gpus=0, + expected_aggregate_gpus=2, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual["avg_power_w"] == 200 + assert "prefill_gpu_energy_j" not in actual + + +def test_utc_context_replays_in_a_different_timezone(tmp_path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text("timestamp,index,power.draw [W]\n2026/01/01 00:00:00,0,100\n2026/01/01 00:00:02,0,100\n") + (tmp_path / "gpu_metrics_context.json").write_text('{"timestamp_timezone":"UTC"}') + script = "from pathlib import Path; from infx.results.power.single_node import integrate_power; " + \ + f"r=integrate_power(Path({str(csv)!r}),start_unix=1767225600,end_unix=1767225602,expected_num_gpus=1); assert r.power_valid; assert r.total_gpu_energy_j == 200" + subprocess.run([sys.executable, "-c", script], cwd=REPO, + env={**os.environ, "TZ": "America/Los_Angeles"}, check=True, timeout=10) + + +@pytest.mark.parametrize("clock_value, synchronized", [ + ("yes", True), ("true", True), ("no", False), ("false", False), ("", False), +]) +def test_native_supervisor_reaps_monitor_and_writes_completion(tmp_path, clock_value, synchronized): + binary = tmp_path / "bin"; binary.mkdir() + (binary / "python3").symlink_to(sys.executable) + fake = binary / "nvidia-smi" + fake.write_text(f'''#!{sys.executable} +import datetime, os, sys, time +if any("index,uuid" in arg for arg in sys.argv): + print("index, uuid, pci.bus_id"); print("0, gpu-0, 0000:01:00.0") +else: + with open({str(tmp_path / 'monitor.pid')!r}, "w") as stream: stream.write(str(os.getpid())) + if "-l" in sys.argv: print("timestamp,index,power.draw [W]", flush=True) + while True: + print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y/%m/%d %H:%M:%S.%f") + ",0,100", flush=True) + if "-l" not in sys.argv: break + time.sleep(0.1) +''') + fake.chmod(0o755) + control = tmp_path / "control"; control.mkdir() + node = tmp_path / "node-0" + process = subprocess.Popen(["bash", str(REPO / "benchmarks/native_power_collect.sh"), + str(node), str(control), "nvidia", "0", "aggregate", "0", "1"], + env={**os.environ, "PATH": f"{binary}:{os.environ['PATH']}", + "SLURM_JOB_ID": "test-job", "POWERX_COLLECTOR_REVISION": "revision", + "POWERX_CLOCK_SYNCHRONIZED": clock_value}, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + try: + deadline = time.monotonic() + 10 + while not (control / "ready-0").exists(): + assert process.poll() is None + assert time.monotonic() < deadline + time.sleep(0.02) + (control / "stop").write_text("stop") + stdout, stderr = process.communicate(timeout=10) + assert process.returncode == 0, (stdout, stderr) + assert (control / "done-0").read_text().strip() == "0" + manifest = json.loads((node / "manifest.json").read_text()) + assert manifest["lifecycle"] == "complete" + assert manifest["clock_synchronized"] is synchronized + assert (node / "gpu_metrics_identity_end.csv").exists() + finally: + if process.poll() is None: + process.terminate(); process.communicate(timeout=10) + + +def test_result_processor_discovers_staged_native_package(tmp_path, monkeypatch): + from infx.results.fixed_sequence import aggregate_power_result + + root, bench, agg = _package(tmp_path) + logs = tmp_path / "LOGS" + logs.mkdir() + root.rename(logs / "native_power") + monkeypatch.chdir(tmp_path) + env = {"IS_MULTINODE": "true", "PREFILL_GPUS": "1", "DECODE_GPUS": "1", + "RESULT_FILENAME": "result", "REQUIRE_POWER": "1"} + assert aggregate_power_result(env, bench, agg) == 0 + assert json.loads(agg.read_text())["total_gpu_energy_j"] == 800 + assert (tmp_path / "power_validation_result.json").is_file() + + # Two competing formats must not silently select one package. + (logs / "power").mkdir() + assert aggregate_power_result(env, bench, agg) == 1 + invalid = json.loads(agg.read_text()) + assert invalid["power_valid"] == 0 + assert "total_gpu_energy_j" not in invalid From f691e7e7d72b49bd3db08ddbf89f414ddb8cb620 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:30:16 -0700 Subject: [PATCH 03/11] fix: retain per-window SMI context artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留每个测量窗口的 SMI 上下文工件，覆盖原 #3027 的共享上传规则。 --- .github/workflows/benchmark-tmpl.yml | 8 ++++---- utils/test_process_result.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 1da933f63..632401822 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -445,13 +445,13 @@ jobs: name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }} path: | gpu_metrics.csv - gpu_metrics_context.json + gpu_metrics*_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv results/gpu_metrics*.csv - results/gpu_metrics_context.json + results/gpu_metrics*_context.json results/gpu_metrics_identity.json if-no-files-found: ignore @@ -464,14 +464,14 @@ jobs: ${{ env.RESULT_FILENAME }}.json agg_${{ env.RESULT_FILENAME }}.json gpu_metrics.csv - gpu_metrics_context.json + gpu_metrics*_context.json gpu_metrics_energy_start.csv gpu_metrics_energy_end.csv gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json results/gpu_metrics*.csv - results/gpu_metrics_context.json + results/gpu_metrics*_context.json results/gpu_metrics_identity.json results/agentic_power_window.json results/agentic_power_timezone_offset.txt diff --git a/utils/test_process_result.py b/utils/test_process_result.py index d14d6a87a..7bc8dc4d9 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -1623,3 +1623,19 @@ def test_public_power_audit_bounds_text_and_device_identifiers(): assert audit['exporter_image_sha256'] == 'a' * 64 assert len(audit['observed_gpu_ids']) == 1024 assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] + + +@pytest.mark.parametrize('step_name', ['Upload GPU metrics', 'Upload power audit bundle']) +def test_workflow_retains_context_for_each_metrics_csv(tmp_path, step_name): + import yaml + + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == step_name) + patterns = step['with']['path'].splitlines() + for relative in ['gpu_metrics_concurrency_4_context.json', + 'results/gpu_metrics_concurrency_4_context.json']: + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('{"timestamp_timezone":"UTC"}') + assert any(path in tmp_path.glob(pattern.strip()) for pattern in patterns) From b230e7077dc24a58b4773457de911946b4a6bb2f Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:41:07 -0700 Subject: [PATCH 04/11] fix: preserve native role metrics and abort receipts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 独立生成 prefill 与 decode 角色指标，并在 AMD 采集器被回收前完成取消状态记录。 --- benchmarks/benchmark_lib.sh | 2 +- infx/results/power/native_multinode.py | 10 ++-- utils/test_native_multinode_power.py | 76 ++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 9d99a4cc9..ab6699c9f 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -225,7 +225,7 @@ stop_gpu_monitor() { # tick in the same second as the window end still fails bracketing — # the stream needs a tick at the NEXT whole second (measured on MI355X: # end=...153.325 vs last sample ...153.0). - if [[ "$GPU_MONITOR_VENDOR" == "amd" ]]; then + if [[ "$GPU_MONITOR_VENDOR" == "amd" && "${AMD_MONITOR_STOP_TIMEOUT_S:-}" != "0" ]]; then sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 )) fi kill "$GPU_MONITOR_PID" 2>/dev/null diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py index 1ffae5bd9..86b5d683b 100644 --- a/infx/results/power/native_multinode.py +++ b/infx/results/power/native_multinode.py @@ -221,13 +221,15 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, metrics = {} if not reasons and integration is not None and benchmark is not None: metrics = _derived_metrics(integration, benchmark) - if expected_decode_gpus: + if roles["prefill"]: prefill = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["prefill"]) - decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) - metrics.update(prefill_gpu_energy_j=prefill, decode_gpu_energy_j=decode, + metrics.update(prefill_gpu_energy_j=prefill, prefill_avg_power_w=prefill / benchmark.integration_duration_s / expected_prefill_gpus, + prefill_joules_per_input_token=prefill / benchmark.total_input_tokens) + if roles["decode"]: + decode = sum(integration.per_gpu_energy_j[uuid] for uuid in roles["decode"]) + metrics.update(decode_gpu_energy_j=decode, decode_avg_power_w=decode / benchmark.integration_duration_s / expected_decode_gpus, - prefill_joules_per_input_token=prefill / benchmark.total_input_tokens, decode_joules_per_output_token=decode / benchmark.total_output_tokens) valid = not reasons try: diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py index 0aeee2511..c2945cb75 100644 --- a/utils/test_native_multinode_power.py +++ b/utils/test_native_multinode_power.py @@ -199,3 +199,79 @@ def test_result_processor_discovers_staged_native_package(tmp_path, monkeypatch) invalid = json.loads(agg.read_text()) assert invalid["power_valid"] == 0 assert "total_gpu_energy_j" not in invalid + + +@pytest.mark.parametrize('role', ['prefill', 'decode']) +def test_native_single_role_preserves_whole_fleet_and_role_metrics(tmp_path, role): + root, bench, agg = _package(tmp_path) + for path in root.glob('*/manifest.json'): + manifest = json.loads(path.read_text()) + manifest['role'] = role + path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=2 if role == 'prefill' else 0, + expected_decode_gpus=2 if role == 'decode' else 0, require_power=True) == 0 + actual = json.loads(agg.read_text()) + assert actual['total_gpu_energy_j'] == 800 + assert actual[f'{role}_gpu_energy_j'] == 800 + assert actual[f'{role}_avg_power_w'] == 200 + opposite = 'decode' if role == 'prefill' else 'prefill' + assert f'{opposite}_gpu_energy_j' not in actual + assert json.loads((tmp_path / 'power_validation_result.json').read_text())['power_valid'] + + +def test_native_amd_abort_skips_legacy_tail_wait(tmp_path): + result = subprocess.run(['bash', '-c', '''source "$1" +kill() { return 0; }; wait() { return 0; } +sleep() { echo unexpected-tail-wait >&2; } +_write_amd_smi_sidecar() { return 0; } +GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd AMD_MONITOR_STOP_TIMEOUT_S=0 +GPU_METRICS_CSV="$2/missing.csv" +stop_gpu_monitor +''', 'bash', str(REPO / 'benchmarks/benchmark_lib.sh'), str(tmp_path)], + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert 'unexpected-tail-wait' not in result.stderr + + +def test_native_amd_abort_publishes_receipt_before_reaper_deadline(tmp_path): + binary = tmp_path / 'bin' + binary.mkdir() + (binary / 'python3').symlink_to(sys.executable) + fake = binary / 'amd-smi' + fake.write_text(f'''#!{sys.executable} +import sys, time +if "-w" in sys.argv: + print("timestamp,gpu,socket_power", flush=True) + while True: + print(str(int(time.time())) + ",0,100", flush=True) + time.sleep(0.1) +else: + print("[]") +''') + fake.chmod(0o755) + control = tmp_path / 'control' + control.mkdir() + node = tmp_path / 'node-0' + process = subprocess.Popen(['bash', '-c', '\n'.join([ + 'source "$1"', + 'bash "$4" "$2" "$3" amd 0 aggregate 0 1 &', + 'POWERX_COLLECTOR_PID=$! POWERX_CONTROL_DIR=$3 POWERX_NUM_NODES=1', + 'POWERX_BARRIER_TIMEOUT_S=5', + 'powerx_wait_collectors ready || exit 1', + 'POWERX_BARRIER_TIMEOUT_S=0', + 'powerx_reap_collector', + ]), 'bash', str(REPO / 'benchmarks/native_power_lifecycle.sh'), str(node), str(control), str(REPO / 'benchmarks/native_power_collect.sh')], + env={**os.environ, 'PATH': f'{binary}:/usr/bin:/bin', 'SLURM_JOB_ID': 'test-job'}, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) + try: + stdout, stderr = process.communicate(timeout=8) + assert process.returncode == 143, (stdout, stderr) + assert (control / 'done-0').read_text().strip() == '143' + assert json.loads((node / 'manifest.json').read_text())['lifecycle'] == 'failed' + finally: + import signal + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() From 880f1153b489da0965261ededb090932d5a374dd Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:43:51 -0700 Subject: [PATCH 05/11] ci: run native collector contract regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在现有 CPU 工作流中执行原生采集契约回归，并跟踪对应文件变更。 --- .github/workflows/test-process-result.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 4d4cf0adf..7d5071770 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -10,6 +10,11 @@ on: - '.github/workflows/e2e-tests.yml' - '.github/workflows/test-process-result.yml' - 'benchmarks/benchmark_lib.sh' + - 'benchmarks/native_power_collect.sh' + - 'benchmarks/native_power_lifecycle.sh' + - 'runners/test_native_collector_barriers.py' + - 'runners/test_native_collector_receipts.py' + - 'utils/test_native_multinode_power.py' - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/**/*.yaml' - 'benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/**/*.yaml' - 'benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/**/*.yaml' @@ -57,4 +62,4 @@ jobs: run: | cd utils uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \ - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py -v From 654c33b2f71584f81a4c11a65e97a3dbd0fc41a9 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 02:48:30 -0700 Subject: [PATCH 06/11] test: synchronize shared replay signal readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将已验证的信号就绪屏障移至共享原生采集前置，避免重放 exec 前的竞态。 --- utils/agentic/aggregation/test_power_lifecycle.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/utils/agentic/aggregation/test_power_lifecycle.py b/utils/agentic/aggregation/test_power_lifecycle.py index 20710e68e..6d955af12 100644 --- a/utils/agentic/aggregation/test_power_lifecycle.py +++ b/utils/agentic/aggregation/test_power_lifecycle.py @@ -280,8 +280,13 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( }} stop_gpu_monitor() {{ printf 'monitor-stop\n' >> {str(event_log)!r}; }} fake_replay() {{ - printf 'replay-ready\n' >> {str(event_log)!r} - exec sleep 30 + exec {sys.executable!r} -c ' +import signal, sys, time +signal.signal(signal.SIGINT, signal.SIG_DFL) +signal.signal(signal.SIGTERM, signal.SIG_DFL) +print("replay-ready", file=open(sys.argv[1], "a"), flush=True) +time.sleep(30) +' {str(event_log)!r} }} trap 'printf "parent-exit\\n" >> {str(event_log)!r}' EXIT trap 'printf "parent-int\\n" >> {str(event_log)!r}; exit 130' INT @@ -301,7 +306,8 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( ) try: # The monitor starts before the production signal traps are installed. - # Wait for replay so the signal actually exercises those traps. + # Publish readiness from the execed process after restoring signal handling; + # a shell marker before exec races with the group SIGINT. deadline = time.monotonic() + 5 while time.monotonic() < deadline: if event_log.exists() and "replay-ready" in event_log.read_text().splitlines(): From d141e71c8be2bb6c57c37a2a623bfb6c274e5921 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:02:16 -0700 Subject: [PATCH 07/11] fix: preserve diagnostic sidecars and legacy result processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 只忽略已定义的诊断 sidecar，保留损坏与缺失点门禁；旧处理器逐点处理并累计失败，新版导入错误不回退。 --- .../workflows/benchmark-multinode-tmpl.yml | 42 ++++++++- infx/results/fixed_sequence.py | 9 +- utils/test_process_result.py | 87 +++++++++++++++++++ 3 files changed, 136 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 80b0842f8..d26d4e0bb 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -480,7 +480,47 @@ jobs: export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)" echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA" fi - python3 utils/process_result.py --all + # Older measured refs predate the batch CLI. Import failures in a + # present processor are errors, not a reason to downgrade validation. + has_batch=$(python3 - <<'PY' + import importlib + from pathlib import Path + if Path('infx/results/fixed_sequence.py').is_file(): + module = importlib.import_module('infx.results.fixed_sequence') + print(int(hasattr(module, 'process_multinode_results'))) + else: + print(0) + PY + ) + if [ "$has_batch" = 1 ]; then + python3 utils/process_result.py --all + else + batch_rc=0 + observed=() + for result_file in "${RESULT_FILENAME}"_*.json; do + [ -f "$result_file" ] || continue + case "$result_file" in + *.pytorch.json|"${RESULT_FILENAME}_gpu_metrics_context.json"|"${RESULT_FILENAME}_gpu_metrics_identity.json") continue ;; + esac + if [[ "$result_file" =~ _(c|conc|concurrency_)([0-9]+)(_req_rate_[^_]+)?_gpus_([0-9]+)(_ctx_([0-9]+)_gen_([0-9]+))?\.json$ ]]; then + observed+=("${BASH_REMATCH[2]}") + gpus=${BASH_REMATCH[4]} + prefill_gpus=${BASH_REMATCH[6]:-$gpus} + decode_gpus=${BASH_REMATCH[7]:-0} + RESULT_FILENAME=${result_file%.json} IS_MULTINODE=true PREFILL_GPUS="$prefill_gpus" DECODE_GPUS="$decode_gpus" \ + python3 utils/process_result.py || batch_rc=1 + else + echo "Unrecognized result filename: $result_file" >&2 + batch_rc=1 + fi + done + read -r -a expected <<< "$CONC_LIST" + if [[ "$(printf '%s\n' "${observed[@]}" | sort -n)" != "$(printf '%s\n' "${expected[@]}" | sort -n)" ]]; then + echo 'Legacy result batch has missing, duplicate or unexpected concurrencies' >&2 + batch_rc=1 + fi + exit "$batch_rc" + fi - name: Upload result if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} diff --git a/infx/results/fixed_sequence.py b/infx/results/fixed_sequence.py index f6fa76102..3db53a3b7 100644 --- a/infx/results/fixed_sequence.py +++ b/infx/results/fixed_sequence.py @@ -318,9 +318,16 @@ def process_multinode_results(env: Mapping[str, str]) -> int: if not expected or min(expected) <= 0: raise ValueError('CONC_LIST must contain positive concurrencies') points: list[dict[str, Any]] = [] + ignored_sidecars: list[str] = [] observed: set[int] = set() status = 0 for path in sorted(Path('.').glob(f"{env['RESULT_FILENAME']}_*.json")): + if path.name.endswith('.pytorch.json') or path.name in { + f"{env['RESULT_FILENAME']}_gpu_metrics_context.json", + f"{env['RESULT_FILENAME']}_gpu_metrics_identity.json", + }: + ignored_sidecars.append(path.name) + continue point: dict[str, Any] = {'source': path.name} try: match = re.search( @@ -364,7 +371,7 @@ def process_multinode_results(env: Mapping[str, str]) -> int: status = max(status, int(bool(missing or unexpected))) summary = {'expected_concurrencies': sorted(expected), 'missing_concurrencies': missing, 'unexpected_concurrencies': unexpected, - 'points': points, 'exit_code': status} + 'points': points, 'ignored_sidecars': ignored_sidecars, 'exit_code': status} Path(f"result_processing_{env['RESULT_FILENAME']}.json").write_text(json.dumps(summary, indent=2)) if missing or unexpected: print(f'[process_result] incomplete sweep: missing={missing}, unexpected={unexpected}', file=sys.stderr) diff --git a/utils/test_process_result.py b/utils/test_process_result.py index d14d6a87a..2b394e3bb 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -1623,3 +1623,90 @@ def test_public_power_audit_bounds_text_and_device_identifiers(): assert audit['exporter_image_sha256'] == 'a' * 64 assert len(audit['observed_gpu_ids']) == 1024 assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1'] + + +@pytest.mark.parametrize('sidecar', ['run_recipe_conc4_gpus_4_ctx_2_gen_2.pytorch.json', + 'run_gpu_metrics_context.json', 'run_gpu_metrics_identity.json']) +@pytest.mark.parametrize('point_state', ['valid', 'missing', 'malformed']) +def test_multinode_batch_retains_sidecars_without_counting_them_as_points( + tmp_path, multinode_env_vars, sample_benchmark_result, sidecar, point_state, +): + (tmp_path / sidecar).write_text('{"diagnostic": true}') + if point_state != 'missing': + (tmp_path / 'run_recipe_conc4_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({**sample_benchmark_result, 'max_concurrency': 4}) + if point_state == 'valid' else '{broken') + env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4', + 'PYTHONPATH': str(REPO_ROOT)} + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, env=env, + capture_output=True, text=True) + assert result.returncode == int(point_state != 'valid'), result.stderr + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['ignored_sidecars'] == [sidecar] + assert receipt['missing_concurrencies'] == ([] if point_state == 'valid' else [4]) + assert (tmp_path / sidecar).read_text() == '{"diagnostic": true}' + + +def test_multinode_batch_rejects_unknown_point_filename( + tmp_path, multinode_env_vars, sample_benchmark_result, +): + for name in ['run_conc4_gpus_4_ctx_2_gen_2.json', 'run_conc16_gpus_bad.json']: + (tmp_path / name).write_text(json.dumps({**sample_benchmark_result, 'max_concurrency': 4})) + result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, + env={**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', + 'CONC_LIST': '4 16', 'PYTHONPATH': str(REPO_ROOT)}, + capture_output=True, text=True) + assert result.returncode == 1 + receipt = json.loads((tmp_path / 'result_processing_run.json').read_text()) + assert receipt['missing_concurrencies'] == [16] + assert any('filename lacks' in point.get('error', '') for point in receipt['points']) + + +@pytest.mark.parametrize('first_rc', [0, 7]) +@pytest.mark.parametrize('expected_concs', ['4 16', '4 8 16', '4']) +def test_multinode_workflow_processes_every_point_with_legacy_processor(tmp_path, first_rc, expected_concs): + (tmp_path / 'utils').mkdir() + (tmp_path / 'utils/process_result.py').write_text('''import json, os, sys +from pathlib import Path +stem = os.environ['RESULT_FILENAME'] +data = json.loads(Path(stem + '.json').read_text()) +Path('agg_' + stem + '.json').write_text(json.dumps(data)) +sys.exit(data['exit_code']) +''') + for conc, rc in [(4, first_rc), (16, 0)]: + (tmp_path / f'run_conc{conc}_gpus_4_ctx_2_gen_2.json').write_text( + json.dumps({'exit_code': rc})) + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == 'Process result') + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step['run']], cwd=tmp_path, + env={**os.environ, 'RESULT_FILENAME': 'run', 'POWER_PRODUCER_SHA': '', + 'CONC_LIST': expected_concs, 'PYTHONPATH': '', + 'PATH': f"{Path(sys.executable).parent}:{os.environ['PATH']}"}, + capture_output=True, text=True, timeout=10) + assert result.returncode == int(bool(first_rc) or expected_concs != '4 16'), result.stderr + for conc in [4, 16]: + assert (tmp_path / f'agg_run_conc{conc}_gpus_4_ctx_2_gen_2.json').is_file() + + +def test_multinode_workflow_does_not_downgrade_processor_import_errors(tmp_path): + (tmp_path / 'infx/results').mkdir(parents=True) + (tmp_path / 'infx/__init__.py').touch() + (tmp_path / 'infx/results/__init__.py').touch() + (tmp_path / 'infx/results/fixed_sequence.py').write_text( + 'raise RuntimeError("broken processor import")\n') + (tmp_path / 'utils').mkdir() + (tmp_path / 'utils/process_result.py').write_text( + 'from pathlib import Path\nPath("legacy-called").touch()\n') + (tmp_path / 'run_conc4_gpus_4.json').write_text('{}') + workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml').read_text()) + step = next(step for job in workflow['jobs'].values() for step in job.get('steps', []) + if step.get('name') == 'Process result') + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step['run']], cwd=tmp_path, + env={**os.environ, 'RESULT_FILENAME': 'run', 'POWER_PRODUCER_SHA': '', + 'CONC_LIST': '4', 'PYTHONPATH': '', + 'PATH': f"{Path(sys.executable).parent}:{os.environ['PATH']}"}, + capture_output=True, text=True, timeout=10) + assert result.returncode != 0 + assert 'broken processor import' in result.stderr + assert not (tmp_path / 'legacy-called').exists() From 8408779c22817e681238ac323f7b4c4ad34977b0 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:02:55 -0700 Subject: [PATCH 08/11] test: preserve existing result test section formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 恢复合并时意外改动的测试注释分隔线与空行。 --- utils/test_process_result.py | 42 ++++++++++++------------------------ 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 350856ea7..9d3ba8ddf 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -155,11 +155,9 @@ def test_fixed_topology_rejects_empty_parallelism( build_result(sample_benchmark_result, {**single_node_env_vars, name: ""}) -# ====================================================================== - +# ============================================================================= # Test Fixtures - Based on real benchmark output structure -# ====================================================================== - +# ============================================================================= @pytest.fixture def sample_benchmark_result(): @@ -298,11 +296,9 @@ def failing_import(name, *args, **kwargs): ) -# ====================================================================== - +# ============================================================================= # Test script execution via subprocess -# ====================================================================== - +# ============================================================================= class TestProcessResultScript: """Tests for process_result.py script execution.""" @@ -532,11 +528,9 @@ def test_missing_result_file(self, tmp_path, single_node_env_vars): assert result.returncode != 0 -# ====================================================================== - +# ============================================================================= # Test latency and throughput calculations -# ====================================================================== - +# ============================================================================= class TestCalculations: """Tests for throughput and latency calculations.""" @@ -665,11 +659,9 @@ def test_multinode_zero_total_gpus_fails(self, tmp_path, sample_benchmark_result assert "Multinode results require at least one GPU" in result.stderr -# ====================================================================== - +# ============================================================================= # Test output file generation -# ====================================================================== - +# ============================================================================= class TestOutputFile: """Tests for output file generation.""" @@ -698,11 +690,9 @@ def test_output_file_has_correct_prefix(self, tmp_path, sample_benchmark_result, assert output_file.exists() -# ====================================================================== - +# ============================================================================= # Test edge cases -# ====================================================================== - +# ============================================================================= class TestEdgeCases: """Tests for edge cases and special scenarios.""" @@ -751,11 +741,9 @@ def test_integer_conversion(self, tmp_path, single_node_env_vars): assert isinstance(output_data["isl"], int) assert isinstance(output_data["osl"], int) -# ====================================================================== - +# ============================================================================= # Integration: power aggregation patches the agg JSON -# ====================================================================== - +# ============================================================================= class TestPowerAggregationIntegration: """End-to-end wiring: process_result.py invokes aggregate_power.py and @@ -1335,11 +1323,9 @@ def test_start_stop_gpu_monitor_amd_lifecycle(self, tmp_path): assert identity == {"gpu_data": []} -# ====================================================================== - +# ============================================================================= # Integration: multinode power aggregation patches the agg JSON -# ====================================================================== - +# ============================================================================= class TestMultinodePower: From 6ed8f8b5ed69e71fdd6972ceaeaa8d97a160f347 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:05:34 -0700 Subject: [PATCH 09/11] fix: validate native role counts and retain boundary sample audits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 独立核对每种角色的 GPU 数量，并记录窗口外异常样本，窗口内异常仍阻止功耗有效性。 --- infx/results/power/native_multinode.py | 7 +++--- utils/test_native_multinode_power.py | 30 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/infx/results/power/native_multinode.py b/infx/results/power/native_multinode.py index 86b5d683b..78ff89889 100644 --- a/infx/results/power/native_multinode.py +++ b/infx/results/power/native_multinode.py @@ -202,10 +202,8 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, if expected_aggregate_gpus: if roles["prefill"] or roles["decode"] or len(roles["aggregate"]) != expected_aggregate_gpus: reasons.append("native_role_gpu_count_mismatch") - elif expected_decode_gpus: - if roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or len(roles["decode"]) != expected_decode_gpus: - reasons.append("native_role_gpu_count_mismatch") - elif roles["decode"] or len(roles["aggregate"] + roles["prefill"]) != expected_prefill_gpus: + elif (roles["aggregate"] or len(roles["prefill"]) != expected_prefill_gpus or + len(roles["decode"]) != expected_decode_gpus): reasons.append("native_role_gpu_count_mismatch") combined_path = validation_result.with_name(f"{validation_result.stem}_native.csv") combined_path.parent.mkdir(parents=True, exist_ok=True) @@ -247,6 +245,7 @@ def run(power_dir: Path, bench_result: Path, agg_result: Path, *, "observed_gpu_count": integration.observed_num_gpus if integration else 0, "per_gpu_role": {uuid: role for role, uuids in roles.items() for uuid in uuids}, "per_gpu_sample_counts": integration.per_gpu_sample_counts if integration else {}, + "boundary_degenerate_rows": integration.boundary_degenerate_rows if integration else {}, "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s if integration else {}, "producer": {"name": "inferencex-native-smi", "revisions": sorted(revisions), "producer_git_commit": next(iter(revisions)) if len(revisions) == 1 else None}, diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py index c2945cb75..e4234facc 100644 --- a/utils/test_native_multinode_power.py +++ b/utils/test_native_multinode_power.py @@ -275,3 +275,33 @@ def test_native_amd_abort_publishes_receipt_before_reaper_deadline(tmp_path): except ProcessLookupError: pass process.communicate() + + +def test_native_prefill_only_rejects_aggregate_role_devices(tmp_path): + root, bench, agg = _package(tmp_path) + manifest_path = root / 'node-1/manifest.json' + manifest = json.loads(manifest_path.read_text()) + manifest['role'] = 'aggregate' + manifest_path.write_text(json.dumps(manifest)) + assert run(root, bench, agg, expected_prefill_gpus=2, expected_decode_gpus=0, + require_power=True) == 1 + audit = json.loads((tmp_path / 'power_validation_result.json').read_text()) + assert 'native_role_gpu_count_mismatch' in audit['reasons'] + assert json.loads(agg.read_text())['power_valid'] == 0 + assert 'prefill_avg_power_w' not in json.loads(agg.read_text()) + + +@pytest.mark.parametrize('tick', [2, 4]) +def test_native_audit_retains_boundary_noise_without_relaxing_in_window_errors(tmp_path, tick): + root, bench, agg = _package(tmp_path) + for rank in [0, 1]: + with (root / f'node-{rank}/gpu_metrics.csv').open('a') as stream: + stream.write(f'{tick},0,N/A\n') + if rank == 0: + stream.write(f'{tick},0,0\n') + assert run(root, bench, agg, expected_prefill_gpus=1, expected_decode_gpus=1, + require_power=True) == int(tick == 2) + audit = json.loads((tmp_path / 'power_validation_result.json').read_text()) + assert audit['boundary_degenerate_rows'] == ({'uuid-0': 2, 'uuid-1': 1} if tick == 4 else {}) + if tick == 4: + assert json.loads(agg.read_text())['total_gpu_energy_j'] == 800 From 85384dd3ba00223513ec5ef65a2f27b422b8c7ae Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 03:32:14 -0700 Subject: [PATCH 10/11] fix: validate native collector readiness and identity completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 发布就绪收据前检查采集进程仍存活，并将 AMD 结束身份采集失败写入失败收据。补充真实进程与清单回归测试。 --- benchmarks/native_power_collect.sh | 3 +- runners/test_native_collector_receipts.py | 77 +++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/benchmarks/native_power_collect.sh b/benchmarks/native_power_collect.sh index 69df07951..445d00ac3 100644 --- a/benchmarks/native_power_collect.sh +++ b/benchmarks/native_power_collect.sh @@ -35,7 +35,7 @@ finish() { if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi stop_gpu_monitor if [[ "$vendor" == amd ]]; then - _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices_end.json" list --json + amd-smi list --json > "$power_dir/gpu_metrics_devices_end.json" || collector_rc=1 else nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \ > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1 @@ -65,6 +65,7 @@ start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 if [[ "$vendor" == amd ]]; then _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json fi +_background_process_is_running "$GPU_MONITOR_PID" || exit 1 write_control "ready-$rank" ready while [[ ! -f "$control_dir/stop" ]]; do _background_process_is_running "$GPU_MONITOR_PID" || exit 1 diff --git a/runners/test_native_collector_receipts.py b/runners/test_native_collector_receipts.py index b981f57ce..63d9de677 100644 --- a/runners/test_native_collector_receipts.py +++ b/runners/test_native_collector_receipts.py @@ -14,3 +14,80 @@ def test_native_control_receipt_is_owned_before_publication(tmp_path): assert result.returncode == 0, result.stderr assert (tmp_path / 'done-0').read_text() == '7\n' assert not list(tmp_path.glob('*.tmp')) + + +def _collector_with_monitor(tmp_path, *, alive, end_identity_rc=0): + import shutil + import sys + + scripts = tmp_path / 'repo/benchmarks' + scripts.mkdir(parents=True) + shutil.copyfile(ROOT / 'benchmarks/native_power_collect.sh', scripts / 'native_power_collect.sh') + library = (ROOT / 'benchmarks/benchmark_lib.sh').read_text() + functions = [] + for name in ['_background_process_is_running', '_write_amd_smi_sidecar']: + start = library.index(name + '() {') + functions.append(library[start:library.index('\n}', start) + 2]) + (scripts / 'benchmark_lib.sh').write_text('\n'.join(functions) + ''' +GPU_MONITOR_PID="" GPU_MONITOR_VENDOR="" +start_gpu_monitor() { + GPU_MONITOR_VENDOR=amd + if [[ "$MONITOR_ALIVE" == 1 ]]; then + sleep 30 & + GPU_MONITOR_PID=$! + else + false & + GPU_MONITOR_PID=$! + wait "$GPU_MONITOR_PID" || true + fi +} +stop_gpu_monitor() { + kill "$GPU_MONITOR_PID" 2>/dev/null || true + wait "$GPU_MONITOR_PID" 2>/dev/null || true +} +amd-smi() { + if [[ -f "$IDENTITY_CALLED" && "$END_IDENTITY_RC" != 0 ]]; then return "$END_IDENTITY_RC"; fi + touch "$IDENTITY_CALLED" + printf '[{"gpu":0,"uuid":"test-gpu"}]\\n' +} +''') + control = tmp_path / 'control' + control.mkdir() + (control / 'stop').touch() + power = tmp_path / 'power' + result = subprocess.run(['bash', str(scripts / 'native_power_collect.sh'), str(power), + str(control), 'amd', '0', 'prefill', '0', '1'], + env={'PATH': f'{Path(sys.executable).parent}:/usr/bin:/bin', + 'PYTHONPATH': str(ROOT), 'MONITOR_ALIVE': str(int(alive)), + 'END_IDENTITY_RC': str(end_identity_rc), + 'IDENTITY_CALLED': str(tmp_path / 'identity-called')}, + capture_output=True, text=True, timeout=5) + return result, control, power + + +def test_dead_monitor_never_publishes_ready(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=False) + assert not (control / 'ready-0').exists(), result.stderr + assert (control / 'done-0').read_text().strip() != '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'failed' + + +def test_amd_end_identity_failure_is_not_a_successful_done_receipt(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=True, end_identity_rc=7) + assert (control / 'ready-0').exists(), result.stderr + assert (control / 'done-0').read_text().strip() != '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'failed' + + +def test_live_monitor_with_successful_identity_completes(tmp_path): + import json + + result, control, power = _collector_with_monitor(tmp_path, alive=True) + assert result.returncode == 0, result.stderr + assert (control / 'ready-0').exists() + assert (control / 'done-0').read_text().strip() == '0' + assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'complete' From 583c388b405b2353b597038b70bcc4d40ae3c531 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 13:26:27 -0700 Subject: [PATCH 11/11] fix: confine native telemetry setup to its collector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 UTC 上下文限制在原生采集器，保留常规监控行为，并为取消后的尾部采样和失败回执预留有界清理时间。 --- benchmarks/benchmark_lib.sh | 9 ++--- benchmarks/native_power_collect.sh | 2 + benchmarks/native_power_lifecycle.sh | 6 ++- docs/results-and-ingestion.md | 2 +- docs/results-and-ingestion_zh.md | 2 +- perf-changelog.yaml | 11 ------ runners/test_native_collector_receipts.py | 47 +++++++++++++++++++++++ utils/test_native_multinode_power.py | 14 ------- 8 files changed, 60 insertions(+), 33 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index ab6699c9f..305a118ef 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -177,7 +177,6 @@ start_gpu_monitor() { GPU_METRICS_CSV="$output" GPU_MONITOR_INTERVAL="$interval" export GPU_METRICS_CSV - printf '{"timestamp_timezone":"UTC"}\n' > "${output%.csv}_context.json" if command -v nvidia-smi &>/dev/null; then GPU_MONITOR_VENDOR="nvidia" @@ -186,7 +185,7 @@ start_gpu_monitor() { rm -f "${output%.csv}_identity.csv" echo "[GPU Monitor] Warning: NVIDIA identity sidecar failed" >&2 fi - TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv -l "$interval" > "$output" 2>/dev/null & GPU_MONITOR_PID=$! echo "[GPU Monitor] Started NVIDIA (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)" @@ -197,7 +196,7 @@ start_gpu_monitor() { # Python; measured on MI355X: trailing ticks were lost at kill without it). # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated # headers, and flush every row so killing the pipe cannot discard buffered samples. - TZ=UTC PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ + PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ | awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" & GPU_MONITOR_PID=$! # Hardware energy-accumulator + identity snapshots; the end-side twin in @@ -225,7 +224,7 @@ stop_gpu_monitor() { # tick in the same second as the window end still fails bracketing — # the stream needs a tick at the NEXT whole second (measured on MI355X: # end=...153.325 vs last sample ...153.0). - if [[ "$GPU_MONITOR_VENDOR" == "amd" && "${AMD_MONITOR_STOP_TIMEOUT_S:-}" != "0" ]]; then + if [[ "$GPU_MONITOR_VENDOR" == "amd" ]]; then sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 )) fi kill "$GPU_MONITOR_PID" 2>/dev/null @@ -233,7 +232,7 @@ stop_gpu_monitor() { case "$GPU_MONITOR_VENDOR" in nvidia) if _repair_truncated_gpu_metrics_tail; then - TZ=UTC nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ + nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \ --format=csv,noheader >> "$GPU_METRICS_CSV" 2>/dev/null || echo "[GPU Monitor] Warning: final NVIDIA sample failed" >&2 fi diff --git a/benchmarks/native_power_collect.sh b/benchmarks/native_power_collect.sh index 445d00ac3..058b8aeb2 100644 --- a/benchmarks/native_power_collect.sh +++ b/benchmarks/native_power_collect.sh @@ -10,6 +10,7 @@ role=$5 gpu_indices=$6 num_nodes=$7 repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export TZ=UTC export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}" source "$repo_root/benchmarks/benchmark_lib.sh" mkdir -p "$power_dir" @@ -60,6 +61,7 @@ esac python3 -m infx.results.power.native_multinode begin --directory "$power_dir" \ --vendor "$vendor" --rank "$rank" --role "$role" --gpu-indices "$gpu_indices" \ --num-nodes "$num_nodes" --clock-synchronized "$clock_synchronized" || exit 1 +printf '{"timestamp_timezone":"UTC"}\n' > "$power_dir/gpu_metrics_context.json" || exit 1 start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1 [[ "$GPU_MONITOR_VENDOR" == "$vendor" ]] || exit 1 if [[ "$vendor" == amd ]]; then diff --git a/benchmarks/native_power_lifecycle.sh b/benchmarks/native_power_lifecycle.sh index b24d1dada..d38a0cc3f 100644 --- a/benchmarks/native_power_lifecycle.sh +++ b/benchmarks/native_power_lifecycle.sh @@ -64,7 +64,11 @@ powerx_reap_collector() { while kill -0 "$POWERX_COLLECTOR_PID" 2>/dev/null; do if (( SECONDS >= deadline )); then kill -TERM "$POWERX_COLLECTOR_PID" 2>/dev/null || true - sleep 2 + # The shared AMD monitor drains for three seconds before writing receipts. + local grace_deadline=$((SECONDS + 5)) + while kill -0 "$POWERX_COLLECTOR_PID" 2>/dev/null && (( SECONDS < grace_deadline )); do + sleep 1 + done kill -KILL "$POWERX_COLLECTOR_PID" 2>/dev/null || true rc=1 break diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index 60338f625..725cac79d 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -112,7 +112,7 @@ Processing and diagnostic power-audit uploads run after launcher or validation f `native_power_collect.sh` and `native_power_lifecycle.sh` provide per-node collection and bounded ready/stop receipts. Launchers opt into the native package under `LOGS/native_power`; this prerequisite enables no new recipe. The adapter validates serving GPU identity, synchronized clocks, collector completion, and complete formal-window coverage. It preserves per-node failures, sample counts, and collector revision in the audit. -SMI collection records UTC context beside each CSV for portable replay. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts. +The native collector sets UTC and records context beside its CSV for portable replay; existing benchmark monitors keep their current behavior. Its launcher integration requires separate hardware qualification. The offline adapter accepts this context without changing producers. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts. ## Eval artifacts diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index 656e53595..7d6ec24a0 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -112,7 +112,7 @@ PR changelog 选择具有代表性的 NVIDIA 和 AMD 覆盖，并非所有受影 `native_power_collect.sh` 和 `native_power_lifecycle.sh` 提供每节点采集及有时限的就绪/停止状态文件。启动器可使用 `LOGS/native_power` 下的原生产物；此前置改动不会启用新 recipe。适配器验证服务 GPU 身份、时钟同步、采集完成及正式窗口完整覆盖，并在审计中保留节点故障、样本数和采集器版本。 -SMI 采集在 CSV 旁记录 UTC 上下文以支持跨环境回放。正式窗口外的无效样本不能构成覆盖；`boundary_degenerate_rows` 保留其逐 GPU 计数。 +原生采集器单独设置 UTC，并在 CSV 旁记录上下文以支持跨环境回放；现有基准监控行为保持不变。启动器接入需要另行完成硬件验证。离线适配器接受该上下文，不改变现有生产端。正式窗口外的无效样本不能构成覆盖；`boundary_degenerate_rows` 保留其逐 GPU 计数。 ## 评测工件 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3936a0c1d..75b77afe1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7428,14 +7428,3 @@ - "Add representative AMD coverage for fixed-sequence request diagnostics and multinode concurrency filename processing; shared processor changes also affect unlisted fixed-sequence recipes." - "增加固定序列请求诊断与多节点并发文件名处理的代表性 AMD 覆盖；共享处理器变更也影响未列出的固定序列配置。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 - -- config-keys: - - qwen3.5-fp8-b200-sglang - - qwen3.5-fp8-h100-sglang - - qwen3.5-fp8-gb200-dynamo-sglang - scenario-type: - - fixed-seq-len - description: - - "Retain UTC SMI telemetry context and validate native multinode collector artifacts without enabling new recipes." - - "保留 UTC SMI 遥测上下文并验证原生多节点采集产物，不启用新 recipe。" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3051 diff --git a/runners/test_native_collector_receipts.py b/runners/test_native_collector_receipts.py index 63d9de677..d9b966296 100644 --- a/runners/test_native_collector_receipts.py +++ b/runners/test_native_collector_receipts.py @@ -91,3 +91,50 @@ def test_live_monitor_with_successful_identity_completes(tmp_path): assert (control / 'ready-0').exists() assert (control / 'done-0').read_text().strip() == '0' assert json.loads((power / 'manifest.json').read_text())['lifecycle'] == 'complete' + + +def test_native_timezone_is_scoped_to_collector(tmp_path): + import json + import os + import sys + + binary = tmp_path / 'bin' + binary.mkdir() + smi = binary / 'nvidia-smi' + smi.write_text(f'''#!{sys.executable} +import os, sys, time +with open(os.environ['SMI_ENV_LOG'], 'a') as output: + print(os.environ.get('TZ', ''), file=output, flush=True) +if '-l' in sys.argv: + print('timestamp, index, power.draw [W]', flush=True) + print('2026/09/12 00:00:00.000, 0, 100', flush=True) + time.sleep(30) +else: + print('index, uuid, pci.bus_id, name, driver_version') + print('0, GPU-fixture, 0000:00:00.0, fixture, fixture') +''') + smi.chmod(0o755) + env = {**os.environ, 'PATH': f'{binary}:{Path(sys.executable).parent}:/usr/bin:/bin', + 'TZ': 'Pacific/Honolulu', 'SMI_ENV_LOG': str(tmp_path / 'ordinary-env')} + ordinary = tmp_path / 'ordinary.csv' + result = subprocess.run(['bash', '-c', '''source "$1" +start_gpu_monitor --output "$2" +stop_gpu_monitor +''', 'bash', str(ROOT / 'benchmarks/benchmark_lib.sh'), str(ordinary)], + env=env, capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert set((tmp_path / 'ordinary-env').read_text().splitlines()) == {'Pacific/Honolulu'} + assert not ordinary.with_name('ordinary_context.json').exists() + + control = tmp_path / 'control' + control.mkdir() + (control / 'stop').touch() + power = tmp_path / 'native' + env['SMI_ENV_LOG'] = str(tmp_path / 'native-env') + result = subprocess.run(['bash', str(ROOT / 'benchmarks/native_power_collect.sh'), + str(power), str(control), 'nvidia', '0', 'aggregate', '0', '1'], + env=env, capture_output=True, text=True, timeout=5) + assert result.returncode == 0, result.stderr + assert set((tmp_path / 'native-env').read_text().splitlines()) == {'UTC'} + assert json.loads((power / 'gpu_metrics_context.json').read_text()) == {'timestamp_timezone': 'UTC'} + assert (control / 'done-0').read_text().strip() == '0' diff --git a/utils/test_native_multinode_power.py b/utils/test_native_multinode_power.py index e4234facc..31e563737 100644 --- a/utils/test_native_multinode_power.py +++ b/utils/test_native_multinode_power.py @@ -219,20 +219,6 @@ def test_native_single_role_preserves_whole_fleet_and_role_metrics(tmp_path, rol assert json.loads((tmp_path / 'power_validation_result.json').read_text())['power_valid'] -def test_native_amd_abort_skips_legacy_tail_wait(tmp_path): - result = subprocess.run(['bash', '-c', '''source "$1" -kill() { return 0; }; wait() { return 0; } -sleep() { echo unexpected-tail-wait >&2; } -_write_amd_smi_sidecar() { return 0; } -GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd AMD_MONITOR_STOP_TIMEOUT_S=0 -GPU_METRICS_CSV="$2/missing.csv" -stop_gpu_monitor -''', 'bash', str(REPO / 'benchmarks/benchmark_lib.sh'), str(tmp_path)], - capture_output=True, text=True, timeout=5) - assert result.returncode == 0, result.stderr - assert 'unexpected-tail-wait' not in result.stderr - - def test_native_amd_abort_publishes_receipt_before_reaper_deadline(tmp_path): binary = tmp_path / 'bin' binary.mkdir()