diff --git a/scripts/benchmark.mts b/scripts/benchmark.mts index 0237f5101bc4..b51e6443e2c9 100644 --- a/scripts/benchmark.mts +++ b/scripts/benchmark.mts @@ -9,6 +9,10 @@ import fs from 'node:fs'; import path from 'node:path'; import { type BenchmarkCliOptions, runI18nBenchmarks } from './benchmarks/i18n/index.mts'; +import { + type LibraryBuilderBenchmarkOptions, + runLibraryBuilderBenchmarks, +} from './benchmarks/library-builder/index.mts'; function checkBuildStatus(logger: Console): boolean { const distFile = 'dist/@angular/build/src/tools/i18n/i18n-inliner.js'; @@ -36,6 +40,88 @@ function checkBuildStatus(logger: Console): boolean { return true; } +function parseIntList(raw: string | number | undefined): number[] | undefined { + if (raw === undefined) { + return undefined; + } + return String(raw) + .split(',') + .map((s) => Number(s.trim())); +} + +async function runLibraryBuilderSubsystem( + options: { + layout?: string; + count?: string; + depth?: string; + iterations?: string | number; + json?: boolean; + csv?: string | boolean; + }, + cwd: string, +): Promise { + if (options.layout !== undefined && options.layout !== 'flat' && options.layout !== 'deep') { + // eslint-disable-next-line no-console + console.error('Error: --layout must be "flat" or "deep", got "' + options.layout + '".'); + + return 1; + } + + const count = parseIntList(options.count); + if (count !== undefined && count.some((n) => !Number.isInteger(n) || n < 1)) { + // eslint-disable-next-line no-console + console.error( + 'Error: --count must be a comma-separated list of positive integers, got "' + + options.count + + '".', + ); + + return 1; + } + + const depth = parseIntList(options.depth); + if (depth !== undefined && depth.some((n) => !Number.isInteger(n) || n < 1)) { + // eslint-disable-next-line no-console + console.error( + 'Error: --depth must be a comma-separated list of positive integers, got "' + + options.depth + + '".', + ); + + return 1; + } + + const iterations = options.iterations !== undefined ? Number(options.iterations) : undefined; + if (iterations !== undefined && (!Number.isInteger(iterations) || iterations < 1)) { + // eslint-disable-next-line no-console + console.error( + 'Error: --iterations must be a positive integer, got "' + options.iterations + '".', + ); + + return 1; + } + + if (options.csv !== undefined && (typeof options.csv !== 'string' || options.csv.length === 0)) { + // eslint-disable-next-line no-console + console.error('Error: --csv requires a file path, got "' + options.csv + '".'); + + return 1; + } + + const cliOptions: LibraryBuilderBenchmarkOptions = { + layout: options.layout as LibraryBuilderBenchmarkOptions['layout'], + count, + depth, + iterations, + json: Boolean(options.json), + csv: options.csv !== undefined ? path.resolve(cwd, options.csv as string) : undefined, + }; + + const { exitCode } = await runLibraryBuilderBenchmarks(cliOptions); + + return exitCode; +} + export default async function ( options: { _?: string[]; @@ -51,6 +137,9 @@ export default async function ( 'save-baseline'?: string; compareBaseline?: string; 'compare-baseline'?: string; + layout?: string; + count?: string; + depth?: string; help?: boolean; [key: string]: unknown; }, @@ -69,8 +158,9 @@ Usage: Subsystems: i18n Run i18n inliner performance benchmarks (default) + library-builder Run @angular/build:library scaling benchmarks -Options: +Options (i18n): --scenario= Run a specific scenario (e.g. standard-app, enterprise-multilingual) --iterations= Number of measured iterations (default: 5) --warmup= Number of warmup iterations (default: 2) @@ -80,15 +170,33 @@ Options: --json Output results in machine-readable JSON --save-baseline= Save run results to a baseline JSON file --compare-baseline= Compare run results against an existing baseline JSON file + +Options (library-builder): + --layout= Fixture layout (default: flat) + --count= Entry-point count targets, drives both layouts + (default: 10,20,50,100,200,300,400,500,1000,2000) + --depth= Explicit tree depths, layout=deep only. Overrides the depths + that would otherwise be derived from --count (the smallest + depth with 2^depth >= each target count). + --iterations= Measured iterations (default: 3) + --json Output raw measurements as JSON to stdout + --csv= Write raw per-iteration measurements as CSV to + --help Show this help message `); return 0; } + if (targetSubsystem === 'library-builder') { + return runLibraryBuilderSubsystem(options, _cwd); + } + if (targetSubsystem !== 'i18n') { // eslint-disable-next-line no-console - console.error(`Unknown benchmark subsystem: "${targetSubsystem}". Supported subsystems: i18n`); + console.error( + `Unknown benchmark subsystem: "${targetSubsystem}". Supported subsystems: i18n, library-builder`, + ); return 1; } diff --git a/scripts/benchmarks/library-builder/README.md b/scripts/benchmarks/library-builder/README.md new file mode 100644 index 000000000000..b0bbc8cd87ea --- /dev/null +++ b/scripts/benchmarks/library-builder/README.md @@ -0,0 +1,177 @@ +# Library Builds Scaling Benchmark + +This benchmark suites measures build time metrics for Angular libraries. + +## Context + +Angular CLI is in the process of transitioning from `ng-packagr` to Angular CLI's native library builder. +Both library build systems are registered Architect builders: + +- `@angular/build:ng-packagr`: runs library builds through the legacy `ng-packagr` build pipeline. +- `@angular/build:library`: runs library builds through Angular CLI's native library builder. + +This benchmark suite is designed to quantify performance impacts and ensure scalability. + +### Previous Benchmarks + +For context, when running a comparable benchmark suite directly against `ng-packagr`, it hinted at a "U-curve" in build times, +where build time per entry point in `ng-packagr` drops until entry point N (with N varying in the range 200..500). +This shape indicates an initial overhead as a possible explanation for a small number of entry points. +However, we discovered that `ng-packagr` build time per entry point increases +as the number of entry points scale past a certain N (in the test runs N=200+). + +[Landing Angular CLI's native `@angular/build:library` builder](https://github.com/angular/angular-cli/pull/34106#issuecomment-5744311681), +indicated very favourable performance metrics for the new library builder that can be explained by a number of +architectural changes in the design of the library build pipeline: + +- Shared Stylesheet Bundling +- Public API Invalidation via Declaration Hashing +- Dedicated Worker Pool for Production Builds +- Single Pass Output Emission +- Direct Async Pipeline + +The benchmark design of this suite is built to not only verify the previous results, +but also to confirm or contradict the architectural choices in the library builder pipeline. + +### Goals + +This benchmark suite runs explorative research on the performance of library builds in Angular CLI. +Interesting results from the previous benchmarks indicate 2 measurable goals: + +- Measure the new library builder against itself: How does it scale with the number of entry points? +- Enable A/B benchmarks of the `@angular/build:ng-packagr` builder against `@angular/build:library` builder: How much do library builds benefit from the transition to the native library builder? + +As the previous benchmark runs hint at significant performance improvements in the new library builder, +this benchmark suite aims to quantify the impact of that transition. + +## Benchmark Design + +This suite supports two primary layouts to test different filesystem and entry-point scaling scenarios: + +- **Flat (`--layout=flat`)**: Generates $N$ secondary entry points as direct siblings, where $N$ + comes straight from `--count`. + - Design: Each entry point is named `comp-i` (e.g., `comp-0001`, `comp-0002`). + +- **Deep (`--layout=deep`)**: Generates $2^N$ secondary entry points in a binary tree structure. + - Design: Creates a nested directory structure (`sub-a`/`sub-b` at each level). + - Depth $N$ is not an independent sweep parameter -- see "Sweep rounds" below. + +Each entry point consists of a `public-api.ts` and a component file with an inline template +containing a bit of per-entry content derived from the entry's id (e.g. `

Component no. +0042

`), so entries aren't byte-identical copies of each other. + +### Sweep rounds + +`--count` is the one sweep parameter: a comma-separated list of entry-point count targets. The +default sweep spans small to large entry-point counts in one round, chosen to resolve the +U-shaped scaling pattern observed with `ng-packagr` (see above): +`10, 20, 50, 100, 200, 300, 400, 500, 1000, 2000`. + +For `--layout=deep`, there's no "number of entry points" parameter directly -- a binary tree of +depth $N$ always has exactly $2^N$ leaves/entry points. So `--count`'s targets are translated into +tree depths: for each target `n`, the smallest `depth` with `2^depth >= n`, de-duplicated (e.g. +`300`, `400`, and `500` all round up to the same `depth=9`, i.e. 512 entry points). This keeps +both layouts comparable at roughly the same entry-point scale without exposing depth as a second, +independently-tunable sweep axis. `--depth` still exists as an **explicit override** for +`--layout=deep` -- pass it directly to sweep exact tree depths instead of entry-point targets -- +but it is not part of the default sweep design and is ignored for `--layout=flat`. + +## Running the Benchmarks + +```sh +# Ensure the builder is built first +pnpm build + +# Run the benchmark +pnpm admin benchmark library-builder [options] +``` + +### Options + +| Option | Description | Default | +| :---------------------- | :--------------------------------------------------------------------------------------------------- | :--------------------------------------- | +| `--layout=` | Fixture layout | `flat` | +| `--count=` | Entry-point count targets, drives both layouts | `10,20,50,100,200,300,400,500,1000,2000` | +| `--depth=` | **`layout=deep` only.** Explicit tree depths, overriding the depths otherwise derived from `--count` | derived from `--count` | +| `--iterations=` | Measured iterations per size | `3` | +| `--json` | Output raw per-iteration measurements as JSON to stdout | N/A | +| `--csv=` | Write raw per-iteration measurements as CSV to `` | N/A (opt-in only) | + +> `--count` and `--depth` are not independent sweep axes for `layout=deep`: `--count` is translated +> into depths (smallest `depth` with `2^depth >= n` per target, de-duplicated); `--depth` only +> matters when you want to bypass that translation and name exact tree depths yourself. Passing +> both for `layout=deep` makes `--depth` win; `--depth` is ignored entirely for `layout=flat`. + +### Example + +Sweep the flat layout at a single size: + +```sh +pnpm admin benchmark library-builder --layout=flat --count=500 +``` + +## Benchmark Results + +Each entry below is one benchmark run, with its own summary and conclusions. + +### 2026-10-12: Iteration 1 baseline sweep (native library builder, flat & deep) + +First full run of the default sweep against `@angular/build:library` only — no `ng-packagr` A/B comparison yet (still Future Work). +Fixtures use inline HTML templates only, no stylesheets. +Default `--count` sweep, `--iterations=3`, both layouts. +Environment: 4-core / 16 GiB sandbox, NVMe SSD. +Raw log: `tmp/benchmark-run-1791795379.log` (gitignored; not included in this commit). + +#### Flat layout (inline templates only) + +| Entries | Median | ms/entry | +| ------: | -----: | -------: | +| 10 | 3.25s | 324.63 | +| 20 | 3.28s | 164.14 | +| 50 | 3.33s | 66.61 | +| 100 | 3.75s | 37.53 | +| 200 | 4.31s | 21.56 | +| 300 | 4.90s | 16.35 | +| 400 | 5.51s | 13.78 | +| 500 | 6.18s | 12.36 | +| 1000 | 10.29s | 10.29 | +| 2000 | 19.47s | 9.74 | + +#### Deep layout (inline templates only) + +| Entries | Median | ms/entry | +| ------: | -----: | -------: | +| 16 | 3.27s | 204.47 | +| 32 | 3.30s | 103.16 | +| 64 | 3.57s | 55.74 | +| 128 | 4.20s | 32.84 | +| 256 | 4.70s | 18.36 | +| 512 | 6.31s | 12.32 | +| 1024 | 10.25s | 10.01 | +| 2048 | 21.67s | 10.58 | + +**Conclusion:** per-entry build time decreases monotonically across both layouts and flattens out +around 9.6-10.6ms/entry at the largest sizes tested -- no U-shaped climb-back-up at any tested +size (up to 2000/2048 entries), unlike the pattern reported for `ng-packagr` past Nā‰ˆ200-500. Flat +and deep track each other closely at comparable entry-point scale, suggesting the directory +layout itself isn't a meaningful cost driver for the native builder, just raw entry-point count. +`deep`'s depth-11 run (2048 entries) showed slightly higher run-to-run variance (20.78s-22.46s) +than `flat`'s 2000-entry run (19.46s-19.63s) -- possibly directory-tree-walk overhead at depth, +but not a regression, and not yet investigated further. + +All 60 builds (10 sizes x 3 iterations x 2 layouts) completed with `status: "ok"`; zero failures. + +## Future Work + +- Style-flavor matrix: inline / inline-css / inline-scss / external / external-css / external-scss, swept across both layouts. +- A/B comparison against `ng-packagr` (both are already registered Architect builders ā€” `@angular/build:ng-packagr` and `@angular/build:library` ā€” so this is a same-builder-package comparison, not a separate tool invocation). +- Dependencies between entry points. +- Shared stylesheet tokens imported by multiple components. +- Watch mode. +- Peak memory and CPU time metrics per build, in addition to wall-clock build time. Note: + `child.resourceUsage()` does **not** exist on Node's `ChildProcess` (only `process.resourceUsage()` + exists, and it reports the calling process's own usage, not a child's) -- the real path is + polling `/proc//status` (`VmHWM`) and `/proc//stat` (`utime`+`stime`) on a timer while + the build subprocess is alive, which is Linux-only and requires moving `buildOnce` off + `spawnSync` onto `spawn()` so a polling timer can interleave with the child's lifetime. +- Library "taste" variants beyond plain flat/deep, e.g. flat/deep with inter-entry-point dependencies or shared SCSS tokens (currently out of scope, listed above). diff --git a/scripts/benchmarks/library-builder/fixtures.mts b/scripts/benchmarks/library-builder/fixtures.mts new file mode 100644 index 000000000000..9e661a00dc8e --- /dev/null +++ b/scripts/benchmarks/library-builder/fixtures.mts @@ -0,0 +1,232 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Generates synthetic Angular Package Format (APF) library fixtures for benchmarking +// `@angular/build:library`, and converts them into the builder's project shape (a flat +// `exports` map in one root package.json, rather than the per-entry-point `ng-package.json` +// convention used by third-party tools such as ng-packagr). + +import fs from 'node:fs'; +import path from 'node:path'; + +export type Layout = 'flat' | 'deep'; + +export interface FixtureOptions { + layout: Layout; + + /** Number of secondary entry points for `layout: 'flat'`. */ + count?: number; + + /** Binary-tree depth for `layout: 'deep'` (2^depth leaf entry points). */ + depth?: number; +} + +function buildComponentSource(className: string, selector: string, id: string): string { + return `import { Component } from '@angular/core'; + +@Component({ + selector: '${selector}', + template: '

Component no. ${id}

', +}) +export class ${className} {} +`; +} + +/** Writes one secondary entry point's source files (component + public-api.ts) into `entryDir`. */ +function writeEntryPointSource(entryDir: string, id: string): void { + fs.mkdirSync(entryDir, { recursive: true }); + + const className = `RefComponent${id}`; + const selector = `ref-comp-${id}`; + const compBase = `comp-${id}`; + + fs.writeFileSync( + path.join(entryDir, 'public-api.ts'), + `export * from './${compBase}.component';\n`, + ); + + fs.writeFileSync( + path.join(entryDir, `${compBase}.component.ts`), + buildComponentSource(className, selector, id), + ); +} + +/** Generates `count` flat secondary entry points as direct siblings, returning their relative dirs. */ +function generateFlatEntries(outDir: string, count: number): string[] { + const width = String(count).length; + const dirs: string[] = []; + for (let i = 1; i <= count; i++) { + const id = String(i).padStart(width, '0'); + const relDir = `comp-${id}`; + writeEntryPointSource(path.join(outDir, relDir), id); + dirs.push(relDir); + } + + return dirs; +} + +/** + * Generates a binary tree `depth` levels deep (`sub-a`/`sub-b` at each level), placing one + * secondary entry point at each of the 2^depth leaves. Returns their relative dirs. + */ +function generateDeepEntries(outDir: string, depth: number): string[] { + const total = 2 ** depth; + const width = String(total).length; + const dirs: string[] = []; + let counter = 0; + + function build(currentRelDir: string, level: number): void { + if (level === depth) { + counter++; + const id = String(counter).padStart(width, '0'); + writeEntryPointSource(path.join(outDir, currentRelDir), id); + dirs.push(currentRelDir); + + return; + } + for (const child of ['sub-a', 'sub-b']) { + build(path.join(currentRelDir, child), level + 1); + } + } + + build('.', 0); + + return dirs.map((d) => path.normalize(d)); +} + +/** "sub-a/sub-b" -> "sub-a-sub-b"; "comp-0001" -> "comp-0001". Used as the exports map subpath key. */ +function entryKey(relDir: string): string { + return relDir.split(path.sep).join('-'); +} + +/** + * Generates a complete `@angular/build:library`-buildable project at `outDir`: a primary entry + * point, N secondary entry points per `options`, and the angular.json/tsconfig/package.json + * scaffolding the builder needs. Returns the number of secondary entry points generated. + */ +export function generateFixture(outDir: string, options: FixtureOptions): number { + fs.rmSync(outDir, { recursive: true, force: true }); + fs.mkdirSync(outDir, { recursive: true }); + + fs.writeFileSync( + path.join(outDir, 'public-api.ts'), + `export const REFERENCE_VERSION = '1.0.0';\n`, + ); + + const secondaryDirs = + options.layout === 'flat' + ? generateFlatEntries(outDir, options.count ?? 10) + : generateDeepEntries(outDir, options.depth ?? 2); + + const exportsMap: Record = { '.': './public-api.ts' }; + for (const relDir of secondaryDirs) { + exportsMap[`./${entryKey(relDir)}`] = `./${relDir.split(path.sep).join('/')}/public-api.ts`; + } + + // Fallback to a hardcoded version if the path lookup fails in the sandbox/environment. + let peerRange = '^22.0.0'; + try { + const angularCoreVersion = JSON.parse( + fs.readFileSync( + new URL('../../../node_modules/@angular/core/package.json', import.meta.url), + 'utf8', + ), + ).version as string; + peerRange = `^${angularCoreVersion.replace(/\.\d+$/, '')}`; + } catch { + // Ignore error, use fallback + } + + fs.writeFileSync( + path.join(outDir, 'package.json'), + JSON.stringify( + { + name: `ref-${options.layout}`, + version: '0.0.1', + peerDependencies: { + '@angular/core': peerRange, + '@angular/common': peerRange, + }, + exports: exportsMap, + }, + null, + 2, + ) + '\n', + ); + + fs.writeFileSync( + path.join(outDir, 'tsconfig.json'), + JSON.stringify( + { + compileOnSave: false, + compilerOptions: { + outDir: './dist/out-tsc', + sourceMap: true, + declaration: false, + moduleResolution: 'bundler', + experimentalDecorators: true, + target: 'es2022', + module: 'es2022', + useDefineForClassFields: false, + lib: ['es2022', 'dom'], + }, + angularCompilerOptions: { + enableIvy: true, + disableTypeScriptVersionCheck: true, + }, + }, + null, + 2, + ) + '\n', + ); + + fs.writeFileSync( + path.join(outDir, 'tsconfig.lib.json'), + JSON.stringify( + { + extends: './tsconfig.json', + compilerOptions: { + declaration: true, + declarationMap: true, + inlineSources: true, + types: [], + }, + exclude: ['**/*.spec.ts'], + }, + null, + 2, + ) + '\n', + ); + + fs.writeFileSync( + path.join(outDir, 'angular.json'), + JSON.stringify( + { + version: 1, + cli: { cache: { enabled: false } }, + projects: { + lib: { + root: '.', + projectType: 'library', + sourceRoot: '.', + targets: { + build: { + builder: '@angular/build:library', + options: { tsConfig: './tsconfig.lib.json', outputPath: 'dist' }, + }, + }, + }, + }, + }, + null, + 2, + ) + '\n', + ); + + return secondaryDirs.length; +} diff --git a/scripts/benchmarks/library-builder/harness.mts b/scripts/benchmarks/library-builder/harness.mts new file mode 100644 index 000000000000..5220b95965e9 --- /dev/null +++ b/scripts/benchmarks/library-builder/harness.mts @@ -0,0 +1,82 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Runs and measures a single `@angular/build:library` cold build via Architect's standalone CLI. +// `fixtures.mts` owns generating the on-disk project; this module owns spawning the builder +// against it and reporting what happened. Kept separate so each file has one job. + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import type { Layout } from './fixtures.mts'; + +export interface BuildMeasurement { + layout: Layout; + entryPoints: number; + iteration: number; + durationMs: number; + status: 'ok' | 'fail'; +} + +export interface BuilderPaths { + buildPkg: string; + architectPkg: string; + architectCli: string; +} + +/** Verifies `@angular/build` and `@angular-devkit/architect` are built before attempting a build. */ +export function checkBuilderIsBuilt(repoRoot: string, logger: Console): BuilderPaths | undefined { + const buildPkg = path.join(repoRoot, 'dist', '@angular', 'build'); + const architectPkg = path.join(repoRoot, 'dist', '@angular-devkit', 'architect'); + const architectCli = path.join(architectPkg, 'bin', 'cli.js'); + + if (!fs.existsSync(path.join(buildPkg, 'package.json')) || !fs.existsSync(architectCli)) { + logger.error( + 'Error: @angular/build and/or @angular-devkit/architect have not been built yet.\n' + + 'Run "pnpm build" (or "pnpm admin build --local" for a faster local build) first.', + ); + + return undefined; + } + + return { buildPkg, architectPkg, architectCli }; +} + +/** Spawns one cold `lib:build` run against `projectDir` and times it. */ +export function buildOnce( + projectDir: string, + architectCli: string, +): { durationMs: number; status: 'ok' | 'fail' } { + const start = performance.now(); + const result = spawnSync(process.execPath, [architectCli, 'lib:build'], { + cwd: projectDir, + stdio: 'pipe', + maxBuffer: Infinity, + }); + const durationMs = performance.now() - start; + const status = result.status === 0 ? 'ok' : 'fail'; + if (status === 'fail') { + if (result.error) { + // eslint-disable-next-line no-console + console.error(result.error); + } + // eslint-disable-next-line no-console + console.error(result.stdout?.toString()); + // eslint-disable-next-line no-console + console.error(result.stderr?.toString()); + } + + return { durationMs, status }; +} + +export function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} diff --git a/scripts/benchmarks/library-builder/index.mts b/scripts/benchmarks/library-builder/index.mts new file mode 100644 index 000000000000..7615f2b3598c --- /dev/null +++ b/scripts/benchmarks/library-builder/index.mts @@ -0,0 +1,142 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Benchmarks `@angular/build:library`'s cold build time across a sweep of library sizes. +// This module only orchestrates the sweep: fixture generation lives in fixtures.mts, build +// execution and measurement live in harness.mts. + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { type FixtureOptions, type Layout, generateFixture } from './fixtures.mts'; +import { type BuildMeasurement, buildOnce, checkBuilderIsBuilt, median } from './harness.mts'; +import { formatCsvReport, formatJsonReport } from './reporters.mts'; + +export interface LibraryBuilderBenchmarkOptions { + layout?: Layout; + count?: number[]; + depth?: number[]; + iterations?: number; + json?: boolean; + csv?: string; +} + +const repoRoot = fileURLToPath(new URL('../../../', import.meta.url)); +const scratchRoot = path.join(repoRoot, 'dist', '.library-benchmark-fixtures'); + +/** + * Default entry-point counts for the `flat` layout, chosen to resolve a U-shaped scaling curve. + * Also drives the `deep` layout's default depths (see `depthsForEntryPointTargets`) -- `--count` + * is the one sweep parameter users pick; `--depth` is only ever an explicit override. + */ +const DEFAULT_FLAT_COUNTS = [10, 20, 50, 100, 200, 300, 400, 500, 1000, 2000]; + +/** + * Derives default `deep` layout tree depths from a set of target entry-point counts, picking for + * each target `n` the smallest `depth` with `2^depth >= n`, de-duplicated in ascending order. + * This keeps both layouts comparable at roughly the same entry-point scale instead of an + * arbitrary, independently-chosen depth list. + */ +function depthsForEntryPointTargets(targets: number[]): number[] { + const depths = new Set(); + for (const target of targets) { + depths.add(Math.max(1, Math.ceil(Math.log2(target)))); + } + + return [...depths].sort((a, b) => a - b); +} + +export async function runLibraryBuilderBenchmarks( + options: LibraryBuilderBenchmarkOptions = {}, +): Promise<{ measurements: BuildMeasurement[]; exitCode: number }> { + const builder = checkBuilderIsBuilt(repoRoot, console); + if (!builder) { + return { measurements: [], exitCode: 1 }; + } + + const layout = options.layout ?? 'flat'; + const iterations = options.iterations ?? 3; + const sizes = + layout === 'flat' + ? (options.count ?? DEFAULT_FLAT_COUNTS) + : (options.depth ?? depthsForEntryPointTargets(options.count ?? DEFAULT_FLAT_COUNTS)); + + fs.mkdirSync(scratchRoot, { recursive: true }); + const measurements: BuildMeasurement[] = []; + + try { + for (const sizeOrDepth of sizes) { + const label = layout + '-' + (layout === 'deep' ? 'depth' + sizeOrDepth : sizeOrDepth); + const projectDir = path.join(scratchRoot, label); + + const fixtureOptions: FixtureOptions = + layout === 'flat' ? { layout, count: sizeOrDepth } : { layout, depth: sizeOrDepth }; + const entryPoints = generateFixture(projectDir, fixtureOptions); + + if (!options.json) { + // eslint-disable-next-line no-console + console.log('\n[library-builder] ' + label + ' (' + entryPoints + ' entry points)'); + } + + const durations: number[] = []; + for (let i = 1; i <= iterations; i++) { + const { durationMs, status } = buildOnce(projectDir, builder.architectCli); + if (!options.json) { + // eslint-disable-next-line no-console + console.log( + ' run ' + + i + + '/' + + iterations + + ': ' + + (status === 'ok' ? 'done' : 'FAILED') + + ' in ' + + (durationMs / 1000).toFixed(2) + + 's', + ); + } + if (status === 'ok') { + durations.push(durationMs); + } + measurements.push({ layout, entryPoints, iteration: i, durationMs, status }); + } + + if (!options.json && durations.length > 0) { + const medianMs = median(durations); + // eslint-disable-next-line no-console + console.log( + ' median: ' + + (medianMs / 1000).toFixed(2) + + 's (' + + (medianMs / entryPoints).toFixed(2) + + 'ms/entry)', + ); + } + + fs.rmSync(projectDir, { recursive: true, force: true }); + } + } finally { + fs.rmSync(scratchRoot, { recursive: true, force: true }); + } + + const anyFailed = measurements.some((m) => m.status === 'fail'); + if (options.json) { + // eslint-disable-next-line no-console + console.log(formatJsonReport(measurements)); + } + + if (options.csv) { + fs.writeFileSync(options.csv, formatCsvReport(measurements)); + if (!options.json) { + // eslint-disable-next-line no-console + console.log('CSV written to: ' + options.csv); + } + } + + return { measurements, exitCode: anyFailed ? 1 : 0 }; +} diff --git a/scripts/benchmarks/library-builder/reporters.mts b/scripts/benchmarks/library-builder/reporters.mts new file mode 100644 index 000000000000..d175df7332bf --- /dev/null +++ b/scripts/benchmarks/library-builder/reporters.mts @@ -0,0 +1,25 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Pure output formatting for the library-builder benchmark's measurements. No console/file I/O +// here -- `index.mts` owns deciding when and where to print or write what this module formats. + +import type { BuildMeasurement } from './harness.mts'; + +export function formatJsonReport(measurements: BuildMeasurement[]): string { + return JSON.stringify(measurements, null, 2); +} + +export function formatCsvReport(measurements: BuildMeasurement[]): string { + const header = 'layout,entryPoints,iteration,durationMs,status'; + const rows = measurements.map((m) => + [m.layout, m.entryPoints, m.iteration, m.durationMs.toFixed(2), m.status].join(','), + ); + + return [header, ...rows].join('\n') + '\n'; +}