Skip to content

Commit 90b8b5f

Browse files
authored
fixup! test(@angular/build): add library builder scaling benchmark suite
1 parent 0111149 commit 90b8b5f

5 files changed

Lines changed: 288 additions & 323 deletions

File tree

‎scripts/benchmark.mts‎

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ function checkBuildStatus(logger: Console): boolean {
3232
if (srcMtime > distMtime) {
3333
logger.warn(
3434
'Warning: Source files in packages/angular/build are newer than dist/.\n' +
35-
'Run "pnpm build" to ensure your benchmark reflects your latest local edits.\n',
35+
'Run "pnpm build" to ensure your benchmark reflects your latest local edits.\n',
3636
);
3737
}
3838
}
@@ -44,64 +44,63 @@ function parseIntList(raw: string | number | undefined): number[] | undefined {
4444
if (raw === undefined) {
4545
return undefined;
4646
}
47-
return String(raw).split(',').map((s) => Number(s.trim()));
47+
return String(raw)
48+
.split(',')
49+
.map((s) => Number(s.trim()));
4850
}
4951

5052
async function runLibraryBuilderSubsystem(options: {
5153
layout?: string;
52-
style?: string;
53-
sizes?: string;
54-
depths?: string;
54+
count?: string;
55+
depth?: string;
5556
iterations?: string | number;
5657
json?: boolean;
5758
}): Promise<number> {
5859
if (options.layout !== undefined && options.layout !== 'flat' && options.layout !== 'deep') {
5960
// eslint-disable-next-line no-console
60-
console.error("Error: --layout must be \"flat\" or \"deep\", got \"" + options.layout + "\".");
61+
console.error('Error: --layout must be "flat" or "deep", got "' + options.layout + '".');
6162

6263
return 1;
6364
}
64-
if (
65-
options.style !== undefined &&
66-
!['inline', 'inline-scss', 'external'].includes(options.style)
67-
) {
65+
66+
const count = parseIntList(options.count);
67+
if (count !== undefined && count.some((n) => !Number.isInteger(n) || n < 1)) {
6868
// eslint-disable-next-line no-console
6969
console.error(
70-
"Error: --style must be one of \"inline\", \"inline-scss\", \"external\", got \"" + options.style + "\"."
70+
'Error: --count must be a comma-separated list of positive integers, got "' +
71+
options.count +
72+
'".',
7173
);
7274

7375
return 1;
7476
}
7577

76-
const sizes = parseIntList(options.sizes);
77-
if (sizes !== undefined && sizes.some((n) => !Number.isInteger(n) || n < 1)) {
78-
// eslint-disable-next-line no-console
79-
console.error("Error: --sizes must be a comma-separated list of positive integers, got \"" + options.sizes + "\".");
80-
81-
return 1;
82-
}
83-
84-
const depths = parseIntList(options.depths);
85-
if (depths !== undefined && depths.some((n) => !Number.isInteger(n) || n < 1)) {
78+
const depth = parseIntList(options.depth);
79+
if (depth !== undefined && depth.some((n) => !Number.isInteger(n) || n < 1)) {
8680
// eslint-disable-next-line no-console
87-
console.error("Error: --depths must be a comma-separated list of positive integers, got \"" + options.depths + "\".");
81+
console.error(
82+
'Error: --depth must be a comma-separated list of positive integers, got "' +
83+
options.depth +
84+
'".',
85+
);
8886

8987
return 1;
9088
}
9189

9290
const iterations = options.iterations !== undefined ? Number(options.iterations) : undefined;
9391
if (iterations !== undefined && (!Number.isInteger(iterations) || iterations < 1)) {
9492
// eslint-disable-next-line no-console
95-
console.error("Error: --iterations must be a positive integer, got \"" + options.iterations + "\".");
93+
console.error(
94+
'Error: --iterations must be a positive integer, got "' + options.iterations + '".',
95+
);
9696

9797
return 1;
9898
}
9999

100100
const cliOptions: LibraryBuilderBenchmarkOptions = {
101101
layout: options.layout as LibraryBuilderBenchmarkOptions['layout'],
102-
style: options.style as LibraryBuilderBenchmarkOptions['style'],
103-
sizes,
104-
depths,
102+
count,
103+
depth,
105104
iterations,
106105
json: Boolean(options.json),
107106
};
@@ -127,9 +126,8 @@ export default async function (
127126
compareBaseline?: string;
128127
'compare-baseline'?: string;
129128
layout?: string;
130-
style?: string;
131-
sizes?: string;
132-
depths?: string;
129+
count?: string;
130+
depth?: string;
133131
help?: boolean;
134132
[key: string]: unknown;
135133
},
@@ -163,10 +161,12 @@ Options (i18n):
163161
164162
Options (library-builder):
165163
--layout=<flat|deep> Fixture layout (default: flat)
166-
--style=<inline|inline-scss|external> Component style variant (default: inline)
167-
--sizes=<n,n,...> Entry-point counts to sweep, layout=flat only (default: 10,50,300,1000,2000)
168-
--depths=<n,n,...> Tree depths to sweep, layout=deep only (default: 2,5,8,11)
169-
--iterations=<n> Measured iterations per size (default: 2)
164+
--count=<n,n,...> Entry-point count targets, drives both layouts
165+
(default: 10,20,50,100,200,300,400,500,1000,2000)
166+
--depth=<n,n,...> Explicit tree depths, layout=deep only. Overrides the depths
167+
that would otherwise be derived from --count (the smallest
168+
depth with 2^depth >= each target count).
169+
--iterations=<n> Measured iterations (default: 3)
170170
--json Output raw measurements as JSON
171171
172172
--help Show this help message
Lines changed: 104 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,77 +1,119 @@
1-
# `@angular/build:library` scaling benchmark
1+
# Library Builds Scaling Benchmark
22

3-
This benchmark measures how `@angular/build:library`'s cold build time scales with the number
4-
of secondary entry points in an Angular Package Format (APF) library, across two structurally
5-
different layouts (flat siblings vs. a deeply nested tree) and three component style variants
6-
(inline template, inline template + styles, external template/stylesheet files).
3+
This benchmark measures build time metrics for Angular libraries.
74

8-
## Why
5+
## Context
96

10-
A comparison against [ng-packagr](https://github.com/ng-packagr/ng-packagr) (a third-party APF
11-
build tool) found that its build time **per entry point** grows super-linearly once a library
12-
has more than roughly 300-500 secondary entry points -- doubling the library size past that
13-
point costs noticeably more than double the build time. This suite exists to check whether
14-
`@angular/build:library`, a from-scratch build pipeline unrelated to ng-packagr's implementation,
15-
shows the same growth on equivalent fixtures.
7+
Angular CLI is in the process of transitioning from `ng-packagr` to Angular CLI's native library builder.
8+
Both library build systems are registered Architect builders:
9+
- `@angular/build:ng-packagr`: runs library builds through the legacy `ng-packagr` build pipeline.
10+
- `@angular/build:library`: runs library builds through Angular CLI's native library builder.
1611

17-
**Finding:** it does not. Across the same sweep (flat layout: 10/50/300/1000/2000 entries; deep
18-
layout: tree depths 2/5/8/11, i.e. 4/32/256/2048 entries), `@angular/build:library`'s per-entry
19-
cost keeps falling as the library grows, with no climb at the largest sizes tested. It is also
20-
1.8-13x faster than ng-packagr at matching sizes, with the gap widening as size increases. See
21-
the PR description this suite was added alongside for the full writeup and caveats.
12+
This benchmark suite is designed to quantify performance impacts and ensure scalability.
2213

23-
## Running it
14+
### Previous Benchmarks
15+
16+
For context, when running a similar benchmark suite directly against `ng-packagr`, it hinted at a "U-curve" in build times,
17+
where build time per entry point in `ng-packagr` drops until entry point N (with N varying in the range 200..500).
18+
This shape indicates an initial overhead as a possible explanation for a small number of entry points.
19+
However, we discovered that `ng-packagr` build time per entry point increases
20+
as the number of entry points scale past a certain N (in the test runs N=200+).
21+
22+
[Landing Angular CLI's native `@angular/build:library` builder](https://github.com/angular/angular-cli/pull/34106#issuecomment-5744311681),
23+
indicated very favourable performance metrics for the new library builder that can be explained by number of
24+
architectural changes in the library build pipeline:
25+
- Shared Stylesheet Bundling
26+
- Public API Invalidation via Declaration Hashing
27+
- Dedicated Worker Pool for Production Builds
28+
- Single Pass Output Emission
29+
- Direct Async Pipeline
30+
31+
### Goals
32+
33+
This benchmark suite runs explorative research on the performance of library builds in Angular CLI.
34+
Interesting results from the previous benchmarks indicate 2 measurable goals:
35+
36+
- Measure the new library builder against itself: How does it scale with the number of entry points?
37+
- Enable A/B benchmarks of the `@angular/build:ng-packagr` builder against `@angular/build:library` builder: How much do library builds benefit from the transition to the native library builder?
38+
39+
As the previous benchmark runs hint at significant performance improvements in the new library builder,
40+
this benchmark suite aims to quantify the impact of that transition.
41+
42+
## Benchmark Design
43+
44+
This suite supports two primary layouts to test different filesystem and entry-point scaling scenarios:
45+
46+
- **Flat (`--layout=flat`)**: Generates $N$ secondary entry points as direct siblings, where $N$
47+
comes straight from `--count`.
48+
- Design: Each entry point is named `comp-i` (e.g., `comp-0001`, `comp-0002`).
49+
50+
- **Deep (`--layout=deep`)**: Generates $2^N$ secondary entry points in a binary tree structure.
51+
- Design: Creates a nested directory structure (`sub-a`/`sub-b` at each level).
52+
- Depth $N$ is not an independent sweep parameter -- see "Sweep rounds" below.
53+
54+
Each entry point consists of a `public-api.ts` and a component file with an inline template
55+
containing a bit of per-entry content derived from the entry's id (e.g. `<h2>Component no.
56+
0042</h2>`), so entries aren't byte-identical copies of each other.
57+
58+
### Sweep rounds
59+
60+
`--count` is the one sweep parameter: a comma-separated list of entry-point count targets. The
61+
default sweep spans small to large entry-point counts in one round, chosen to resolve the
62+
U-shaped scaling pattern observed with `ng-packagr` (see above):
63+
`10, 20, 50, 100, 200, 300, 400, 500, 1000, 2000`.
64+
65+
For `--layout=deep`, there's no "number of entry points" parameter directly -- a binary tree of
66+
depth $N$ always has exactly $2^N$ leaves/entry points. So `--count`'s targets are translated into
67+
tree depths: for each target `n`, the smallest `depth` with `2^depth >= n`, de-duplicated (e.g.
68+
`300`, `400`, and `500` all round up to the same `depth=9`, i.e. 512 entry points). This keeps
69+
both layouts comparable at roughly the same entry-point scale without exposing depth as a second,
70+
independently-tunable sweep axis. `--depth` still exists as an **explicit override** for
71+
`--layout=deep` -- pass it directly to sweep exact tree depths instead of entry-point targets --
72+
but it is not part of the default sweep design and is ignored for `--layout=flat`.
73+
74+
## Running the Benchmarks
2475

2576
```sh
26-
pnpm build # or: pnpm admin build --local -- the benchmark measures the compiled builder,
27-
# not TypeScript source, so it must exist at dist/@angular/build and
28-
# dist/@angular-devkit/architect first
29-
pnpm admin benchmark library-builder
77+
# Ensure the builder is built first
78+
pnpm build
79+
80+
# Run the benchmark
81+
pnpm admin benchmark library-builder [options]
3082
```
3183

32-
Options:
84+
### Options
3385

34-
```
35-
--layout=<flat|deep> Fixture layout (default: flat)
36-
--style=<inline|inline-scss|external> Component style variant (default: inline)
37-
--sizes=<n,n,...> Entry-point counts to sweep, layout=flat only (default: 10,50,300,1000,2000)
38-
--depths=<n,n,...> Tree depths to sweep, layout=deep only (default: 2,5,8,11)
39-
--iterations=<n> Measured iterations per size (default: 2)
40-
--json Output raw measurements as JSON instead of a human-readable log
41-
```
86+
| Option | Description | Default |
87+
| :--------------------- | :----------------------------------------------- | :---------------------------------------- |
88+
| `--layout=<flat\|deep>` | Fixture layout | `flat` |
89+
| `--count=<n,n,...>` | Entry-point count targets, drives both layouts | `10,20,50,100,200,300,400,500,1000,2000` |
90+
| `--depth=<n,n,...>` | **`layout=deep` only.** Explicit tree depths, overriding the depths otherwise derived from `--count` | derived from `--count` |
91+
| `--iterations=<n>` | Measured iterations per size | `3` |
92+
| `--json` | Output raw measurements as JSON | N/A |
4293

43-
Example: sweep the external-style variant at a few sizes with 3 iterations each:
94+
> `--count` and `--depth` are not independent sweep axes for `layout=deep`: `--count` is translated
95+
> into depths (smallest `depth` with `2^depth >= n` per target, de-duplicated); `--depth` only
96+
> matters when you want to bypass that translation and name exact tree depths yourself. Passing
97+
> both for `layout=deep` makes `--depth` win; `--depth` is ignored entirely for `layout=flat`.
98+
99+
### Example
100+
101+
Sweep the flat layout at a single size:
44102

45103
```sh
46-
pnpm admin benchmark library-builder --style=external --sizes=100,500,1000 --iterations=3
104+
pnpm admin benchmark library-builder --layout=flat --count=500
47105
```
48106

49-
## How it works
50-
51-
- `fixtures.mts` generates a synthetic library at a given size/layout/style: a primary entry
52-
point, N secondary entry points (each a minimal standalone Angular component), and the
53-
`angular.json`/`tsconfig.lib.json`/`package.json` scaffolding `@angular/build:library` needs.
54-
Secondary entry points are declared as a single flat `exports` map in the root `package.json`
55-
(the builder's actual convention), not as separate per-entry config files.
56-
- `index.mts` times `architect lib:build` (the standalone `@angular-devkit/architect` CLI,
57-
invoked directly -- no `ng` CLI needed) once per iteration via `child_process.spawnSync`,
58-
discards the generated project afterwards, and reports per-size median build time.
59-
- Because `@angular/build` and `@angular-devkit/architect` are Bazel-built workspace packages
60-
rather than ordinary npm dependencies, a plain `pnpm install` never links them (or their own
61-
dependencies) into the repo's root `node_modules` -- nothing in `dist/` is resolvable via a
62-
bare `require()`/`import` otherwise. On first run, `index.mts` symlinks `@angular/build` and
63-
`@angular-devkit/architect` from `dist/` into the root `node_modules`, then walks their
64-
declared `dependencies` and links any of those still missing from the pnpm store. This only
65-
touches `node_modules` (already gitignored) and is idempotent -- safe to run repeatedly or
66-
alongside other work in the same checkout.
67-
68-
## Methodology notes
69-
70-
- Each reported number is the **median** of the measured iterations, not the mean -- process-spawn
71-
wall-clock timings are noisy and right-skewed, and the median is more representative of steady
72-
state than an outlier-sensitive mean.
73-
- Results are specific to the machine they were collected on. Run on one otherwise-idle machine
74-
for a self-consistent sweep; absolute numbers will differ across hardware, though the _shape_
75-
of the curve (does per-entry cost grow with size, or not) is the more portable finding.
76-
- This benchmark only covers **cold, one-shot builds**. It does not exercise incremental rebuilds
77-
or watch mode.
107+
## Benchmark Results
108+
109+
_(To be filled in once a sweep has been run and reviewed. Out of scope for this iteration.)_
110+
111+
## Future Work
112+
113+
- Style-flavor matrix: inline / inline-css / inline-scss / external / external-css / external-scss, swept across both layouts.
114+
- A/B comparison against `ng-packagr` (both are already registered Architect builders — `@angular/build:ng-packagr` and `@angular/build:library` — so this is a same-builder-package comparison, not a separate tool invocation).
115+
- Dependencies between entry points.
116+
- Shared stylesheet tokens imported by multiple components.
117+
- Watch mode.
118+
- Peak memory (`ru_maxrss`) and CPU time (`ru_utime` + `ru_stime`) metrics per build, in addition to wall-clock build time.
119+
- Library "taste" variants beyond plain flat/deep, e.g. flat/deep with inter-entry-point dependencies or shared SCSS tokens (currently out of scope, listed above).

0 commit comments

Comments
 (0)