diff --git a/scripts/benchmark.mts b/scripts/benchmark.mts index 0237f5101bc4..5c024cd117b2 100644 --- a/scripts/benchmark.mts +++ b/scripts/benchmark.mts @@ -9,6 +9,16 @@ import fs from 'node:fs'; import path from 'node:path'; import { type BenchmarkCliOptions, runI18nBenchmarks } from './benchmarks/i18n/index.mts'; +import { + LIBRARY_BUILDERS, + STYLE_FLAVORS, + type LibraryBuilder, + type StyleFlavor, +} from './benchmarks/library-builder/fixtures.mts'; +import { + type LibraryBuilderBenchmarkOptions, + runLibraryBuilderBenchmarks, +} from './benchmarks/library-builder/index.mts'; function checkBuildStatus(logger: Console): boolean { const distFile = 'dist/@angular/build/src/tools/i18n/i18n-inliner.js'; @@ -36,6 +46,131 @@ function checkBuildStatus(logger: Console): boolean { return true; } +function parseIntList(raw: string | number | undefined): number[] | undefined { + if (raw === undefined) { + return undefined; + } + + return String(raw) + .split(',') + .map((s) => Number(s.trim())); +} + +function parseStringList(raw: string | undefined): string[] | undefined { + if (raw === undefined) { + return undefined; + } + + return raw.split(',').map((s) => s.trim()); +} + +async function runLibraryBuilderSubsystem( + options: { + layout?: string; + builder?: string; + count?: string; + depth?: string; + 'style-flavor'?: string; + iterations?: string | number; + json?: boolean; + csv?: string | boolean; + }, + cwd: string, +): Promise { + if (options.layout !== undefined && options.layout !== 'flat' && options.layout !== 'deep') { + // eslint-disable-next-line no-console + console.error('Error: --layout must be "flat" or "deep", got "' + options.layout + '".'); + + return 1; + } + + const count = parseIntList(options.count); + if (count !== undefined && count.some((n) => !Number.isInteger(n) || n < 1)) { + // eslint-disable-next-line no-console + console.error( + 'Error: --count must be a comma-separated list of positive integers, got "' + + options.count + + '".', + ); + + return 1; + } + + const depth = parseIntList(options.depth); + if (depth !== undefined && depth.some((n) => !Number.isInteger(n) || n < 1)) { + // eslint-disable-next-line no-console + console.error( + 'Error: --depth must be a comma-separated list of positive integers, got "' + + options.depth + + '".', + ); + + return 1; + } + + const iterations = options.iterations !== undefined ? Number(options.iterations) : undefined; + if (iterations !== undefined && (!Number.isInteger(iterations) || iterations < 1)) { + // eslint-disable-next-line no-console + console.error( + 'Error: --iterations must be a positive integer, got "' + options.iterations + '".', + ); + + return 1; + } + + if (options.csv !== undefined && (typeof options.csv !== 'string' || options.csv.length === 0)) { + // eslint-disable-next-line no-console + console.error('Error: --csv requires a file path, got "' + options.csv + '".'); + + return 1; + } + + const builder = parseStringList(options.builder); + const validBuilders: readonly string[] = LIBRARY_BUILDERS; + if (builder !== undefined && builder.some((b) => !validBuilders.includes(b))) { + // eslint-disable-next-line no-console + console.error( + 'Error: --builder must be a comma-separated list of: ' + + LIBRARY_BUILDERS.join(', ') + + '. Got "' + + options.builder + + '".', + ); + + return 1; + } + + const styleFlavor = parseStringList(options['style-flavor']); + const validStyleFlavors: readonly string[] = STYLE_FLAVORS; + if (styleFlavor !== undefined && styleFlavor.some((f) => !validStyleFlavors.includes(f))) { + // eslint-disable-next-line no-console + console.error( + 'Error: --style-flavor must be a comma-separated list of: ' + + STYLE_FLAVORS.join(', ') + + '. Got "' + + options['style-flavor'] + + '".', + ); + + return 1; + } + + const cliOptions: LibraryBuilderBenchmarkOptions = { + layout: options.layout as LibraryBuilderBenchmarkOptions['layout'], + builder: builder as LibraryBuilder[] | undefined, + count, + depth, + styleFlavor: styleFlavor as StyleFlavor[] | undefined, + iterations, + json: Boolean(options.json), + csv: options.csv !== undefined ? path.resolve(cwd, options.csv as string) : undefined, + }; + + const { exitCode } = await runLibraryBuilderBenchmarks(cliOptions); + + return exitCode; +} + export default async function ( options: { _?: string[]; @@ -51,6 +186,11 @@ export default async function ( 'save-baseline'?: string; compareBaseline?: string; 'compare-baseline'?: string; + layout?: string; + builder?: string; + count?: string; + depth?: string; + 'style-flavor'?: string; help?: boolean; [key: string]: unknown; }, @@ -69,8 +209,9 @@ Usage: Subsystems: i18n Run i18n inliner performance benchmarks (default) + library-builder Run @angular/build:library (and :ng-packagr) scaling benchmarks -Options: +Options (i18n): --scenario= Run a specific scenario (e.g. standard-app, enterprise-multilingual) --iterations= Number of measured iterations (default: 5) --warmup= Number of warmup iterations (default: 2) @@ -80,15 +221,38 @@ Options: --json Output results in machine-readable JSON --save-baseline= Save run results to a baseline JSON file --compare-baseline= Compare run results against an existing baseline JSON file + +Options (library-builder): + --layout= Fixture layout (default: flat) + --builder= Builders to benchmark, one or more of: library, ng-packagr. + Pass both for an A/B run (default: library) + --count= Entry-point count targets, drives both layouts + (default: 10,20,50,100,200,300,400,500,1000,2000) + --depth= Explicit tree depths, layout=deep only. Overrides the depths + that would otherwise be derived from --count (the smallest + depth with 2^depth >= each target count). + --style-flavor= Style-flavor targets, one or more of: inline, inline-css, + inline-scss, external, external-css, external-scss + (default: inline) + --iterations= Measured iterations (default: 3) + --json Output raw measurements as JSON to stdout + --csv= Write raw per-iteration measurements as CSV to + --help Show this help message `); return 0; } + if (targetSubsystem === 'library-builder') { + return runLibraryBuilderSubsystem(options, _cwd); + } + if (targetSubsystem !== 'i18n') { // eslint-disable-next-line no-console - console.error(`Unknown benchmark subsystem: "${targetSubsystem}". Supported subsystems: i18n`); + console.error( + `Unknown benchmark subsystem: "${targetSubsystem}". Supported subsystems: i18n, library-builder`, + ); return 1; } diff --git a/scripts/benchmarks/library-builder/README.md b/scripts/benchmarks/library-builder/README.md new file mode 100644 index 000000000000..29d0963ff744 --- /dev/null +++ b/scripts/benchmarks/library-builder/README.md @@ -0,0 +1,640 @@ +# Library Builds Scaling Benchmark + +This benchmark suites measures build time metrics for Angular libraries. + +## Context + +Angular CLI is in the process of transitioning from `ng-packagr` to Angular CLI's native library builder. +Both library build systems are registered Architect builders: + +- `@angular/build:ng-packagr`: runs library builds through the legacy `ng-packagr` build pipeline. +- `@angular/build:library`: runs library builds through Angular CLI's native library builder. + +This benchmark suite is designed to quantify performance impacts and ensure scalability. + +### Previous Benchmarks + +For context, when running a comparable benchmark suite directly against `ng-packagr`, it hinted at a "U-curve" in build times, +where build time per entry point in `ng-packagr` drops until entry point N (with N varying in the range 200..500). +This shape indicates an initial overhead as a possible explanation for a small number of entry points. +However, we discovered that `ng-packagr` build time per entry point increases +as the number of entry points scale past a certain N (in the test runs N=200+). + +[Landing Angular CLI's native `@angular/build:library` builder](https://github.com/angular/angular-cli/pull/34106#issuecomment-5744311681), +indicated very favourable performance metrics for the new library builder that can be explained by a number of +architectural changes in the design of the library build pipeline: + +- Shared Stylesheet Bundling +- Public API Invalidation via Declaration Hashing +- Dedicated Worker Pool for Production Builds +- Single Pass Output Emission +- Direct Async Pipeline + +The benchmark design of this suite is built to not only verify the previous results, +but also to confirm or contradict the architectural choices in the library builder pipeline. + +### Goals + +This benchmark suite runs explorative research on the performance of library builds in Angular CLI. +Interesting results from the previous benchmarks indicate 2 measurable goals: + +- Measure the new library builder against itself: How does it scale with the number of entry points? +- Enable A/B benchmarks of the `@angular/build:ng-packagr` builder against `@angular/build:library` builder: How much do library builds benefit from the transition to the native library builder? + +As the previous benchmark runs hint at significant performance improvements in the new library builder, +this benchmark suite aims to quantify the impact of that transition. + +## Benchmark Design + +This suite supports two primary layouts to test different filesystem and entry-point scaling scenarios: + +- **Flat (`--layout=flat`)**: Generates $N$ secondary entry points as direct siblings, where $N$ + comes straight from `--count`. + - Design: Each entry point is named `comp-i` (e.g., `comp-0001`, `comp-0002`). + +- **Deep (`--layout=deep`)**: Generates $2^N$ secondary entry points in a binary tree structure. + - Design: Creates a nested directory structure (`sub-a`/`sub-b` at each level). + - Depth $N$ is not an independent sweep parameter -- see "Sweep rounds" below. + +Each entry point consists of a `public-api.ts` and a component file with a bit of per-entry +content derived from the entry's id (e.g. `

Component no. 0042

`), so entries aren't +byte-identical copies of each other. + +### Builder axis (A/B) + +`--builder=` selects which Architect builder each fixture targets. The same synthetic sources +are emitted in either builder's project shape: + +| Builder | Architect builder | Project shape | +| :----------- | :-------------------------- | :----------------------------------------------------------------------------------------------------------------------- | +| `library` | `@angular/build:library` | One root `package.json` with a flat `exports` map; no `ng-package.json`. | +| `ng-packagr` | `@angular/build:ng-packagr` | Root `ng-package.json` (`dest`, `lib.entryFile`) plus one `ng-package.json` per secondary entry point; no `exports` map. | + +Defaults to `library` only. Pass `--builder=library,ng-packagr` for an A/B run: the builder is the +innermost loop, so the two builds of an identical fixture size run back to back. `inline-scss` +sets `inlineStyleLanguage` on the builder for `library` and in `ng-package.json` for `ng-packagr`. + +### Style-flavor matrix + +`--style-flavor` is a second, independent sweep axis (crossed with `--layout` and the +count/depth sweep) controlling each entry point's template location and stylesheet +presence/language: + +| Flavor | Template | Stylesheet | +| :-------------- | :--------------------- | :------------------------------------------------ | +| `inline` | inline `template` | none | +| `inline-css` | inline `template` | inline `styles`, plain CSS | +| `inline-scss` | inline `template` | inline `styles`, SCSS (via `inlineStyleLanguage`) | +| `external` | external `templateUrl` | none | +| `external-css` | external `templateUrl` | external `styleUrl`, `.css` file | +| `external-scss` | external `templateUrl` | external `styleUrl`, `.scss` file | + +`external` isolates the cost of the extra per-entry template file read from the cost of +stylesheet processing (it has no stylesheet, like `inline`, but reads its template from disk +like the other `external-*` flavors). SCSS content uses a per-entry variable +(`$ref-color-NNNN`) so the preprocessor does real work rather than passing through literal CSS. +Defaults to `inline` only -- the full 6-flavor matrix is opt-in via `--style-flavor=` +since it multiplies the number of builds run by up to 6x. + +### Sweep rounds + +`--count` is the one sweep parameter: a comma-separated list of entry-point count targets. The +default sweep spans small to large entry-point counts in one round, chosen to resolve the +U-shaped scaling pattern observed with `ng-packagr` (see above): +`10, 20, 50, 100, 200, 300, 400, 500, 1000, 2000`. + +For `--layout=deep`, there's no "number of entry points" parameter directly -- a binary tree of +depth $N$ always has exactly $2^N$ leaves/entry points. So `--count`'s targets are translated into +tree depths: for each target `n`, the smallest `depth` with `2^depth >= n`, de-duplicated (e.g. +`300`, `400`, and `500` all round up to the same `depth=9`, i.e. 512 entry points). This keeps +both layouts comparable at roughly the same entry-point scale without exposing depth as a second, +independently-tunable sweep axis. `--depth` still exists as an **explicit override** for +`--layout=deep` -- pass it directly to sweep exact tree depths instead of entry-point targets -- +but it is not part of the default sweep design and is ignored for `--layout=flat`. + +## Running the Benchmarks + +```sh +# Ensure the builder is built first +pnpm build + +# Run the benchmark +pnpm admin benchmark library-builder [options] +``` + +### Options + +| Option | Description | Default | +| :------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------- | :--------------------------------------- | +| `--layout=` | Fixture layout | `flat` | +| `--builder=` | Builders to benchmark, one or more of `library`, `ng-packagr`. Pass both for an A/B run | `library` | +| `--count=` | Entry-point count targets, drives both layouts | `10,20,50,100,200,300,400,500,1000,2000` | +| `--depth=` | **`layout=deep` only.** Explicit tree depths, overriding the depths otherwise derived from `--count` | derived from `--count` | +| `--style-flavor=` | Style-flavor targets (see "Style-flavor matrix" above), one or more of `inline`, `inline-css`, `inline-scss`, `external`, `external-css`, `external-scss` | `inline` | +| `--iterations=` | Measured iterations per size | `3` | +| `--json` | Output raw per-iteration measurements as JSON to stdout | N/A | +| `--csv=` | Write raw per-iteration measurements as CSV to `` | N/A (opt-in only) | + +> `--count` and `--depth` are not independent sweep axes for `layout=deep`: `--count` is translated +> into depths (smallest `depth` with `2^depth >= n` per target, de-duplicated); `--depth` only +> matters when you want to bypass that translation and name exact tree depths yourself. Passing +> both for `layout=deep` makes `--depth` win; `--depth` is ignored entirely for `layout=flat`. + +### Example + +Sweep the flat layout at a single size: + +```sh +pnpm admin benchmark library-builder --layout=flat --count=500 +``` + +A/B run of both builders at a few sizes: + +```sh +pnpm admin benchmark library-builder --builder=library,ng-packagr --layout=flat --count=10,50,200 +``` + +Sweep the full style-flavor matrix at a single size, both layouts: + +```sh +pnpm admin benchmark library-builder --layout=flat --count=500 \ + --style-flavor=inline,inline-css,inline-scss,external,external-css,external-scss +pnpm admin benchmark library-builder --layout=deep --count=500 \ + --style-flavor=inline,inline-css,inline-scss,external,external-css,external-scss +``` + +## Benchmark Results + +Each entry below is one benchmark run, with its own summary and conclusions. + +### 2026-10-12: Iteration 1 baseline sweep (native library builder, flat & deep) + +First full run of the default sweep against `@angular/build:library` only — no `ng-packagr` A/B comparison yet (still Future Work). +Fixtures use inline HTML templates only, no stylesheets. +Default `--count` sweep, `--iterations=3`, both layouts. +Environment: 4 vCPU (a CFS quota on 48 cores @ 2.4Ghz base / 3.5Ghz burst), 16 GiB cgroup limit, NVMe SSD. +Raw log: `tmp/benchmark-run-1791795379.log` (gitignored; not included in this commit). + +#### Flat layout (inline templates only) + +| Entries | Median | ms/entry | +| ------: | -----: | -------: | +| 10 | 3.25s | 324.63 | +| 20 | 3.28s | 164.14 | +| 50 | 3.33s | 66.61 | +| 100 | 3.75s | 37.53 | +| 200 | 4.31s | 21.56 | +| 300 | 4.90s | 16.35 | +| 400 | 5.51s | 13.78 | +| 500 | 6.18s | 12.36 | +| 1000 | 10.29s | 10.29 | +| 2000 | 19.47s | 9.74 | + +#### Deep layout (inline templates only) + +| Entries | Median | ms/entry | +| ------: | -----: | -------: | +| 16 | 3.27s | 204.47 | +| 32 | 3.30s | 103.16 | +| 64 | 3.57s | 55.74 | +| 128 | 4.20s | 32.84 | +| 256 | 4.70s | 18.36 | +| 512 | 6.31s | 12.32 | +| 1024 | 10.25s | 10.01 | +| 2048 | 21.67s | 10.58 | + +**Conclusion:** per-entry build time decreases monotonically across both layouts and flattens out +around 9.6-10.6ms/entry at the largest sizes tested -- no U-shaped climb-back-up at any tested +size (up to 2000/2048 entries), unlike the pattern reported for `ng-packagr` past N≈200-500. Flat +and deep track each other closely at comparable entry-point scale, suggesting the directory +layout itself isn't a meaningful cost driver for the native builder, just raw entry-point count. +`deep`'s depth-11 run (2048 entries) showed slightly higher run-to-run variance (20.78s-22.46s) +than `flat`'s 2000-entry run (19.46s-19.63s) -- possibly directory-tree-walk overhead at depth, +but not a regression, and not yet investigated further. + +All 60 builds (10 sizes x 3 iterations x 2 layouts) completed with `status: "ok"`; zero failures. + +### 2026-10-16: Style-flavor matrix implemented, fixture-correctness verification + +Implemented the style-flavor matrix (`--style-flavor`, see "Style-flavor matrix" above): +`inline`, `inline-css`, `inline-scss`, `external`, `external-css`, `external-scss`, as a sweep +axis independent of `--layout`/`--count`/`--depth`. Defaults to `inline` only (matching the +prior behavior) since the full matrix multiplies build count by up to 6x; opt in with +`--style-flavor=`. + +This entry only verifies fixture correctness at small scale, not full-sweep performance +numbers -- a full style-flavor sweep is still Future Work. + +Verification performed: + +- `--layout=flat --count=5` and `--layout=deep --count=5` (8 entries, depth 3), all 6 flavors, + `--iterations=1`: all 12 builds completed with `status: "ok"`. +- Manually inspected a generated `inline-scss` fixture's `angular.json` to confirm + `inlineStyleLanguage: "scss"` is set only for that flavor, and compiled it standalone to + confirm the per-entry SCSS variable (`$ref-color-NNNN`) resolves to a literal color in the + built `.mjs` output (not left as unresolved `$...` syntax) -- i.e. SCSS is actually being + preprocessed, not passed through as plain CSS. +- Repeated the same compiled-output check for `external-scss` with the `deep` layout -- + confirms external `.scss` stylesheets referenced via `styleUrl` also compile correctly. +- Verified `--csv` and `--json` output both include the new `styleFlavor` field. +- Verified `--style-flavor` rejects an unrecognized flavor name with a clear error listing the + valid set, consistent with existing `--layout`/`--count`/`--depth` validation. + +### 2026-10-23: First A/B benchmark library builder vs. ng-packagr + +Added the `--builder` axis: the fixture generator can now emit `ng-packagr`-shaped projects +(per-entry-point `ng-package.json`) next to the native builder's `exports`-map projects, +and `builder` is part of the JSON/CSV output. + +We now have a basis for a first real A/B run. The full sweep takes hours and needs proper +preparation, so we decided to reduce the scope of a first A/B benchmark run: + +- First A/B run of `@angular/build:library` against `@angular/build:ng-packagr`, using the fixtures above: + - flat layout, + - inline templates only (no stylesheets), + - `--iterations=3`, + - sizes 10 to 1000 (reduced from the default sweep to fit a ~30 minute timw window; no 2000-entry run). +- Environment: 4 vCPU (a CFS quota on 48 cores @ 2.4Ghz base / 3.5Ghz burst), 16 GiB cgroup limit, NVMe SSD, Linux, Node v24.15.0, `ng-packagr` 22.3.0-next.2 +- Result:All 36 builds completed with `status: "ok"`. + +```sh +pnpm admin benchmark library-builder --builder=library,ng-packagr --layout=flat \ + --count=10,50,100,200,500,1000 --iterations=3 --csv= +``` + +| Entries | `library` median | `ng-packagr` median | Speedup | `library` ms/entry | `ng-packagr` ms/entry | +| ------: | ---------------: | ------------------: | ------: | -----------------: | --------------------: | +| 10 | 3.33s | 5.10s | 1.5x | 333.2 | 510.3 | +| 50 | 3.54s | 14.20s | 4.0x | 70.8 | 283.9 | +| 100 | 3.93s | 25.21s | 6.4x | 39.3 | 252.1 | +| 200 | 4.47s | 48.16s | 10.8x | 22.3 | 240.8 | +| 500 | 6.03s | 122.58s | 20.3x | 12.1 | 245.2 | +| 1000 | 10.01s | 269.98s | 27.0x | 10.0 | 270.0 | + +**Conclusion:** `library` costs ~3s fixed plus 5-8ms per additional entry point, so ms/entry keeps +falling to ~10 at N=1000. `ng-packagr` costs ~225ms per additional entry up to N=200, then 248ms +(200-500) and 295ms (500-1000). The speedup grows from 1.5x (N=10) to 27x (N=1000). + +Caveats: + +- The `ng-packagr` rise past N=500 is a trend on medians of 3 runs (N=1000 runs span 256-274s), not yet the U-curve reported in earlier benchmarks. +- Output equivalence (bundles, declarations) of the two builders is not verified. +- Not measured: `deep` layout, stylesheets, N>=2000, other machine shapes. + +Per-iteration CSV and the raw log are kept outside the repository (`tmp/benchmark-2026-10-23.md`). + +### 2026-10-24: A/B Benchmark across `--style-flavor` and `--count` axis + +For the fixtures in `flat` layout: + +```sh +$ pnpm admin benchmark library-builder \ + --builder=library,ng-packagr \ + --layout=flat \ + --style-flavor=inline,inline-css,inline-scss,external,external-css,external-scss \ + --count=10,50,100 --iterations=5 \ + --csv=./tmp/benchmark-2026-10-24.csv +``` + +Environment: GitHub Codespace (2 cores, 8 GB): + +```sh +$ cat /proc/meminfo +MemTotal: 8131876 kB +MemFree: 260780 kB +MemAvailable: 2296448 kB + +$ cat /proc/cpuinfo +processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 106 +model name : Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +cpu MHz : 3393.038 + +processor : 1 +vendor_id : GenuineIntel +cpu family : 6 +model : 106 +model name : Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +cpu MHz : 3472.520 +``` + +Results: + +- `@angular/build:library` beats `@angular/build:ng-packagr` in all sweep axis, starting from the small N <= 100 entry point counts +- Build times of `@angular/build:library` stay consistent across the `--style-flavor` axis, indicating roughly x-y ms per entry point: + - build time per entry point stays roughly flat regardless of inline/external templates and stylesheet language + - the primary driver of total build time (for the full library) is the number of entry points (`count` axis) +- Total build time (for the full library) `@angular/build:library` + +Build log: + +```log +$ node --no-warnings=ExperimentalWarning --experimental-transform-types ./scripts/devkit-admin.mts benchmark library-builder --builder=library,ng-packagr --layout=flat --style-flavor=inline,inline-css,inline-scss,external,external-css,external-scss --count=10,50,100 --iterations=5 --csv=./tmp/benchmark-2026-10-24.csv + +[library-builder] library-flat-10-inline (10 entry points) + run 1/5: done in 10.63s + run 2/5: done in 4.11s + run 3/5: done in 4.08s + run 4/5: done in 4.08s + run 5/5: done in 4.28s + median: 4.11s (410.52ms/entry) + +[library-builder] ng-packagr-flat-10-inline (10 entry points) + run 1/5: done in 7.57s + run 2/5: done in 6.12s + run 3/5: done in 6.18s + run 4/5: done in 6.44s + run 5/5: done in 6.30s + median: 6.30s (630.33ms/entry) + +[library-builder] library-flat-50-inline (50 entry points) + run 1/5: done in 4.58s + run 2/5: done in 4.52s + run 3/5: done in 4.54s + run 4/5: done in 4.70s + run 5/5: done in 4.52s + median: 4.54s (90.88ms/entry) + +[library-builder] ng-packagr-flat-50-inline (50 entry points) + run 1/5: done in 16.46s + run 2/5: done in 17.38s + run 3/5: done in 17.73s + run 4/5: done in 18.60s + run 5/5: done in 16.53s + median: 17.38s (347.50ms/entry) + +[library-builder] library-flat-100-inline (100 entry points) + run 1/5: done in 5.23s + run 2/5: done in 5.11s + run 3/5: done in 5.54s + run 4/5: done in 5.14s + run 5/5: done in 5.30s + median: 5.23s (52.27ms/entry) + +[library-builder] ng-packagr-flat-100-inline (100 entry points) + run 1/5: done in 31.45s + run 2/5: done in 30.15s + run 3/5: done in 31.59s + run 4/5: done in 30.58s + run 5/5: done in 29.47s + median: 30.58s (305.78ms/entry) + +[library-builder] library-flat-10-inline-css (10 entry points) + run 1/5: done in 4.91s + run 2/5: done in 3.98s + run 3/5: done in 4.28s + run 4/5: done in 4.11s + run 5/5: done in 4.12s + median: 4.12s (412.22ms/entry) + +[library-builder] ng-packagr-flat-10-inline-css (10 entry points) + run 1/5: done in 6.05s + run 2/5: done in 6.41s + run 3/5: done in 6.13s + run 4/5: done in 6.19s + run 5/5: done in 6.29s + median: 6.19s (619.22ms/entry) + +[library-builder] library-flat-50-inline-css (50 entry points) + run 1/5: done in 4.69s + run 2/5: done in 5.00s + run 3/5: done in 5.02s + run 4/5: done in 5.58s + run 5/5: done in 5.39s + median: 5.02s (100.46ms/entry) + +[library-builder] ng-packagr-flat-50-inline-css (50 entry points) + run 1/5: done in 18.59s + run 2/5: done in 17.77s + run 3/5: done in 16.63s + run 4/5: done in 17.64s + run 5/5: done in 16.26s + median: 17.64s (352.87ms/entry) + +[library-builder] library-flat-100-inline-css (100 entry points) + run 1/5: done in 5.21s + run 2/5: done in 5.57s + run 3/5: done in 5.15s + run 4/5: done in 5.25s + run 5/5: done in 5.15s + median: 5.21s (52.12ms/entry) + +[library-builder] ng-packagr-flat-100-inline-css (100 entry points) + run 1/5: done in 28.20s + run 2/5: done in 27.88s + run 3/5: done in 27.33s + run 4/5: done in 28.18s + run 5/5: done in 27.30s + median: 27.88s (278.84ms/entry) + +[library-builder] library-flat-10-inline-scss (10 entry points) + run 1/5: done in 5.94s + run 2/5: done in 4.45s + run 3/5: done in 4.34s + run 4/5: done in 4.37s + run 5/5: done in 4.18s + median: 4.37s (436.70ms/entry) + +[library-builder] ng-packagr-flat-10-inline-scss (10 entry points) + run 1/5: done in 7.37s + run 2/5: done in 6.54s + run 3/5: done in 6.29s + run 4/5: done in 6.32s + run 5/5: done in 6.54s + median: 6.54s (653.80ms/entry) + +[library-builder] library-flat-50-inline-scss (50 entry points) + run 1/5: done in 4.84s + run 2/5: done in 4.77s + run 3/5: done in 4.81s + run 4/5: done in 4.72s + run 5/5: done in 4.98s + median: 4.81s (96.29ms/entry) + +[library-builder] ng-packagr-flat-50-inline-scss (50 entry points) + run 1/5: done in 16.12s + run 2/5: done in 17.95s + run 3/5: done in 15.98s + run 4/5: done in 15.99s + run 5/5: done in 16.54s + median: 16.12s (322.37ms/entry) + +[library-builder] library-flat-100-inline-scss (100 entry points) + run 1/5: done in 5.48s + run 2/5: done in 5.42s + run 3/5: done in 5.40s + run 4/5: done in 5.38s + run 5/5: done in 5.43s + median: 5.42s (54.18ms/entry) + +[library-builder] ng-packagr-flat-100-inline-scss (100 entry points) + run 1/5: done in 28.00s + run 2/5: done in 28.23s + run 3/5: done in 28.66s + run 4/5: done in 27.14s + run 5/5: done in 25.12s + median: 28.00s (279.99ms/entry) + +[library-builder] library-flat-10-external (10 entry points) + run 1/5: done in 3.59s + run 2/5: done in 3.65s + run 3/5: done in 3.93s + run 4/5: done in 3.65s + run 5/5: done in 3.66s + median: 3.65s (365.17ms/entry) + +[library-builder] ng-packagr-flat-10-external (10 entry points) + run 1/5: done in 6.10s + run 2/5: done in 5.66s + run 3/5: done in 5.49s + run 4/5: done in 5.71s + run 5/5: done in 5.88s + median: 5.71s (570.85ms/entry) + +[library-builder] library-flat-50-external (50 entry points) + run 1/5: done in 4.15s + run 2/5: done in 4.10s + run 3/5: done in 4.13s + run 4/5: done in 4.12s + run 5/5: done in 4.16s + median: 4.13s (82.62ms/entry) + +[library-builder] ng-packagr-flat-50-external (50 entry points) + run 1/5: done in 16.06s + run 2/5: done in 15.63s + run 3/5: done in 14.68s + run 4/5: done in 17.45s + run 5/5: done in 17.25s + median: 16.06s (321.17ms/entry) + +[library-builder] library-flat-100-external (100 entry points) + run 1/5: done in 4.52s + run 2/5: done in 4.71s + run 3/5: done in 4.67s + run 4/5: done in 4.57s + run 5/5: done in 4.51s + median: 4.57s (45.68ms/entry) + +[library-builder] ng-packagr-flat-100-external (100 entry points) + run 1/5: done in 27.89s + run 2/5: done in 28.68s + run 3/5: done in 29.08s + run 4/5: done in 26.58s + run 5/5: done in 27.23s + median: 27.89s (278.87ms/entry) + +[library-builder] library-flat-10-external-css (10 entry points) + run 1/5: done in 3.74s + run 2/5: done in 3.82s + run 3/5: done in 3.72s + run 4/5: done in 3.91s + run 5/5: done in 3.80s + median: 3.80s (379.65ms/entry) + +[library-builder] ng-packagr-flat-10-external-css (10 entry points) + run 1/5: done in 5.63s + run 2/5: done in 5.86s + run 3/5: done in 5.66s + run 4/5: done in 5.50s + run 5/5: done in 5.98s + median: 5.66s (565.62ms/entry) + +[library-builder] library-flat-50-external-css (50 entry points) + run 1/5: done in 4.13s + run 2/5: done in 4.35s + run 3/5: done in 4.17s + run 4/5: done in 4.22s + run 5/5: done in 4.34s + median: 4.22s (84.47ms/entry) + +[library-builder] ng-packagr-flat-50-external-css (50 entry points) + run 1/5: done in 14.36s + run 2/5: done in 14.75s + run 3/5: done in 14.42s + run 4/5: done in 15.15s + run 5/5: done in 15.48s + median: 14.75s (295.02ms/entry) + +[library-builder] library-flat-100-external-css (100 entry points) + run 1/5: done in 4.87s + run 2/5: done in 4.90s + run 3/5: done in 4.87s + run 4/5: done in 4.66s + run 5/5: done in 4.77s + median: 4.87s (48.71ms/entry) + +[library-builder] ng-packagr-flat-100-external-css (100 entry points) + run 1/5: done in 26.43s + run 2/5: done in 26.93s + run 3/5: done in 27.01s + run 4/5: done in 26.46s + run 5/5: done in 24.85s + median: 26.46s (264.56ms/entry) + +[library-builder] library-flat-10-external-scss (10 entry points) + run 1/5: done in 3.87s + run 2/5: done in 3.98s + run 3/5: done in 3.85s + run 4/5: done in 3.92s + run 5/5: done in 3.83s + median: 3.87s (387.06ms/entry) + +[library-builder] ng-packagr-flat-10-external-scss (10 entry points) + run 1/5: done in 5.81s + run 2/5: done in 5.89s + run 3/5: done in 5.97s + run 4/5: done in 5.91s + run 5/5: done in 6.05s + median: 5.91s (591.38ms/entry) + +[library-builder] library-flat-50-external-scss (50 entry points) + run 1/5: done in 4.59s + run 2/5: done in 4.49s + run 3/5: done in 4.42s + run 4/5: done in 4.34s + run 5/5: done in 4.48s + median: 4.48s (89.69ms/entry) + +[library-builder] ng-packagr-flat-50-external-scss (50 entry points) + run 1/5: done in 16.15s + run 2/5: done in 16.11s + run 3/5: done in 15.45s + run 4/5: done in 16.25s + run 5/5: done in 16.53s + median: 16.15s (323.04ms/entry) + +[library-builder] library-flat-100-external-scss (100 entry points) + run 1/5: done in 5.68s + run 2/5: done in 5.58s + run 3/5: done in 5.46s + run 4/5: done in 5.49s + run 5/5: done in 6.03s + median: 5.58s (55.81ms/entry) + +[library-builder] ng-packagr-flat-100-external-scss (100 entry points) + run 1/5: done in 29.19s + run 2/5: done in 28.73s + run 3/5: done in 30.11s + run 4/5: done in 25.30s + run 5/5: done in 25.74s + median: 28.73s (287.26ms/entry) +``` + +Results CSV: because of human mistake, the csv output was not written. + +## Future Work + +- Extend the A/B run against `ng-packagr` (first run: 2026-10-23, flat/inline, N up to 1000): `deep` layout, the style-flavor matrix, N=2000, more iterations, and an output-equivalence check of the two builders. +- Full style-flavor matrix sweep (default `--count` sweep x 6 flavors x 2 layouts) to quantify the actual performance impact of stylesheet processing and external file reads, now that the matrix is implemented. +- Dependencies between entry points. +- Shared stylesheet tokens imported by multiple components. +- Watch mode. +- Peak memory and CPU time metrics per build, in addition to wall-clock build time. Note: + `child.resourceUsage()` does **not** exist on Node's `ChildProcess` (only `process.resourceUsage()` + exists, and it reports the calling process's own usage, not a child's) -- the real path is + polling `/proc//status` (`VmHWM`) and `/proc//stat` (`utime`+`stime`) on a timer while + the build subprocess is alive, which is Linux-only and requires moving `buildOnce` off + `spawnSync` onto `spawn()` so a polling timer can interleave with the child's lifetime. +- Library "taste" variants beyond plain flat/deep, e.g. flat/deep with inter-entry-point dependencies or shared SCSS tokens (currently out of scope, listed above). diff --git a/scripts/benchmarks/library-builder/fixtures.mts b/scripts/benchmarks/library-builder/fixtures.mts new file mode 100644 index 000000000000..9d7ed4bb97bb --- /dev/null +++ b/scripts/benchmarks/library-builder/fixtures.mts @@ -0,0 +1,393 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Generates synthetic Angular Package Format (APF) library fixtures for benchmarking +// `@angular/build:library` and `@angular/build:ng-packagr`. The same synthetic sources are emitted +// in either builder's project shape: a flat `exports` map in one root package.json for the native +// builder, or the per-entry-point `ng-package.json` convention for ng-packagr. + +import fs from 'node:fs'; +import path from 'node:path'; + +export type Layout = 'flat' | 'deep'; + +/** Which Architect builder the generated project targets. */ +export type LibraryBuilder = 'library' | 'ng-packagr'; + +/** All library builders, in A/B order (native builder first, legacy builder second). */ +export const LIBRARY_BUILDERS: readonly LibraryBuilder[] = ['library', 'ng-packagr']; + +export const DEFAULT_LIBRARY_BUILDER: LibraryBuilder = 'library'; + +/** + * Style-flavor matrix: crosses template location (inline vs. external `templateUrl`) with + * stylesheet presence/language (none, CSS, SCSS). `external` has no stylesheet, matching + * `inline`'s "no stylesheet" baseline but with an external `templateUrl` file instead of an + * inline `template` string -- isolating the cost of extra per-entry file reads from the cost of + * stylesheet processing. + */ +export type StyleFlavor = + 'inline' | 'inline-css' | 'inline-scss' | 'external' | 'external-css' | 'external-scss'; + +/** All style flavors, in matrix order (inline row, then external row). */ +export const STYLE_FLAVORS: readonly StyleFlavor[] = [ + 'inline', + 'inline-css', + 'inline-scss', + 'external', + 'external-css', + 'external-scss', +]; + +export const DEFAULT_STYLE_FLAVOR: StyleFlavor = 'inline'; + +export interface FixtureOptions { + layout: Layout; + + /** Number of secondary entry points for `layout: 'flat'`. */ + count?: number; + + /** Binary-tree depth for `layout: 'deep'` (2^depth leaf entry points). */ + depth?: number; + + /** Template/stylesheet flavor for every entry point in the fixture. Defaults to `inline`. */ + styleFlavor?: StyleFlavor; + + /** Builder the generated project targets. Defaults to `library`. */ + builder?: LibraryBuilder; +} + +/** Per-entry CSS/SCSS content, varied by `id` like the template so entries aren't byte-identical. */ +function buildCssContent(selector: string, id: string): string { + return `/* entry ${id} */\n.${selector} {\n color: #336699;\n}\n`; +} + +function buildScssContent(selector: string, id: string): string { + return `$ref-color-${id}: #336699;\n.${selector} {\n color: $ref-color-${id};\n}\n`; +} + +function buildComponentSource( + className: string, + selector: string, + id: string, + compBase: string, + styleFlavor: StyleFlavor, +): string { + const metadata: string[] = [` selector: '${selector}',`]; + + switch (styleFlavor) { + case 'inline': + metadata.push(` template: '

Component no. ${id}

',`); + break; + case 'inline-css': + metadata.push(` template: '

Component no. ${id}

',`); + metadata.push(` styles: [\`.${selector} { color: #336699; }\`],`); + break; + case 'inline-scss': + metadata.push(` template: '

Component no. ${id}

',`); + metadata.push( + ` styles: [\`$ref-color-${id}: #336699; .${selector} { color: $ref-color-${id}; }\`],`, + ); + break; + case 'external': + metadata.push(` templateUrl: './${compBase}.component.html',`); + break; + case 'external-css': + metadata.push(` templateUrl: './${compBase}.component.html',`); + metadata.push(` styleUrl: './${compBase}.component.css',`); + break; + case 'external-scss': + metadata.push(` templateUrl: './${compBase}.component.html',`); + metadata.push(` styleUrl: './${compBase}.component.scss',`); + break; + } + + return `import { Component } from '@angular/core'; + +@Component({ +${metadata.join('\n')} +}) +export class ${className} {} +`; +} + +/** Writes one secondary entry point's source files (component + public-api.ts) into `entryDir`. */ +function writeEntryPointSource( + entryDir: string, + id: string, + styleFlavor: StyleFlavor, + builder: LibraryBuilder, +): void { + fs.mkdirSync(entryDir, { recursive: true }); + + if (builder === 'ng-packagr') { + // ng-packagr discovers secondary entry points by the presence of a per-directory ng-package.json. + fs.writeFileSync( + path.join(entryDir, 'ng-package.json'), + JSON.stringify({ lib: { entryFile: 'public-api.ts' } }, null, 2) + '\n', + ); + } + + const className = `RefComponent${id}`; + const selector = `ref-comp-${id}`; + const compBase = `comp-${id}`; + + fs.writeFileSync( + path.join(entryDir, 'public-api.ts'), + `export * from './${compBase}.component';\n`, + ); + + fs.writeFileSync( + path.join(entryDir, `${compBase}.component.ts`), + buildComponentSource(className, selector, id, compBase, styleFlavor), + ); + + if ( + styleFlavor === 'external' || + styleFlavor === 'external-css' || + styleFlavor === 'external-scss' + ) { + fs.writeFileSync( + path.join(entryDir, `${compBase}.component.html`), + `

Component no. ${id}

\n`, + ); + } + + if (styleFlavor === 'external-css') { + fs.writeFileSync( + path.join(entryDir, `${compBase}.component.css`), + buildCssContent(selector, id), + ); + } + + if (styleFlavor === 'external-scss') { + fs.writeFileSync( + path.join(entryDir, `${compBase}.component.scss`), + buildScssContent(selector, id), + ); + } +} + +/** Generates `count` flat secondary entry points as direct siblings, returning their relative dirs. */ +function generateFlatEntries( + outDir: string, + count: number, + styleFlavor: StyleFlavor, + builder: LibraryBuilder, +): string[] { + const width = String(count).length; + const dirs: string[] = []; + for (let i = 1; i <= count; i++) { + const id = String(i).padStart(width, '0'); + const relDir = `comp-${id}`; + writeEntryPointSource(path.join(outDir, relDir), id, styleFlavor, builder); + dirs.push(relDir); + } + + return dirs; +} + +/** + * Generates a binary tree `depth` levels deep (`sub-a`/`sub-b` at each level), placing one + * secondary entry point at each of the 2^depth leaves. Returns their relative dirs. + */ +function generateDeepEntries( + outDir: string, + depth: number, + styleFlavor: StyleFlavor, + builder: LibraryBuilder, +): string[] { + const total = 2 ** depth; + const width = String(total).length; + const dirs: string[] = []; + let counter = 0; + + function build(currentRelDir: string, level: number): void { + if (level === depth) { + counter++; + const id = String(counter).padStart(width, '0'); + writeEntryPointSource(path.join(outDir, currentRelDir), id, styleFlavor, builder); + dirs.push(currentRelDir); + + return; + } + for (const child of ['sub-a', 'sub-b']) { + build(path.join(currentRelDir, child), level + 1); + } + } + + build('.', 0); + + return dirs.map((d) => path.normalize(d)); +} + +/** "sub-a/sub-b" -> "sub-a-sub-b"; "comp-0001" -> "comp-0001". Used as the exports map subpath key. */ +function entryKey(relDir: string): string { + return relDir.split(path.sep).join('-'); +} + +/** + * Generates a complete `@angular/build:library`-buildable project at `outDir`: a primary entry + * point, N secondary entry points per `options`, and the angular.json/tsconfig/package.json + * scaffolding the builder needs. Returns the number of secondary entry points generated. + */ +export function generateFixture(outDir: string, options: FixtureOptions): number { + fs.rmSync(outDir, { recursive: true, force: true }); + fs.mkdirSync(outDir, { recursive: true }); + + const styleFlavor = options.styleFlavor ?? DEFAULT_STYLE_FLAVOR; + const builder = options.builder ?? DEFAULT_LIBRARY_BUILDER; + + fs.writeFileSync( + path.join(outDir, 'public-api.ts'), + `export const REFERENCE_VERSION = '1.0.0';\n`, + ); + + const secondaryDirs = + options.layout === 'flat' + ? generateFlatEntries(outDir, options.count ?? 10, styleFlavor, builder) + : generateDeepEntries(outDir, options.depth ?? 2, styleFlavor, builder); + + // Only the native builder reads an `exports` map; ng-packagr derives (and writes) it itself from + // the per-entry-point ng-package.json files. + const exportsMap: Record = { '.': './public-api.ts' }; + if (builder === 'library') { + for (const relDir of secondaryDirs) { + exportsMap[`./${entryKey(relDir)}`] = `./${relDir.split(path.sep).join('/')}/public-api.ts`; + } + } + + // Fallback to a hardcoded version if the path lookup fails in the sandbox/environment. + let peerRange = '^22.0.0'; + try { + const angularCoreVersion = JSON.parse( + fs.readFileSync( + new URL('../../../node_modules/@angular/core/package.json', import.meta.url), + 'utf8', + ), + ).version as string; + peerRange = `^${angularCoreVersion.replace(/\.\d+$/, '')}`; + } catch { + // Ignore error, use fallback + } + + fs.writeFileSync( + path.join(outDir, 'package.json'), + JSON.stringify( + { + name: `ref-${options.layout}`, + version: '0.0.1', + peerDependencies: { + '@angular/core': peerRange, + '@angular/common': peerRange, + }, + ...(builder === 'library' ? { exports: exportsMap } : {}), + }, + null, + 2, + ) + '\n', + ); + + fs.writeFileSync( + path.join(outDir, 'tsconfig.json'), + JSON.stringify( + { + compileOnSave: false, + compilerOptions: { + outDir: './dist/out-tsc', + sourceMap: true, + declaration: false, + moduleResolution: 'bundler', + experimentalDecorators: true, + target: 'es2022', + module: 'es2022', + useDefineForClassFields: false, + lib: ['es2022', 'dom'], + }, + angularCompilerOptions: { + enableIvy: true, + disableTypeScriptVersionCheck: true, + }, + }, + null, + 2, + ) + '\n', + ); + + fs.writeFileSync( + path.join(outDir, 'tsconfig.lib.json'), + JSON.stringify( + { + extends: './tsconfig.json', + compilerOptions: { + declaration: true, + declarationMap: true, + inlineSources: true, + types: [], + }, + exclude: ['**/*.spec.ts'], + }, + null, + 2, + ) + '\n', + ); + + let buildTarget: { builder: string; options: Record }; + if (builder === 'library') { + const buildOptions: Record = { + tsConfig: './tsconfig.lib.json', + outputPath: 'dist', + }; + if (styleFlavor === 'inline-scss') { + // Inline style language is a project-wide builder option, not per-component -- this is the + // only flavor that needs it set, since external .scss files are detected by extension and + // 'css' (the default) already covers the other flavors. + buildOptions['inlineStyleLanguage'] = 'scss'; + } + buildTarget = { builder: '@angular/build:library', options: buildOptions }; + } else { + const ngPackage: Record = { + dest: 'dist', + lib: { entryFile: 'public-api.ts' }, + }; + if (styleFlavor === 'inline-scss') { + // For ng-packagr the inline style language is configured in ng-package.json, not the builder. + ngPackage['inlineStyleLanguage'] = 'scss'; + } + fs.writeFileSync( + path.join(outDir, 'ng-package.json'), + JSON.stringify(ngPackage, null, 2) + '\n', + ); + buildTarget = { + builder: '@angular/build:ng-packagr', + options: { project: './ng-package.json', tsConfig: './tsconfig.lib.json' }, + }; + } + + fs.writeFileSync( + path.join(outDir, 'angular.json'), + JSON.stringify( + { + version: 1, + cli: { cache: { enabled: false } }, + projects: { + lib: { + root: '.', + projectType: 'library', + sourceRoot: '.', + targets: { build: buildTarget }, + }, + }, + }, + null, + 2, + ) + '\n', + ); + + return secondaryDirs.length; +} diff --git a/scripts/benchmarks/library-builder/harness.mts b/scripts/benchmarks/library-builder/harness.mts new file mode 100644 index 000000000000..c373e74528c8 --- /dev/null +++ b/scripts/benchmarks/library-builder/harness.mts @@ -0,0 +1,161 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Runs and measures a single `@angular/build:library` or `@angular/build:ng-packagr` cold build via Architect's standalone CLI. +// `fixtures.mts` owns generating the on-disk project; this module owns spawning the builder +// against it and reporting what happened. Kept separate so each file has one job. + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import type { Layout, LibraryBuilder, StyleFlavor } from './fixtures.mts'; + +export interface BuildMeasurement { + builder: LibraryBuilder; + layout: Layout; + styleFlavor: StyleFlavor; + entryPoints: number; + iteration: number; + durationMs: number; + status: 'ok' | 'fail'; +} + +export interface BuilderPaths { + buildPkg: string; + corePkg: string; + architectPkg: string; + architectCli: string; +} + +export interface BuilderEnv extends BuilderPaths { + /** NODE_PATH value to pass to a spawned child so its bare `require()`s resolve into dist/. */ + nodePath: string; +} + +/** Verifies `@angular/build` and `@angular-devkit/architect` are built before attempting a build. */ +export function checkBuilderIsBuilt(repoRoot: string, logger: Console): BuilderPaths | undefined { + const buildPkg = path.join(repoRoot, 'dist', '@angular', 'build'); + const corePkg = path.join(repoRoot, 'dist', '@angular-devkit', 'core'); + const architectPkg = path.join(repoRoot, 'dist', '@angular-devkit', 'architect'); + const architectCli = path.join(architectPkg, 'bin', 'cli.js'); + + if (!fs.existsSync(path.join(buildPkg, 'package.json')) || !fs.existsSync(architectCli)) { + logger.error( + 'Error: @angular/build and/or @angular-devkit/architect have not been built yet.\n' + + 'Run "pnpm build" (or "pnpm admin build --local" for a faster local build) first.', + ); + + return undefined; + } + + return { buildPkg, corePkg, architectPkg, architectCli }; +} + +/** Creates/repairs `shimDir/` as a symlink to `target`, skipping if already correct. */ +function linkIntoShim(shimDir: string, name: string, target: string): void { + const linkPath = path.join(shimDir, name); + fs.mkdirSync(path.dirname(linkPath), { recursive: true }); + + try { + if (fs.readlinkSync(linkPath) === target) { + return; + } + fs.rmSync(linkPath, { force: true }); + } catch { + // ENOENT (link doesn't exist yet) -- fall through to create it. + } + + fs.symlinkSync(target, linkPath, 'junction'); +} + +/** + * Prepares the environment a standalone, spawned `architectCli` process needs to resolve its own + * dependencies. `dist/` has no `node_modules` of its own, and a `pnpm install` (`hoist: false`) + * only links `@angular-devkit/core`/`architect`/`@angular/build`'s dependencies into each + * package's own SOURCE directory under `packages/`, not into anything reachable from `dist/`. So + * a bare `require('@angular-devkit/core')` from the built CLI script has nowhere to resolve to. + * + * Fixes this with a small symlink directory mapping each built package's bare specifier to its + * `dist/` output, combined with each package's own `node_modules` (where its actual npm deps -- + * ajv-formats, esbuild, sass, etc. -- are already pnpm-linked) into a `NODE_PATH` value: Node + * treats each `NODE_PATH` entry as an extra global resolution root, so this covers arbitrarily + * nested bare requires without having to enumerate individual third-party packages. + */ +export function prepareBuilderEnv( + repoRoot: string, + builder: BuilderPaths, + shimDir: string, + logger: Console, +): BuilderEnv | undefined { + const ownNodeModulesDirs = [ + path.join(repoRoot, 'packages', 'angular_devkit', 'core', 'node_modules'), + path.join(repoRoot, 'packages', 'angular_devkit', 'architect', 'node_modules'), + path.join(repoRoot, 'packages', 'angular', 'build', 'node_modules'), + ]; + + const missing = ownNodeModulesDirs.filter((dir) => !fs.existsSync(dir)); + if (missing.length > 0) { + logger.error( + 'Error: expected node_modules missing for:\n ' + + missing.join('\n ') + + '\nRun "pnpm install" at the repo root first.', + ); + + return undefined; + } + + fs.mkdirSync(shimDir, { recursive: true }); + linkIntoShim(shimDir, '@angular/build', builder.buildPkg); + linkIntoShim(shimDir, '@angular-devkit/core', builder.corePkg); + linkIntoShim(shimDir, '@angular-devkit/architect', builder.architectPkg); + + const nodePath = [shimDir, ...ownNodeModulesDirs].join(path.delimiter); + + return { ...builder, nodePath }; +} + +/** Spawns one cold `lib:build` run against `projectDir` and times it. */ +export function buildOnce( + projectDir: string, + architectCli: string, + nodePath: string, +): { durationMs: number; status: 'ok' | 'fail' } { + const existingNodePath = process.env['NODE_PATH']; + const mergedNodePath = existingNodePath + ? `${nodePath}${path.delimiter}${existingNodePath}` + : nodePath; + + const start = performance.now(); + const result = spawnSync(process.execPath, [architectCli, 'lib:build'], { + cwd: projectDir, + stdio: 'pipe', + maxBuffer: Infinity, + env: { ...process.env, NODE_PATH: mergedNodePath }, + }); + const durationMs = performance.now() - start; + const status = result.status === 0 ? 'ok' : 'fail'; + if (status === 'fail') { + if (result.error) { + // eslint-disable-next-line no-console + console.error(result.error); + } + // eslint-disable-next-line no-console + console.error(result.stdout?.toString()); + // eslint-disable-next-line no-console + console.error(result.stderr?.toString()); + } + + return { durationMs, status }; +} + +export function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} diff --git a/scripts/benchmarks/library-builder/index.mts b/scripts/benchmarks/library-builder/index.mts new file mode 100644 index 000000000000..3a05b955fccc --- /dev/null +++ b/scripts/benchmarks/library-builder/index.mts @@ -0,0 +1,196 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Benchmarks `@angular/build:library`'s (and optionally `@angular/build:ng-packagr`'s) cold build +// time across a sweep of library sizes. +// This module only orchestrates the sweep: fixture generation lives in fixtures.mts, build +// execution and measurement live in harness.mts. + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + DEFAULT_LIBRARY_BUILDER, + DEFAULT_STYLE_FLAVOR, + type FixtureOptions, + type Layout, + type LibraryBuilder, + type StyleFlavor, + generateFixture, +} from './fixtures.mts'; +import { + type BuildMeasurement, + buildOnce, + checkBuilderIsBuilt, + median, + prepareBuilderEnv, +} from './harness.mts'; +import { formatCsvReport, formatJsonReport } from './reporters.mts'; + +export interface LibraryBuilderBenchmarkOptions { + layout?: Layout; + builder?: LibraryBuilder[]; + count?: number[]; + depth?: number[]; + styleFlavor?: StyleFlavor[]; + iterations?: number; + json?: boolean; + csv?: string; +} + +const repoRoot = fileURLToPath(new URL('../../../', import.meta.url)); +const scratchRoot = path.join(repoRoot, 'dist', '.library-benchmark-fixtures'); +const nodeModulesShimDir = path.join(repoRoot, 'dist', '.library-benchmark-node-modules-shim'); + +/** + * Default entry-point counts for the `flat` layout, chosen to resolve a U-shaped scaling curve. + * Also drives the `deep` layout's default depths (see `depthsForEntryPointTargets`) -- `--count` + * is the one sweep parameter users pick; `--depth` is only ever an explicit override. + */ +const DEFAULT_FLAT_COUNTS = [10, 20, 50, 100, 200, 300, 400, 500, 1000, 2000]; + +/** + * Derives default `deep` layout tree depths from a set of target entry-point counts, picking for + * each target `n` the smallest `depth` with `2^depth >= n`, de-duplicated in ascending order. + * This keeps both layouts comparable at roughly the same entry-point scale instead of an + * arbitrary, independently-chosen depth list. + */ +function depthsForEntryPointTargets(targets: number[]): number[] { + const depths = new Set(); + for (const target of targets) { + depths.add(Math.max(1, Math.ceil(Math.log2(target)))); + } + + return [...depths].sort((a, b) => a - b); +} + +export async function runLibraryBuilderBenchmarks( + options: LibraryBuilderBenchmarkOptions = {}, +): Promise<{ measurements: BuildMeasurement[]; exitCode: number }> { + const builder = checkBuilderIsBuilt(repoRoot, console); + if (!builder) { + return { measurements: [], exitCode: 1 }; + } + + const builderEnv = prepareBuilderEnv(repoRoot, builder, nodeModulesShimDir, console); + if (!builderEnv) { + return { measurements: [], exitCode: 1 }; + } + + const layout = options.layout ?? 'flat'; + const iterations = options.iterations ?? 3; + const sizes = + layout === 'flat' + ? (options.count ?? DEFAULT_FLAT_COUNTS) + : (options.depth ?? depthsForEntryPointTargets(options.count ?? DEFAULT_FLAT_COUNTS)); + // Defaults to just `inline` (matching this suite's original, style-agnostic sweep) rather than + // the full matrix -- the matrix is opt-in via `--style-flavor=` since it multiplies the + // number of builds run by up to 6x. + const styleFlavors = options.styleFlavor ?? [DEFAULT_STYLE_FLAVOR]; + // Builders are the innermost loop so A/B pairs for the same fixture size run back to back. + const builders = options.builder ?? [DEFAULT_LIBRARY_BUILDER]; + + fs.mkdirSync(scratchRoot, { recursive: true }); + const measurements: BuildMeasurement[] = []; + + try { + for (const styleFlavor of styleFlavors) { + for (const sizeOrDepth of sizes) { + for (const builderName of builders) { + const label = + builderName + + '-' + + layout + + '-' + + (layout === 'deep' ? 'depth' + sizeOrDepth : sizeOrDepth) + + '-' + + styleFlavor; + const projectDir = path.join(scratchRoot, label); + + const fixtureOptions: FixtureOptions = + layout === 'flat' + ? { layout, count: sizeOrDepth, styleFlavor, builder: builderName } + : { layout, depth: sizeOrDepth, styleFlavor, builder: builderName }; + const entryPoints = generateFixture(projectDir, fixtureOptions); + + if (!options.json) { + // eslint-disable-next-line no-console + console.log('\n[library-builder] ' + label + ' (' + entryPoints + ' entry points)'); + } + + const durations: number[] = []; + for (let i = 1; i <= iterations; i++) { + const { durationMs, status } = buildOnce( + projectDir, + builder.architectCli, + builderEnv.nodePath, + ); + if (!options.json) { + // eslint-disable-next-line no-console + console.log( + ' run ' + + i + + '/' + + iterations + + ': ' + + (status === 'ok' ? 'done' : 'FAILED') + + ' in ' + + (durationMs / 1000).toFixed(2) + + 's', + ); + } + if (status === 'ok') { + durations.push(durationMs); + } + measurements.push({ + builder: builderName, + layout, + styleFlavor, + entryPoints, + iteration: i, + durationMs, + status, + }); + } + + if (!options.json && durations.length > 0) { + const medianMs = median(durations); + // eslint-disable-next-line no-console + console.log( + ' median: ' + + (medianMs / 1000).toFixed(2) + + 's (' + + (medianMs / entryPoints).toFixed(2) + + 'ms/entry)', + ); + } + + fs.rmSync(projectDir, { recursive: true, force: true }); + } + } + } + } finally { + fs.rmSync(scratchRoot, { recursive: true, force: true }); + } + + const anyFailed = measurements.some((m) => m.status === 'fail'); + if (options.json) { + // eslint-disable-next-line no-console + console.log(formatJsonReport(measurements)); + } + + if (options.csv) { + fs.writeFileSync(options.csv, formatCsvReport(measurements)); + if (!options.json) { + // eslint-disable-next-line no-console + console.log('CSV written to: ' + options.csv); + } + } + + return { measurements, exitCode: anyFailed ? 1 : 0 }; +} diff --git a/scripts/benchmarks/library-builder/reporters.mts b/scripts/benchmarks/library-builder/reporters.mts new file mode 100644 index 000000000000..9c5ba04c2515 --- /dev/null +++ b/scripts/benchmarks/library-builder/reporters.mts @@ -0,0 +1,33 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +// Pure output formatting for the library-builder benchmark's measurements. No console/file I/O +// here -- `index.mts` owns deciding when and where to print or write what this module formats. + +import type { BuildMeasurement } from './harness.mts'; + +export function formatJsonReport(measurements: BuildMeasurement[]): string { + return JSON.stringify(measurements, null, 2); +} + +export function formatCsvReport(measurements: BuildMeasurement[]): string { + const header = 'builder,layout,styleFlavor,entryPoints,iteration,durationMs,status'; + const rows = measurements.map((m) => + [ + m.builder, + m.layout, + m.styleFlavor, + m.entryPoints, + m.iteration, + m.durationMs.toFixed(2), + m.status, + ].join(','), + ); + + return [header, ...rows].join('\n') + '\n'; +}