diff --git a/.github/frameworks.json b/.github/frameworks.json index 77291e23..45078f2d 100644 --- a/.github/frameworks.json +++ b/.github/frameworks.json @@ -27,6 +27,7 @@ "package": "starter-astro", "buildScript": "build", "buildOutputDir": "dist", + "devServerPort": 4321, "measurements": [ { "type": "install", @@ -36,6 +37,10 @@ "type": "build", "runFrequency": 5 }, + { + "type": "devServer", + "runFrequency": 5 + }, { "type": "dependencies" }, @@ -73,6 +78,7 @@ "package": "starter-mastro", "buildScript": "build", "buildOutputDir": "generated", + "devServerPort": 8000, "measurements": [ { "type": "install", @@ -82,6 +88,10 @@ "type": "build", "runFrequency": 5 }, + { + "type": "devServer", + "runFrequency": 5 + }, { "type": "dependencies" }, @@ -116,6 +126,7 @@ "package": "starter-next-js", "buildScript": "build", "buildOutputDir": ".next", + "devServerPort": 3000, "measurements": [ { "type": "install", @@ -125,6 +136,10 @@ "type": "build", "runFrequency": 5 }, + { + "type": "devServer", + "runFrequency": 5 + }, { "type": "dependencies" }, @@ -162,6 +177,7 @@ "package": "starter-nuxt", "buildScript": "build", "buildOutputDir": ".output", + "devServerPort": 3000, "measurements": [ { "type": "install", @@ -171,6 +187,10 @@ "type": "build", "runFrequency": 5 }, + { + "type": "devServer", + "runFrequency": 5 + }, { "type": "dependencies" }, @@ -208,6 +228,7 @@ "package": "starter-react-router", "buildScript": "build", "buildOutputDir": "build", + "devServerPort": 5173, "measurements": [ { "type": "install", @@ -217,6 +238,10 @@ "type": "build", "runFrequency": 5 }, + { + "type": "devServer", + "runFrequency": 5 + }, { "type": "dependencies" }, @@ -254,6 +279,7 @@ "package": "starter-solid-start", "buildScript": "build", "buildOutputDir": ".output", + "devServerPort": 3000, "measurements": [ { "type": "install", @@ -263,6 +289,10 @@ "type": "build", "runFrequency": 5 }, + { + "type": "devServer", + "runFrequency": 5 + }, { "type": "dependencies" }, @@ -300,6 +330,7 @@ "package": "starter-sveltekit", "buildScript": "build", "buildOutputDir": "build", + "devServerPort": 5173, "measurements": [ { "type": "install", @@ -309,6 +340,10 @@ "type": "build", "runFrequency": 5 }, + { + "type": "devServer", + "runFrequency": 5 + }, { "type": "dependencies" }, @@ -346,6 +381,7 @@ "package": "starter-tanstack-start-react", "buildScript": "build", "buildOutputDir": ".output", + "devServerPort": 3000, "measurements": [ { "type": "install", @@ -355,6 +391,10 @@ "type": "build", "runFrequency": 5 }, + { + "type": "devServer", + "runFrequency": 5 + }, { "type": "dependencies" }, diff --git a/.github/workflows/generate-stats.yml b/.github/workflows/generate-stats.yml index 5b6e1667..ef9dcff1 100644 --- a/.github/workflows/generate-stats.yml +++ b/.github/workflows/generate-stats.yml @@ -13,6 +13,7 @@ on: - '!packages/starter-*/stats/**' - '!packages/starter-*/install-stats.json' - '!packages/starter-*/build-stats.json' + - '!packages/starter-*/dev-server-stats.json' - '!packages/starter-*/corejs-stats.json' - '!packages/starter-*/browser-baseline-stats.json' - '!packages/starter-*/e18e-stats.json' @@ -32,6 +33,7 @@ jobs: outputs: install-matrix: ${{ steps.set-matrix.outputs.install }} build-matrix: ${{ steps.set-matrix.outputs.build }} + dev-server-matrix: ${{ steps.set-matrix.outputs.dev-server }} ssr-request-throughput-matrix: ${{ steps.set-matrix.outputs.ssr-request-throughput }} ssr-load-matrix: ${{ steps.set-matrix.outputs.ssr-load }} deps-matrix: ${{ steps.set-matrix.outputs.deps }} @@ -47,6 +49,7 @@ jobs: FRAMEWORKS=$(cat .github/frameworks.json) echo "install=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT echo "build=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "dev-server=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT echo "deps=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT @@ -59,6 +62,7 @@ jobs: with: install-matrix: ${{ needs.setup.outputs.install-matrix }} build-matrix: ${{ needs.setup.outputs.build-matrix }} + dev-server-matrix: ${{ needs.setup.outputs.dev-server-matrix }} ssr-request-throughput-matrix: ${{ needs.setup.outputs.ssr-request-throughput-matrix }} ssr-load-matrix: ${{ needs.setup.outputs.ssr-load-matrix }} deps-matrix: ${{ needs.setup.outputs.deps-matrix }} diff --git a/.github/workflows/measure-framework.yml b/.github/workflows/measure-framework.yml index 60f2e2e4..fb532afc 100644 --- a/.github/workflows/measure-framework.yml +++ b/.github/workflows/measure-framework.yml @@ -14,6 +14,11 @@ on: description: 'JSON array of frameworks to measure build time' type: string required: true + dev-server-matrix: + description: 'JSON array of frameworks to measure dev server startup time' + type: string + required: false + default: '[]' ssr-request-throughput-matrix: description: 'JSON array of frameworks to measure SSR request throughput performance' type: string @@ -135,6 +140,43 @@ jobs: retention-days: 1 if-no-files-found: error + measure-dev-server: + if: inputs.dev-server-matrix != '[]' + runs-on: depot-ubuntu-24.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + framework: ${{ fromJson(inputs.dev-server-matrix) }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + cache: 'pnpm' + + - name: Install workspace dependencies + run: pnpm install --frozen-lockfile + + - name: Run dev server benchmark + run: | + RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "devServer") | .runFrequency') + pnpm --filter @framework-tracker/stats-generator run:dev-server ${{ matrix.framework.package }} $RUN_FREQUENCY + + - name: Upload dev server stats + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dev-server-stats-${{ matrix.framework.name }} + path: packages/${{ matrix.framework.package }}/dev-server-stats.json + retention-days: 1 + if-no-files-found: error + measure-ssr-request-throughput: if: inputs.ssr-request-throughput-matrix != '[]' runs-on: depot-ubuntu-24.04 diff --git a/.github/workflows/preview-stats.yml b/.github/workflows/preview-stats.yml index f585a76f..668d08a3 100644 --- a/.github/workflows/preview-stats.yml +++ b/.github/workflows/preview-stats.yml @@ -12,6 +12,7 @@ jobs: outputs: install-matrix: ${{ steps.detect.outputs.install }} build-matrix: ${{ steps.detect.outputs.build }} + dev-server-matrix: ${{ steps.detect.outputs.dev-server }} ssr-request-throughput-matrix: ${{ steps.detect.outputs.ssr-request-throughput }} deps-matrix: ${{ steps.detect.outputs.deps }} framework-names: ${{ steps.detect.outputs.framework-names }} @@ -48,6 +49,7 @@ jobs: echo "framework-names=[]" >> $GITHUB_OUTPUT echo "install=[]" >> $GITHUB_OUTPUT echo "build=[]" >> $GITHUB_OUTPUT + echo "dev-server=[]" >> $GITHUB_OUTPUT echo "ssr-request-throughput=[]" >> $GITHUB_OUTPUT echo "deps=[]" >> $GITHUB_OUTPUT exit 0 @@ -71,6 +73,7 @@ jobs: echo "install=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT echo "build=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "dev-server=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT echo "deps=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT @@ -81,6 +84,7 @@ jobs: with: install-matrix: ${{ needs.setup.outputs.install-matrix }} build-matrix: ${{ needs.setup.outputs.build-matrix }} + dev-server-matrix: ${{ needs.setup.outputs.dev-server-matrix }} ssr-request-throughput-matrix: ${{ needs.setup.outputs.ssr-request-throughput-matrix }} deps-matrix: ${{ needs.setup.outputs.deps-matrix }} diff --git a/.github/workflows/validate-stats.yml b/.github/workflows/validate-stats.yml index 870fb078..bb3ddb24 100644 --- a/.github/workflows/validate-stats.yml +++ b/.github/workflows/validate-stats.yml @@ -16,6 +16,7 @@ on: - '!packages/starter-*/stats/**' - '!packages/starter-*/install-stats.json' - '!packages/starter-*/build-stats.json' + - '!packages/starter-*/dev-server-stats.json' - '!packages/starter-*/corejs-stats.json' - '!packages/starter-*/browser-baseline-stats.json' - '!packages/starter-*/e18e-stats.json' @@ -82,6 +83,8 @@ jobs: pnpm --filter @framework-tracker/stats-generator run:install $PKG 1 echo "Running build benchmark for $PKG..." pnpm --filter @framework-tracker/stats-generator run:build $PKG + echo "Running dev server benchmark for $PKG (1 run)..." + pnpm --filter @framework-tracker/stats-generator run:dev-server $PKG 1 echo "Running Node engines scan for $PKG..." pnpm --filter @framework-tracker/stats-generator run:node-engines $PKG done diff --git a/.gitignore b/.gitignore index 0cd058d5..edf1aa29 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,7 @@ coverage/ /artifacts packages/*/install-stats.json packages/*/build-stats.json +packages/*/dev-server-stats.json packages/*/corejs-stats.json packages/*/e18e-stats.json packages/*/framework-dependency-stats.json diff --git a/packages/docs/src/components/DevServerStartChart.astro b/packages/docs/src/components/DevServerStartChart.astro new file mode 100644 index 00000000..c0ecd5eb --- /dev/null +++ b/packages/docs/src/components/DevServerStartChart.astro @@ -0,0 +1,23 @@ +--- +import { starterStats } from '../lib/collections' +import ComparisonBarChart from './ComparisonBarChart.astro' + +const validEntries = starterStats.filter( + (f) => f?.name != null && Number.isFinite(f.devServerStartTime?.avgMs), +) + +const data = validEntries + .map((f) => ({ + name: f.name, + value: (f.devServerStartTime?.avgMs ?? 0) / 1000, + focused: f.isFocused, + })) + .sort((a, b) => a.value - b.value || a.name.localeCompare(b.name)) +--- + + diff --git a/packages/docs/src/components/DevServerStartTable.astro b/packages/docs/src/components/DevServerStartTable.astro new file mode 100644 index 00000000..e4b75219 --- /dev/null +++ b/packages/docs/src/components/DevServerStartTable.astro @@ -0,0 +1,24 @@ +--- +import { devServerStartData } from '../lib/collections' +import { getFrameworkSlug } from '../lib/utils' +import StatsTable from './StatsTable.astro' + +const columns = [ + { + key: 'name', + header: 'Framework', + nameCell: true, + href: (row: Record) => + `/framework/${getFrameworkSlug(row.package as string)}`, + }, + { key: 'avgStart', header: 'Avg Start' }, + { key: 'minStart', header: 'Min Start' }, + { key: 'maxStart', header: 'Max Start' }, +] +--- + + diff --git a/packages/docs/src/components/FrameworkDevServerVersionCharts.astro b/packages/docs/src/components/FrameworkDevServerVersionCharts.astro new file mode 100644 index 00000000..d9e693d3 --- /dev/null +++ b/packages/docs/src/components/FrameworkDevServerVersionCharts.astro @@ -0,0 +1,18 @@ +--- +import VersionLineChart from './VersionLineChart.astro' + +interface Props { + versions: unknown[] +} + +const { versions } = Astro.props +--- + + diff --git a/packages/docs/src/content.config.ts b/packages/docs/src/content.config.ts index c9b4c5e0..3fc78185 100644 --- a/packages/docs/src/content.config.ts +++ b/packages/docs/src/content.config.ts @@ -44,6 +44,7 @@ const devtimeSchema = z.object({ installTime: timeSchema, coldBuildTime: timeSchema, warmBuildTime: timeSchema, + devServerStartTime: timeSchema.optional(), buildOutputSize: z.number(), nodeModulesSize: z.number(), duplicateDependencies: z.number().optional(), diff --git a/packages/docs/src/content/docs/all-frameworks.mdx b/packages/docs/src/content/docs/all-frameworks.mdx index 0f9d1e02..5c659755 100644 --- a/packages/docs/src/content/docs/all-frameworks.mdx +++ b/packages/docs/src/content/docs/all-frameworks.mdx @@ -14,6 +14,8 @@ import CoreWebVitalsTable from '../../components/CoreWebVitalsTable.astro' import DependencyCharts from '../../components/DependencyCharts.astro' import DependencyCountText from '../../components/DependencyCountText.astro' import DependencyStatsTable from '../../components/DependencyStatsTable.astro' +import DevServerStartChart from '../../components/DevServerStartChart.astro' +import DevServerStartTable from '../../components/DevServerStartTable.astro' import FrameworkDependencyCharts from '../../components/FrameworkDependencyCharts.astro' import FrameworkDependencyStatsTable from '../../components/FrameworkDependencyStatsTable.astro' import MinimumNodeVersionTable from '../../components/MinimumNodeVersionTable.astro' @@ -57,6 +59,13 @@ Methodology: [Build and Install Times](/methodology/#build-and-install-times). +## Dev Server Startup + +Methodology: [Dev Server Startup](/methodology/#dev-server-startup). + + + + ## Core-JS Polyfills Methodology: [Core-JS Polyfills](/methodology/#core-js-polyfills). diff --git a/packages/docs/src/content/docs/dev-time.mdx b/packages/docs/src/content/docs/dev-time.mdx index cea77b58..e247e2ba 100644 --- a/packages/docs/src/content/docs/dev-time.mdx +++ b/packages/docs/src/content/docs/dev-time.mdx @@ -10,6 +10,8 @@ import CoreJsTable from '../../components/CoreJsTable.astro' import DependencyCharts from '../../components/DependencyCharts.astro' import DependencyCountText from '../../components/DependencyCountText.astro' import DependencyStatsTable from '../../components/DependencyStatsTable.astro' +import DevServerStartChart from '../../components/DevServerStartChart.astro' +import DevServerStartTable from '../../components/DevServerStartTable.astro' import FrameworkDependencyCharts from '../../components/FrameworkDependencyCharts.astro' import FrameworkDependencyStatsTable from '../../components/FrameworkDependencyStatsTable.astro' import MinimumNodeVersionTable from '../../components/MinimumNodeVersionTable.astro' @@ -50,6 +52,13 @@ Methodology: [Build and Install Times](/methodology/#build-and-install-times). +## Dev Server Startup + +Methodology: [Dev Server Startup](/methodology/#dev-server-startup). + + + + ## Core-JS Polyfills Methodology: [Core-JS Polyfills](/methodology/#core-js-polyfills). diff --git a/packages/docs/src/content/docs/framework/astro.mdx b/packages/docs/src/content/docs/framework/astro.mdx index b736d1a3..389252c4 100644 --- a/packages/docs/src/content/docs/framework/astro.mdx +++ b/packages/docs/src/content/docs/framework/astro.mdx @@ -8,6 +8,7 @@ import FrameworkBuildInstallVersionCharts from '../../../components/FrameworkBui import FrameworkClientRenderedVersionCharts from '../../../components/FrameworkClientRenderedVersionCharts.astro' import FrameworkCoreJsVersionChart from '../../../components/FrameworkCoreJsVersionChart.astro' import FrameworkDependencyVersionCharts from '../../../components/FrameworkDependencyVersionCharts.astro' +import FrameworkDevServerVersionCharts from '../../../components/FrameworkDevServerVersionCharts.astro' import FrameworkNodeModulesSizeVersionCharts from '../../../components/FrameworkNodeModulesSizeVersionCharts.astro' import FrameworkServerRenderedVersionCharts from '../../../components/FrameworkServerRenderedVersionCharts.astro' import FrameworkSSRLoadVersionCharts from '../../../components/FrameworkSSRLoadVersionCharts.astro' @@ -44,6 +45,12 @@ export const astroRuntimeVersions = getRuntimeVersionStats('app-astro') +### Dev Server Startup + + + + + ### Core-JS Polyfills diff --git a/packages/docs/src/content/docs/framework/solid-start.mdx b/packages/docs/src/content/docs/framework/solid-start.mdx index be936760..4bf05751 100644 --- a/packages/docs/src/content/docs/framework/solid-start.mdx +++ b/packages/docs/src/content/docs/framework/solid-start.mdx @@ -8,6 +8,7 @@ import FrameworkBuildInstallVersionCharts from '../../../components/FrameworkBui import FrameworkClientRenderedVersionCharts from '../../../components/FrameworkClientRenderedVersionCharts.astro' import FrameworkCoreJsVersionChart from '../../../components/FrameworkCoreJsVersionChart.astro' import FrameworkDependencyVersionCharts from '../../../components/FrameworkDependencyVersionCharts.astro' +import FrameworkDevServerVersionCharts from '../../../components/FrameworkDevServerVersionCharts.astro' import FrameworkNodeModulesSizeVersionCharts from '../../../components/FrameworkNodeModulesSizeVersionCharts.astro' import FrameworkServerRenderedVersionCharts from '../../../components/FrameworkServerRenderedVersionCharts.astro' import FrameworkSSRLoadVersionCharts from '../../../components/FrameworkSSRLoadVersionCharts.astro' @@ -47,6 +48,12 @@ export const solidStartRuntimeVersions = +### Dev Server Startup + + + + + ### Core-JS Polyfills diff --git a/packages/docs/src/content/docs/framework/sveltekit.mdx b/packages/docs/src/content/docs/framework/sveltekit.mdx index 8d2c486a..6d41f763 100644 --- a/packages/docs/src/content/docs/framework/sveltekit.mdx +++ b/packages/docs/src/content/docs/framework/sveltekit.mdx @@ -8,6 +8,7 @@ import FrameworkBuildInstallVersionCharts from '../../../components/FrameworkBui import FrameworkClientRenderedVersionCharts from '../../../components/FrameworkClientRenderedVersionCharts.astro' import FrameworkCoreJsVersionChart from '../../../components/FrameworkCoreJsVersionChart.astro' import FrameworkDependencyVersionCharts from '../../../components/FrameworkDependencyVersionCharts.astro' +import FrameworkDevServerVersionCharts from '../../../components/FrameworkDevServerVersionCharts.astro' import FrameworkNodeModulesSizeVersionCharts from '../../../components/FrameworkNodeModulesSizeVersionCharts.astro' import FrameworkServerRenderedVersionCharts from '../../../components/FrameworkServerRenderedVersionCharts.astro' import FrameworkSSRLoadVersionCharts from '../../../components/FrameworkSSRLoadVersionCharts.astro' @@ -45,6 +46,12 @@ export const sveltekitRuntimeVersions = getRuntimeVersionStats('app-sveltekit') +### Dev Server Startup + + + + + ### Core-JS Polyfills diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index cd17bce9..23b48c43 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -105,6 +105,31 @@ matches the framework version tracked by the starter project. from the measurement; excluding `.next/cache` keeps the comparison consistent. +### Dev Server Startup + +- Dev server startup time measures how long `pnpm dev` takes in a fresh + temporary copy of the tracked starter files until the first + `GET http://localhost:/` returns HTTP 200. Dependencies are installed + outside the timed region with a frozen lockfile and a dedicated store shared + by the repetitions, the same setup as the build benchmark. +- The clock starts when the `pnpm dev` process is spawned and stops when the + response headers of the first 200 arrive, so pnpm's own startup, framework + boot, and the on-demand compile of the home page are all included. The + response body is not read, because several dev servers stream the page and + body time would measure page size rather than startup. +- The port is the starter's default dev port, recorded as `devServerPort` in + `.github/frameworks.json`. The starters are not modified, so whatever they + ship is inside the measurement: Nuxt and TanStack Start enable devtools, + and Astro and React Router generate types at boot. Mastro's `dev` script is + a plain `node --watch` server rather than a framework dev server. +- The dev server runs with a minimal environment (`PATH`, `HOME`, `TMPDIR`, + `LANG`, `LC_ALL`) plus telemetry opt-outs, so the terminal that launches the + benchmark cannot change how the framework behaves. Astro, for example, + switches to a detached background server when it detects an AI agent + terminal. +- Dev server benchmarks run 5 times by default and report average, minimum, + and maximum duration. + ### Core-JS Polyfills - The scanner searches JavaScript build output files for vendored diff --git a/packages/docs/src/lib/collections.ts b/packages/docs/src/lib/collections.ts index d3d8a0c1..c43f8d6c 100644 --- a/packages/docs/src/lib/collections.ts +++ b/packages/docs/src/lib/collections.ts @@ -289,6 +289,15 @@ export const buildInstallData = starterStats.map((f) => ({ buildOutput: formatBytesToMB(f.buildOutputSize), })) +export const devServerStartData = starterStats.map((f) => ({ + name: f.name, + package: f.package, + isFocused: f.isFocused, + avgStart: formatTimeMs(f.devServerStartTime?.avgMs ?? Number.NaN), + minStart: formatTimeMs(f.devServerStartTime?.minMs ?? Number.NaN), + maxStart: formatTimeMs(f.devServerStartTime?.maxMs ?? Number.NaN), +})) + export const chartDuplicateDependencyData = starterStats .filter((f) => Number.isFinite(f.duplicateDependencies)) .map((f) => ({ diff --git a/packages/stats-generator/package.json b/packages/stats-generator/package.json index 1aadc99f..8ffab9f1 100644 --- a/packages/stats-generator/package.json +++ b/packages/stats-generator/package.json @@ -13,6 +13,7 @@ "run:server-side-rendered": "node src/run-server-side-rendered-benchmark.ts", "run:install": "node src/run-install-benchmark.ts", "run:build": "node src/run-build-benchmark.ts", + "run:dev-server": "node src/run-dev-server-benchmark.ts", "run:corejs": "node src/run-corejs-scan.ts", "run:browser-baseline": "node src/run-browser-baseline-scan.ts", "run:framework-dependencies": "node src/run-framework-dependency-scan.ts", diff --git a/packages/stats-generator/src/benchmark-utils.ts b/packages/stats-generator/src/benchmark-utils.ts index 2572a8e0..0ffaf87d 100644 --- a/packages/stats-generator/src/benchmark-utils.ts +++ b/packages/stats-generator/src/benchmark-utils.ts @@ -1,4 +1,7 @@ import { execFileSync } from 'node:child_process' +import { cpSync, mkdirSync } from 'node:fs' +import { dirname, join, relative } from 'node:path' +import { packagesDir } from './constants.ts' export function parseRunFrequency( value: string | undefined, @@ -35,3 +38,28 @@ export function installDependencies( }, ) } + +export function copyTrackedProject( + sourceDir: string, + projectDir: string, +): void { + const repositoryDir = join(packagesDir, '..') + const sourcePathFromRepository = relative(repositoryDir, sourceDir) + const trackedPaths = execFileSync( + 'git', + ['ls-files', '-z', '--', sourcePathFromRepository], + { + cwd: repositoryDir, + encoding: 'utf-8', + }, + ) + .split('\0') + .filter(Boolean) + + for (const trackedPath of trackedPaths) { + const projectPath = relative(sourcePathFromRepository, trackedPath) + const destinationPath = join(projectDir, projectPath) + mkdirSync(dirname(destinationPath), { recursive: true }) + cpSync(join(repositoryDir, trackedPath), destinationPath) + } +} diff --git a/packages/stats-generator/src/dev-server.test.ts b/packages/stats-generator/src/dev-server.test.ts new file mode 100644 index 00000000..132b81eb --- /dev/null +++ b/packages/stats-generator/src/dev-server.test.ts @@ -0,0 +1,238 @@ +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import http from 'node:http' +import net from 'node:net' +import test from 'node:test' +import { setTimeout as sleep } from 'node:timers/promises' +import { + advertisedPorts, + devServerEnv, + isPortListening, + killProcessTree, + probeOnce, + waitForHttpOk, + waitForPortFree, +} from './dev-server.ts' + +async function listen( + handler: http.RequestListener, + host: string, +): Promise<{ server: http.Server; port: number }> { + const server = http.createServer(handler) + server.listen(0, host) + await once(server, 'listening') + const address = server.address() as net.AddressInfo + return { server, port: address.port } +} + +async function close(server: http.Server): Promise { + server.closeAllConnections() + server.close() + await once(server, 'close') +} + +async function waitForExit(pid: number, timeoutMs: number): Promise { + const deadline = performance.now() + timeoutMs + while (performance.now() < deadline) { + try { + process.kill(pid, 0) + } catch { + return + } + await sleep(20) + } + throw new Error(`pid ${pid} still alive after ${timeoutMs}ms`) +} + +async function hasIpv6Loopback(): Promise { + const server = net.createServer() + try { + server.listen(0, '::1') + await once(server, 'listening') + return true + } catch { + return false + } finally { + server.close() + } +} + +async function freePort(): Promise { + const server = net.createServer() + server.listen(0, '127.0.0.1') + await once(server, 'listening') + const { port } = server.address() as net.AddressInfo + server.close() + await once(server, 'close') + return port +} + +test('probeOnce reports the status and preserves the localhost Host header', async (t) => { + let hostHeader = '' + const { server, port } = await listen((req, res) => { + hostHeader = req.headers.host ?? '' + res.writeHead(503).end() + }, '127.0.0.1') + t.after(() => close(server)) + + assert.deepEqual(await probeOnce('127.0.0.1', port, 1000), { status: 503 }) + assert.equal(hostHeader, `localhost:${port}`) +}) + +test('probeOnce reports a refused connection without throwing', async () => { + const port = await freePort() + const result = await probeOnce('127.0.0.1', port, 1000) + assert.equal(result.status, null) + assert.equal(result.errorCode, 'ECONNREFUSED') +}) + +test('waitForHttpOk measures the delay until the first 200 within one poll interval', async (t) => { + const readyAfterMs = 300 + const startedAt = performance.now() + const { server, port } = await listen((_req, res) => { + const ready = performance.now() - startedAt >= readyAfterMs + res.writeHead(ready ? 200 : 503).end() + }, '127.0.0.1') + t.after(() => close(server)) + + const result = await waitForHttpOk(startedAt, { + port, + addresses: ['127.0.0.1'], + pollIntervalMs: 25, + }) + assert.ok( + result.elapsedMs >= readyAfterMs, + `${result.elapsedMs} >= ${readyAfterMs}`, + ) + assert.ok( + result.elapsedMs < readyAfterMs + 25 + 50, + `${result.elapsedMs} overshoot`, + ) + assert.equal(result.address, '127.0.0.1') + assert.deepEqual(result.observedStatuses, [503]) +}) + +test('waitForHttpOk does not treat a non-200 response as ready', async (t) => { + const { server, port } = await listen((_req, res) => { + res.writeHead(503).end() + }, '127.0.0.1') + t.after(() => close(server)) + + await assert.rejects( + waitForHttpOk(performance.now(), { + port, + addresses: ['127.0.0.1'], + timeoutMs: 200, + }), + /within 200ms \(saw statuses 503\)/, + ) +}) + +test('waitForHttpOk finds a server bound to a single loopback family', async (t) => { + const v4 = await listen((_req, res) => res.writeHead(200).end(), '127.0.0.1') + t.after(() => close(v4.server)) + const found4 = await waitForHttpOk(performance.now(), { port: v4.port }) + assert.equal(found4.address, '127.0.0.1') + + if (!(await hasIpv6Loopback())) { + t.diagnostic('::1 unavailable, skipping the IPv6 half') + return + } + const v6 = await listen((_req, res) => res.writeHead(200).end(), '::1') + t.after(() => close(v6.server)) + const found6 = await waitForHttpOk(performance.now(), { port: v6.port }) + assert.equal(found6.address, '::1') +}) + +test('waitForHttpOk stops promptly when aborted', async () => { + const port = await freePort() + const abort = new AbortController() + const waiting = waitForHttpOk(performance.now(), { + port, + signal: abort.signal, + }) + await sleep(60) + abort.abort() + const startedAbort = performance.now() + await assert.rejects(waiting, /Stopped waiting/) + assert.ok(performance.now() - startedAbort < 200) +}) + +test('isPortListening and waitForPortFree track a listener lifecycle', async () => { + const server = net.createServer() + server.listen(0, '127.0.0.1') + await once(server, 'listening') + const { port } = server.address() as net.AddressInfo + + assert.equal(await isPortListening(port), true) + setTimeout(() => server.close(), 100) + await waitForPortFree(port, 2000) + assert.equal(await isPortListening(port), false) +}) + +test('devServerEnv keeps only the allowlisted variables plus opt-outs', () => { + const env = devServerEnv({ + PATH: '/usr/bin', + HOME: '/home/ci', + CLAUDECODE: '1', + TERM_PROGRAM: 'vscode', + }) + assert.equal(env.PATH, '/usr/bin') + assert.equal(env.HOME, '/home/ci') + assert.equal(env.CLAUDECODE, undefined) + assert.equal(env.TERM_PROGRAM, undefined) + assert.equal(env.ASTRO_TELEMETRY_DISABLED, '1') + assert.equal(env.CI, '1') +}) + +test('advertisedPorts extracts ports from server banners', () => { + const output = [ + ' āžœ Local: http://localhost:5174/', + ' āžœ Network: use --host to expose', + '- Local: http://localhost:3000', + 'Server running at http://localhost:8000', + 'ready in 512 ms', + ].join('\n') + assert.deepEqual(advertisedPorts(output), [5174, 3000, 8000]) +}) + +test('killProcessTree kills a SIGTERM-ignoring grandchild and frees its port', async () => { + const grandchildScript = ` + process.on('SIGTERM', () => {}); + const net = require('node:net'); + const server = net.createServer(); + server.listen(0, '127.0.0.1', () => { + process.stdout.write(JSON.stringify({ pid: process.pid, port: server.address().port }) + '\\n'); + }); + ` + const childScript = ` + const { spawn } = require('node:child_process'); + const grandchild = spawn(process.execPath, ['-e', ${JSON.stringify(grandchildScript)}], { stdio: ['ignore', 'inherit', 'inherit'] }); + setInterval(() => {}, 1000); + ` + const child = spawn(process.execPath, ['-e', childScript], { + detached: true, + stdio: ['ignore', 'pipe', 'inherit'], + }) + + let line = '' + for await (const chunk of child.stdout) { + line += chunk.toString() + if (line.includes('\n')) break + } + const grandchild = JSON.parse(line.trim()) as { pid: number; port: number } + assert.equal(await isPortListening(grandchild.port), true) + + await killProcessTree(child, 300) + + assert.notEqual(child.exitCode ?? child.signalCode, null) + await waitForExit(grandchild.pid, 2000) + await waitForPortFree(grandchild.port, 2000) +}) + +test('killProcessTree is a no-op for a child that already exited', async () => { + const child = spawn(process.execPath, ['-e', ''], { detached: true }) + await once(child, 'exit') + await killProcessTree(child, 100) +}) diff --git a/packages/stats-generator/src/dev-server.ts b/packages/stats-generator/src/dev-server.ts new file mode 100644 index 00000000..7f3d4b5b --- /dev/null +++ b/packages/stats-generator/src/dev-server.ts @@ -0,0 +1,286 @@ +import { execFileSync, spawn, type ChildProcess } from 'node:child_process' +import { once } from 'node:events' +import http from 'node:http' +import net from 'node:net' +import { setTimeout as sleep } from 'node:timers/promises' + +export const LOOPBACK_ADDRESSES = ['127.0.0.1', '::1'] + +const INHERITED_ENV = ['PATH', 'HOME', 'TMPDIR', 'LANG', 'LC_ALL'] + +// Astro switches to a detached "background" dev server when it detects an AI +// agent terminal through env vars such as CLAUDECODE, so the caller's +// environment must not leak into the measured process. +export function devServerEnv(source = process.env): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + ASTRO_TELEMETRY_DISABLED: '1', + NEXT_TELEMETRY_DISABLED: '1', + NUXT_TELEMETRY_DISABLED: '1', + DO_NOT_TRACK: '1', + CI: '1', + NO_COLOR: '1', + BROWSER: 'none', + } + for (const name of INHERITED_ENV) { + if (source[name] !== undefined) env[name] = source[name] + } + return env +} + +export interface ProbeResult { + status: number | null + errorCode?: string +} + +function ipFamily(address: string): 4 | 6 { + return address.includes(':') ? 6 : 4 +} + +// Mastro serves its dev routes only when the request hostname is `localhost`, +// and fetch() refuses to send a custom Host header, so this uses node:http. +export function probeOnce( + address: string, + port: number, + timeoutMs: number, +): Promise { + return new Promise((resolve) => { + const request = http.request( + { + host: address, + port, + path: '/', + method: 'GET', + family: ipFamily(address), + agent: false, + setHost: false, + headers: { host: `localhost:${port}`, connection: 'close' }, + timeout: timeoutMs, + }, + (response) => { + resolve({ status: response.statusCode ?? null }) + response.resume() + }, + ) + request.on('timeout', () => request.destroy(new Error('ETIMEDOUT'))) + request.on('error', (error: NodeJS.ErrnoException) => + resolve({ status: null, errorCode: error.code ?? error.message }), + ) + request.end() + }) +} + +export interface WaitForHttpOkOptions { + port: number + addresses?: string[] + pollIntervalMs?: number + requestTimeoutMs?: number + timeoutMs?: number + signal?: AbortSignal +} + +export interface HttpOkResult { + elapsedMs: number + address: string + observedStatuses: number[] +} + +export async function waitForHttpOk( + startedAt: number, + options: WaitForHttpOkOptions, +): Promise { + const { + port, + addresses = LOOPBACK_ADDRESSES, + pollIntervalMs = 25, + requestTimeoutMs = 30_000, + timeoutMs = 120_000, + signal, + } = options + const deadline = startedAt + timeoutMs + const observedStatuses = new Set() + + while (!signal?.aborted) { + const remainingMs = deadline - performance.now() + if (remainingMs <= 0) break + + const attempts = addresses.map(async (address) => { + const result = await probeOnce( + address, + port, + Math.min(requestTimeoutMs, remainingMs), + ) + if (result.status === 200) { + return { address, elapsedMs: performance.now() - startedAt } + } + if (result.status !== null) observedStatuses.add(result.status) + throw result + }) + + try { + const { address, elapsedMs } = await Promise.any(attempts) + return { elapsedMs, address, observedStatuses: [...observedStatuses] } + } catch {} + + try { + await sleep(pollIntervalMs, undefined, { signal }) + } catch { + break + } + } + + const statuses = [...observedStatuses] + const seen = + statuses.length > 0 ? ` (saw statuses ${statuses.join(', ')})` : '' + if (signal?.aborted) { + throw new Error( + `Stopped waiting for http://localhost:${port}/ before it returned 200${seen}`, + ) + } + throw new Error( + `No HTTP 200 from http://localhost:${port}/ within ${timeoutMs}ms${seen}`, + ) +} + +function canConnect(address: string, port: number): Promise { + return new Promise((resolve) => { + const socket = net.connect({ + host: address, + port, + family: ipFamily(address), + }) + socket.once('connect', () => { + socket.destroy() + resolve(true) + }) + socket.once('error', () => resolve(false)) + }) +} + +export async function isPortListening( + port: number, + addresses = LOOPBACK_ADDRESSES, +): Promise { + const results = await Promise.all( + addresses.map((address) => canConnect(address, port)), + ) + return results.some(Boolean) +} + +export async function waitForPortFree( + port: number, + timeoutMs: number, +): Promise { + const deadline = performance.now() + timeoutMs + while (await isPortListening(port)) { + if (performance.now() > deadline) { + throw new Error( + `Port ${port} is still in use ${timeoutMs}ms after stopping the dev server`, + ) + } + await sleep(50) + } +} + +export function advertisedPorts(output: string): number[] { + const ports = new Set() + for (const match of output.matchAll(/https?:\/\/[^\s/]+:(\d{2,5})/g)) { + ports.add(Number(match[1])) + } + return [...ports] +} + +function descendantsOf(rootPid: number): number[] { + const children = new Map() + const table = execFileSync('ps', ['-axo', 'pid=,ppid='], { + encoding: 'utf-8', + }) + for (const row of table.trim().split('\n')) { + const [pid, ppid] = row.trim().split(/\s+/).map(Number) + if (pid === undefined || ppid === undefined) continue + const siblings = children.get(ppid) ?? [] + siblings.push(pid) + children.set(ppid, siblings) + } + + const descendants: number[] = [] + const stack = [rootPid] + for (let parent = stack.pop(); parent !== undefined; parent = stack.pop()) { + for (const child of children.get(parent) ?? []) { + descendants.push(child) + stack.push(child) + } + } + return descendants +} + +// EPERM: the pid was recycled by a process we do not own between the ps +// snapshot and the signal. +function signalPid(pid: number, signal: NodeJS.Signals): void { + try { + process.kill(pid, signal) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code !== 'ESRCH' && code !== 'EPERM') throw error + } +} + +// Astro re-spawns its dev server detached when it detects an agent terminal, +// so the process group alone is not enough; `node --watch` (Mastro) restarts +// children, so the descendant list alone is not enough either. +export async function killProcessTree( + child: ChildProcess, + graceMs = 5_000, +): Promise { + const pid = child.pid + if (pid === undefined || child.exitCode !== null || child.signalCode !== null) + return + + const targets = [-pid, ...descendantsOf(pid)] + const signalAll = (signal: NodeJS.Signals) => { + for (const target of targets) signalPid(target, signal) + } + + const exited = once(child, 'exit') + signalAll('SIGTERM') + const escalation = setTimeout(() => signalAll('SIGKILL'), graceMs) + escalation.unref() + await exited + clearTimeout(escalation) + signalAll('SIGKILL') +} + +export interface DevServerExit { + code: number | null + signal: NodeJS.Signals | null +} + +export interface DevServerHandle { + child: ChildProcess + startedAt: number + exited: Promise + stop: () => Promise +} + +export function spawnDevServer( + projectDir: string, + onOutput: (text: string) => void, +): DevServerHandle { + const startedAt = performance.now() + const child = spawn('pnpm', ['dev'], { + cwd: projectDir, + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + env: devServerEnv(), + }) + + const forward = (chunk: Buffer) => onOutput(chunk.toString()) + child.stdout?.on('data', forward) + child.stderr?.on('data', forward) + + const exited = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', (code, signal) => resolve({ code, signal })) + }) + + return { child, startedAt, exited, stop: () => killProcessTree(child) } +} diff --git a/packages/stats-generator/src/run-build-benchmark.ts b/packages/stats-generator/src/run-build-benchmark.ts index d874584c..85705139 100644 --- a/packages/stats-generator/src/run-build-benchmark.ts +++ b/packages/stats-generator/src/run-build-benchmark.ts @@ -1,8 +1,12 @@ import { execFileSync } from 'node:child_process' -import { cpSync, mkdirSync, rmSync } from 'node:fs' +import { cpSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' -import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' -import { installDependencies, parseRunFrequency } from './benchmark-utils.ts' +import { isAbsolute, join, relative, resolve, sep } from 'node:path' +import { + copyTrackedProject, + installDependencies, + parseRunFrequency, +} from './benchmark-utils.ts' import { packagesDir } from './constants.ts' import { getDirectorySize, @@ -13,28 +17,6 @@ import { import type { BuildStats } from './types.ts' import { summarizeSamples } from './sample-statistics.ts' -function copyTrackedProject(sourceDir: string, projectDir: string): void { - const repositoryDir = join(packagesDir, '..') - const sourcePathFromRepository = relative(repositoryDir, sourceDir) - const trackedPaths = execFileSync( - 'git', - ['ls-files', '-z', '--', sourcePathFromRepository], - { - cwd: repositoryDir, - encoding: 'utf-8', - }, - ) - .split('\0') - .filter(Boolean) - - for (const trackedPath of trackedPaths) { - const projectPath = relative(sourcePathFromRepository, trackedPath) - const destinationPath = join(projectDir, projectPath) - mkdirSync(dirname(destinationPath), { recursive: true }) - cpSync(join(repositoryDir, trackedPath), destinationPath) - } -} - function measureBuildTime(cwd: string, buildScript: string): number { const start = performance.now() execFileSync('pnpm', [buildScript], { diff --git a/packages/stats-generator/src/run-dev-server-benchmark.ts b/packages/stats-generator/src/run-dev-server-benchmark.ts new file mode 100644 index 00000000..dba58863 --- /dev/null +++ b/packages/stats-generator/src/run-dev-server-benchmark.ts @@ -0,0 +1,166 @@ +import { rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + copyTrackedProject, + installDependencies, + parseRunFrequency, +} from './benchmark-utils.ts' +import { packagesDir } from './constants.ts' +import { + advertisedPorts, + isPortListening, + spawnDevServer, + waitForHttpOk, + waitForPortFree, + type DevServerHandle, +} from './dev-server.ts' +import { summarizeSamples } from './sample-statistics.ts' +import type { DevServerStats } from './types.ts' +import { getFrameworkByPackage, parseArgs, writeJsonFile } from './utils.ts' + +const READY_TIMEOUT_MS = 120_000 +const PORT_FREE_TIMEOUT_MS = 10_000 +const OUTPUT_TAIL_LINES = 50 + +let activeServer: DevServerHandle | null = null + +for (const signal of ['SIGINT', 'SIGTERM'] as const) { + process.once(signal, async () => { + await activeServer?.stop() + process.exit(130) + }) +} + +async function measureStartTime( + packageName: string, + projectDir: string, + port: number, +): Promise { + if (await isPortListening(port)) { + throw new Error( + `Port ${port} is already in use; the dev server would start elsewhere and the measurement would be wrong`, + ) + } + + const outputTail: string[] = [] + const server = spawnDevServer(projectDir, (text) => { + process.stdout.write(`[${packageName}] ${text}`) + outputTail.push(...text.split('\n').filter((line) => line.trim())) + outputTail.splice(0, Math.max(0, outputTail.length - OUTPUT_TAIL_LINES)) + }) + activeServer = server + + const abort = new AbortController() + const exitedEarly = server.exited.then(({ code, signal }) => { + abort.abort() + throw new Error( + `Dev server exited before returning 200 (code=${code}, signal=${signal})`, + ) + }) + exitedEarly.catch(() => {}) + + try { + const { elapsedMs, address } = await Promise.race([ + waitForHttpOk(server.startedAt, { + port, + timeoutMs: READY_TIMEOUT_MS, + signal: abort.signal, + }), + exitedEarly, + ]) + console.info( + ` First 200 from ${address}:${port} after ${Math.round(elapsedMs)}ms`, + ) + return Math.round(elapsedMs) + } catch (error) { + const seen = advertisedPorts(outputTail.join('\n')).filter( + (p) => p !== port, + ) + const hint = + seen.length > 0 + ? `\nThe server advertised port ${seen.join(', ')} but devServerPort is ${port}.` + : '' + throw new Error( + `${(error as Error).message}${hint}\nLast output:\n${outputTail.join('\n')}`, + ) + } finally { + abort.abort() + await server.stop() + activeServer = null + await waitForPortFree(port, PORT_FREE_TIMEOUT_MS) + } +} + +async function main() { + const { packageName, args } = parseArgs( + 'Usage: run-dev-server-benchmark [run-frequency]\nExample: run-dev-server-benchmark starter-astro 5', + ) + + const runFrequency = parseRunFrequency(args[0]) + + const { framework, testConfig } = await getFrameworkByPackage(packageName) + const port = testConfig.devServerPort + if (port === undefined) { + throw new Error( + `${packageName} has no devServerPort in .github/frameworks.json`, + ) + } + + console.info( + `Running dev server benchmark for ${framework.displayName} (${packageName})...\n`, + ) + + const packageDir = join(packagesDir, packageName) + const tempDir = join( + tmpdir(), + `framework-dev-server-benchmark-${packageName}-${Date.now()}`, + ) + const storeDir = join(tempDir, 'store') + const cacheDir = join(tempDir, 'cache') + + const startTimesMs: number[] = [] + let previousRunDir = '' + + try { + for (let i = 1; i <= runFrequency; i++) { + if (previousRunDir) { + rmSync(previousRunDir, { recursive: true, force: true }) + } + + const runDir = join(tempDir, `run-${i}`) + const projectDir = join(runDir, 'project') + copyTrackedProject(packageDir, projectDir) + + console.info(`\nDev server run ${i}/${runFrequency}...`) + console.info('Installing dependencies outside the timed region...') + installDependencies(projectDir, storeDir, cacheDir) + + console.info('Starting dev server...') + startTimesMs.push(await measureStartTime(packageName, projectDir, port)) + + previousRunDir = runDir + } + + const devServerStartTime = summarizeSamples(startTimesMs) + console.info(`\nAvg dev server start time: ${devServerStartTime.avgMs} ms`) + console.info( + `\nDev server start standard deviation: ${devServerStartTime.standardDeviationMs} ms`, + ) + console.info(`\nMin dev server start time: ${devServerStartTime.minMs} ms`) + console.info(`\nMax dev server start time: ${devServerStartTime.maxMs} ms`) + + const stats: DevServerStats = { devServerStartTime } + const outputPath = join(packagesDir, packageName, 'dev-server-stats.json') + writeJsonFile(outputPath, stats) + + console.info(`\nāœ“ Saved dev server stats to ${outputPath}`) + } finally { + rmSync(tempDir, { recursive: true, force: true }) + } +} + +main().catch((error) => { + console.error('Dev server benchmark failed:', error) + process.exit(1) +}) diff --git a/packages/stats-generator/src/save-ci-stats.ts b/packages/stats-generator/src/save-ci-stats.ts index 8ba798d6..1d8eb01c 100644 --- a/packages/stats-generator/src/save-ci-stats.ts +++ b/packages/stats-generator/src/save-ci-stats.ts @@ -12,6 +12,7 @@ import type { CIStats, InstallStats, BuildStats, + DevServerStats, CoreJsStats, BrowserBaselineStats, NodeEnginesStats, @@ -91,6 +92,25 @@ async function main() { console.warn(`No build stats artifact found at ${buildStatsPath}`) } + const devServerStatsPath = join( + artifactsDir, + `dev-server-stats-${name}`, + 'dev-server-stats.json', + ) + const devServerStats = readJsonFile(devServerStatsPath) + + if (devServerStats) { + console.info(` āœ“ Found dev server stats artifact`) + stats = { + ...stats, + devServerStartTime: devServerStats.devServerStartTime, + } + } else { + console.warn( + `No dev server stats artifact found at ${devServerStatsPath}`, + ) + } + // Load core-js stats from artifact const coreJsArtifactPath = join( artifactsDir, diff --git a/packages/stats-generator/src/schemas.ts b/packages/stats-generator/src/schemas.ts index 07290b2f..50ffc1de 100644 --- a/packages/stats-generator/src/schemas.ts +++ b/packages/stats-generator/src/schemas.ts @@ -27,6 +27,10 @@ export const BuildStatsSchema = z.object({ buildOutputSize: z.number().nonnegative(), }) +export const DevServerStatsSchema = z.object({ + devServerStartTime: TimeStatSchema, +}) + export const BrowserBaselineStatsSchema = z.object({ baselineStatus: z.union([ z.literal('high'), diff --git a/packages/stats-generator/src/types.ts b/packages/stats-generator/src/types.ts index 489b1ebb..f407fd99 100644 --- a/packages/stats-generator/src/types.ts +++ b/packages/stats-generator/src/types.ts @@ -1,6 +1,7 @@ export type MeasurementType = | 'install' | 'build' + | 'devServer' | 'test' | 'dependencies' | 'browserBaseline' @@ -18,6 +19,7 @@ export interface TestConfig { package: string buildScript: string buildOutputDir: string + devServerPort?: number measurements: MeasurementConfig[] } @@ -65,6 +67,7 @@ export interface CIStats { coldBuildTime?: TimeStat warmBuildTime?: TimeStat buildOutputSize?: number + devServerStartTime?: TimeStat testTimeMs?: number // Direct SSR request throughput stats ssrRequestThroughputTests?: { @@ -167,6 +170,10 @@ export interface BuildStats { buildOutputSize: number } +export interface DevServerStats { + devServerStartTime: TimeStat +} + export interface CoreJsStats { vendoredFiles: Array<{ file: string diff --git a/packages/stats-generator/src/validate-stats.ts b/packages/stats-generator/src/validate-stats.ts index 5895c8d2..7c073cc8 100644 --- a/packages/stats-generator/src/validate-stats.ts +++ b/packages/stats-generator/src/validate-stats.ts @@ -6,6 +6,7 @@ import { readJsonFile } from './utils.ts' import { InstallStatsSchema, BuildStatsSchema, + DevServerStatsSchema, BrowserBaselineStatsSchema, NodeEnginesStatsSchema, SSRRequestThroughputStatsSchema, @@ -17,6 +18,7 @@ import { type BenchmarkType = | 'install' | 'build' + | 'devServer' | 'browserBaseline' | 'nodeEngines' | 'ssrRequestThroughput' @@ -33,6 +35,11 @@ interface BenchmarkConfig { const STARTER_BENCHMARKS: BenchmarkConfig[] = [ { type: 'install', file: 'install-stats.json', schema: InstallStatsSchema }, { type: 'build', file: 'build-stats.json', schema: BuildStatsSchema }, + { + type: 'devServer', + file: 'dev-server-stats.json', + schema: DevServerStatsSchema, + }, { type: 'browserBaseline', file: 'browser-baseline-stats.json',