From ef3fade4b2af4092316b7fb87a643c4a0ed3e239 Mon Sep 17 00:00:00 2001 From: Alexander Karan <47707063+AlexanderKaran@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:19:49 +0800 Subject: [PATCH] Step One Shared Config --- .github/frameworks.json | 444 ------------------ .github/workflows/generate-stats.yml | 29 +- .github/workflows/measure-framework.yml | 9 +- .github/workflows/preview-stats.yml | 21 +- .github/workflows/sync-version.yml | 2 +- .github/workflows/validate-stats.yml | 42 +- packages/docs/src/content/docs/methodology.md | 2 +- packages/stats-generator/src/collect-stats.ts | 6 +- packages/stats-generator/src/frameworks.ts | 261 ++++++++++ .../stats-generator/src/get-frameworks.ts | 8 +- packages/stats-generator/src/measurements.ts | 16 + .../stats-generator/src/print-frameworks.ts | 3 + .../src/run-build-benchmark.ts | 3 +- .../src/run-client-side-rendered-benchmark.ts | 4 +- .../src/run-dev-server-benchmark.ts | 5 +- .../src/run-install-benchmark.ts | 3 +- .../src/run-server-side-rendered-benchmark.ts | 4 +- .../run-ssr-request-throughput-benchmark.ts | 6 +- packages/stats-generator/src/schemas.ts | 4 +- .../src/ssrLoad/run-load-test.ts | 11 +- packages/stats-generator/src/types.ts | 20 +- .../stats-generator/src/validate-stats.ts | 12 +- 22 files changed, 370 insertions(+), 545 deletions(-) delete mode 100644 .github/frameworks.json create mode 100644 packages/stats-generator/src/frameworks.ts create mode 100644 packages/stats-generator/src/measurements.ts create mode 100644 packages/stats-generator/src/print-frameworks.ts diff --git a/.github/frameworks.json b/.github/frameworks.json deleted file mode 100644 index b218551b4..000000000 --- a/.github/frameworks.json +++ /dev/null @@ -1,444 +0,0 @@ -[ - { - "name": "baseline-html", - "displayName": "Baseline HTML", - "frameworkPackage": "node", - "focusedFramework": false, - "app": { - "package": "app-baseline-html", - "buildScript": "build", - "buildOutputDir": ".", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "ssrLoad" - } - ] - } - }, - { - "name": "astro", - "displayName": "Astro", - "frameworkPackage": "astro", - "focusedFramework": true, - "starter": { - "package": "starter-astro", - "buildScript": "build", - "buildOutputDir": "dist", - "devServerPort": 4321, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-astro", - "buildScript": "build", - "buildOutputDir": "dist", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - } - ] - } - }, - { - "name": "next", - "displayName": "Next.js", - "frameworkPackage": "next", - "focusedFramework": true, - "starter": { - "package": "starter-next-js", - "buildScript": "build", - "buildOutputDir": ".next", - "devServerPort": 3000, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-next-js", - "buildScript": "build", - "buildOutputDir": ".next", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - }, - { - "type": "ssrRouterLinkLoad" - } - ] - } - }, - { - "name": "nuxt", - "displayName": "Nuxt", - "frameworkPackage": "nuxt", - "focusedFramework": true, - "starter": { - "package": "starter-nuxt", - "buildScript": "build", - "buildOutputDir": ".output", - "devServerPort": 3000, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-nuxt", - "buildScript": "build", - "buildOutputDir": ".output", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - }, - { - "type": "ssrRouterLinkLoad" - } - ] - } - }, - { - "name": "pracht", - "displayName": "Pracht", - "frameworkPackage": "@pracht/core", - "focusedFramework": false, - "starter": { - "package": "starter-pracht", - "buildScript": "build", - "buildOutputDir": "dist", - "devServerPort": 3000, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-pracht", - "buildScript": "build", - "buildOutputDir": "dist", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - }, - { - "type": "ssrRouterLinkLoad" - } - ] - } - }, - { - "name": "react-router", - "displayName": "React Router", - "frameworkPackage": "@react-router/dev", - "focusedFramework": true, - "starter": { - "package": "starter-react-router", - "buildScript": "build", - "buildOutputDir": "build", - "devServerPort": 5173, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-react-router", - "buildScript": "build", - "buildOutputDir": "build", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - }, - { - "type": "ssrRouterLinkLoad" - } - ] - } - }, - { - "name": "solid-start", - "displayName": "SolidStart", - "frameworkPackage": "@solidjs/start", - "focusedFramework": true, - "starter": { - "package": "starter-solid-start", - "buildScript": "build", - "buildOutputDir": ".output", - "devServerPort": 3000, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-solid-start", - "buildScript": "build", - "buildOutputDir": ".output", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - } - ] - } - }, - { - "name": "sveltekit", - "displayName": "SvelteKit", - "frameworkPackage": "@sveltejs/kit", - "focusedFramework": true, - "starter": { - "package": "starter-sveltekit", - "buildScript": "build", - "buildOutputDir": "build", - "devServerPort": 5173, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-sveltekit", - "buildScript": "build", - "buildOutputDir": "build", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - } - ] - } - }, - { - "name": "tanstack", - "displayName": "TanStack Start", - "frameworkPackage": "@tanstack/react-start", - "focusedFramework": true, - "starter": { - "package": "starter-tanstack-start-react", - "buildScript": "build", - "buildOutputDir": ".output", - "devServerPort": 3000, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-tanstack-start-react", - "buildScript": "build", - "buildOutputDir": ".output", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - }, - { - "type": "ssrRouterLinkLoad" - } - ] - } - } -] diff --git a/.github/workflows/generate-stats.yml b/.github/workflows/generate-stats.yml index ddf417436..5467f954d 100644 --- a/.github/workflows/generate-stats.yml +++ b/.github/workflows/generate-stats.yml @@ -24,7 +24,7 @@ on: - 'packages/stats-generator/**' - '.github/workflows/generate-stats.yml' - '.github/workflows/measure-framework.yml' - - '.github/frameworks.json' + - 'packages/stats-generator/src/frameworks.ts' - 'package.json' jobs: @@ -44,19 +44,24 @@ jobs: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Set matrices from frameworks.json + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + + - name: Set matrices from TypeScript config id: set-matrix run: | - FRAMEWORKS=$(cat .github/frameworks.json) - echo "install=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "build=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "dev-server=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-router-link-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRouterLinkLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "deps=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT - echo "client-side-rendered=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["clientSideRendered"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "server-side-rendered=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["serverSideRendered"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "install=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "build=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "dev-server=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-router-link-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["ssrRouterLinkLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "deps=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT + echo "client-side-rendered=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["clientSideRendered"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "server-side-rendered=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["serverSideRendered"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT measure: needs: setup diff --git a/.github/workflows/measure-framework.yml b/.github/workflows/measure-framework.yml index e14a7e73c..0c166b6fe 100644 --- a/.github/workflows/measure-framework.yml +++ b/.github/workflows/measure-framework.yml @@ -74,8 +74,7 @@ jobs: - name: Run install benchmark run: | - RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "install") | .runFrequency') - pnpm --filter @framework-tracker/stats-generator run:install ${{ matrix.framework.package }} $RUN_FREQUENCY + pnpm --filter @framework-tracker/stats-generator run:install ${{ matrix.framework.package }} - name: Upload install stats uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -110,8 +109,7 @@ jobs: - name: Run build benchmark run: | - RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "build") | .runFrequency') - pnpm --filter @framework-tracker/stats-generator run:build ${{ matrix.framework.package }} $RUN_FREQUENCY + pnpm --filter @framework-tracker/stats-generator run:build ${{ matrix.framework.package }} - name: Scan build output for vendored core-js run: pnpm --filter @framework-tracker/stats-generator run:corejs ${{ matrix.framework.package }} @@ -171,8 +169,7 @@ jobs: - name: Run dev server benchmark run: | - RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "devServer") | .runFrequency') - pnpm --filter @framework-tracker/stats-generator run:dev-server ${{ matrix.framework.package }} $RUN_FREQUENCY + pnpm --filter @framework-tracker/stats-generator run:dev-server ${{ matrix.framework.package }} - name: Upload dev server stats uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/preview-stats.yml b/.github/workflows/preview-stats.yml index f5c272b8a..01144b28d 100644 --- a/.github/workflows/preview-stats.yml +++ b/.github/workflows/preview-stats.yml @@ -26,6 +26,11 @@ jobs: with: fetch-depth: 0 + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + - name: Detect PR, changed frameworks, and build matrices id: detect env: @@ -41,7 +46,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_OUTPUT echo "Detected PR #$PR_NUMBER targeting $BASE_REF" - FRAMEWORKS=$(cat .github/frameworks.json) + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) CHANGED_FILES=$(git diff --name-only origin/$BASE_REF...HEAD -- 'packages/starter-*/package.json' 'packages/starter-*/pnpm-lock.yaml' 'packages/starter-*/first-party-dependencies.json' 'packages/app-*/package.json') @@ -75,13 +80,13 @@ jobs: echo "has-changes=true" >> $GITHUB_OUTPUT echo "framework-names=$NAMES_JSON" >> $GITHUB_OUTPUT - echo "install=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "build=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "dev-server=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | map(.type) | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-router-link-load=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRouterLinkLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "deps=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT + echo "install=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "build=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "dev-server=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-router-link-load=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | contains(["ssrRouterLinkLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "deps=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT measure: needs: setup diff --git a/.github/workflows/sync-version.yml b/.github/workflows/sync-version.yml index cec576611..724accc91 100644 --- a/.github/workflows/sync-version.yml +++ b/.github/workflows/sync-version.yml @@ -5,7 +5,7 @@ on: paths: - 'packages/starter-*/package.json' - 'packages/app-*/package.json' - - '.github/frameworks.json' + - 'packages/stats-generator/src/frameworks.ts' workflow_dispatch: # Allows manual triggering from GitHub Actions UI jobs: diff --git a/.github/workflows/validate-stats.yml b/.github/workflows/validate-stats.yml index bd14eddd1..25f75f2ac 100644 --- a/.github/workflows/validate-stats.yml +++ b/.github/workflows/validate-stats.yml @@ -7,7 +7,7 @@ on: - '.github/workflows/generate-stats.yml' - '.github/workflows/measure-framework.yml' - '.github/workflows/validate-stats.yml' - - '.github/frameworks.json' + - 'packages/stats-generator/src/frameworks.ts' - 'packages/app-*/**' - '!packages/app-*/ci-stats.json' - '!packages/app-*/stats/**' @@ -46,7 +46,7 @@ jobs: - name: Install all package dependencies run: | - FRAMEWORKS=$(cat .github/frameworks.json) + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) for PKG in $(echo "$FRAMEWORKS" | jq -r '.[].starter.package // empty, .[].app.package // empty'); do echo "Installing $PKG..." (cd packages/$PKG && pnpm install --frozen-lockfile) @@ -57,8 +57,8 @@ jobs: set -e echo "=== Running SSR request throughput benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRequestThroughput"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["ssrRequestThroughput"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.app.package') echo "Building $PKG..." (cd packages/$PKG && pnpm build) @@ -68,8 +68,8 @@ jobs: echo "" echo "=== Running SSR load benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrLoad"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["ssrLoad"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.app.package') echo "Building $PKG..." (cd packages/$PKG && pnpm build) @@ -79,8 +79,8 @@ jobs: echo "" echo "=== Running SSR router link load benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRouterLinkLoad"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["ssrRouterLinkLoad"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.app.package') echo "Building $PKG..." (cd packages/$PKG && pnpm build) @@ -90,23 +90,23 @@ jobs: echo "" echo "=== Running starter benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) echo "$FRAMEWORKS" | jq -c '.[] | select(.starter)' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.starter.package') - echo "Running install benchmark for $PKG (1 run)..." - pnpm --filter @framework-tracker/stats-generator run:install $PKG 1 + echo "Running install benchmark for $PKG..." + pnpm --filter @framework-tracker/stats-generator run:install $PKG echo "Running build benchmark for $PKG..." pnpm --filter @framework-tracker/stats-generator run:build $PKG - echo "Running dev server benchmark for $PKG (1 run)..." - pnpm --filter @framework-tracker/stats-generator run:dev-server $PKG 1 + echo "Running dev server benchmark for $PKG..." + pnpm --filter @framework-tracker/stats-generator run:dev-server $PKG echo "Running Node engines scan for $PKG..." pnpm --filter @framework-tracker/stats-generator run:node-engines $PKG done echo "" echo "=== Running dependency scans ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | contains(["dependencies"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.starter.package') echo "Running e18e analysis for $PKG..." (cd packages/$PKG && npx @e18e/cli@0.5.0 analyze --json > e18e-stats.json || true) @@ -120,8 +120,8 @@ jobs: echo "" echo "=== Running browser baseline scans ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["browserBaseline"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | contains(["browserBaseline"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.starter.package') echo "Scanning browser baseline requirements for $PKG..." pnpm --filter @framework-tracker/stats-generator run:browser-baseline $PKG @@ -129,8 +129,8 @@ jobs: echo "" echo "=== Running client-side rendered benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["clientSideRendered"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["clientSideRendered"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.app.package') echo "Building client-side rendered app for $PKG..." (cd packages/$PKG && pnpm build) @@ -140,8 +140,8 @@ jobs: echo "" echo "=== Running server-side rendered benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["serverSideRendered"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["serverSideRendered"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.app.package') echo "Building server-side rendered app for $PKG..." (cd packages/$PKG && pnpm build) diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index 1c01436ed..0f506d037 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -266,7 +266,7 @@ Installed using the CLI with the following setup: response body is not read, because several dev servers stream the page and body time would measure page size rather than startup. - The port is the starter's default dev port, recorded as `devServerPort` in - `.github/frameworks.json`. The starters are not modified, so whatever they + `packages/stats-generator/src/frameworks.ts`. The starters are not modified, so whatever they ship is inside the measurement: Nuxt and TanStack Start enable devtools, and Astro and React Router generate types at boot. - The dev server runs with a minimal environment (`PATH`, `HOME`, `TMPDIR`, diff --git a/packages/stats-generator/src/collect-stats.ts b/packages/stats-generator/src/collect-stats.ts index 7559077a6..80ce250b7 100644 --- a/packages/stats-generator/src/collect-stats.ts +++ b/packages/stats-generator/src/collect-stats.ts @@ -38,7 +38,7 @@ async function processStarter(framework: FrameworkConfig, order: number) { const ciStats = (await getCIStats(pkgDir)) ?? {} - const hasDependencies = measurements.some((m) => m.type === 'dependencies') + const hasDependencies = measurements.includes('dependencies') const dependencyStats = hasDependencies && (ciStats.prodDependencies === undefined || @@ -46,9 +46,7 @@ async function processStarter(framework: FrameworkConfig, order: number) { ciStats.allDependencies === undefined) ? getDependencyCountsFromPackageMetadata(pkgDir) : {} - const hasBrowserBaseline = measurements.some( - (m) => m.type === 'browserBaseline', - ) + const hasBrowserBaseline = measurements.includes('browserBaseline') const browserBaselineStats = hasBrowserBaseline && ciStats.browserBaselineTests === undefined ? await getBrowserBaselineStats(pkgDir) diff --git a/packages/stats-generator/src/frameworks.ts b/packages/stats-generator/src/frameworks.ts new file mode 100644 index 000000000..f88fe5268 --- /dev/null +++ b/packages/stats-generator/src/frameworks.ts @@ -0,0 +1,261 @@ +import type { FrameworkConfig } from './types.ts' + +export const frameworks = [ + { + name: 'baseline-html', + displayName: 'Baseline HTML', + frameworkPackage: 'node', + focusedFramework: false, + app: { + package: 'app-baseline-html', + buildScript: 'build', + buildOutputDir: '.', + measurements: ['ssrRequestThroughput', 'ssrLoad'], + }, + }, + { + name: 'astro', + displayName: 'Astro', + frameworkPackage: 'astro', + focusedFramework: true, + starter: { + package: 'starter-astro', + buildScript: 'build', + buildOutputDir: 'dist', + devServerPort: 4321, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-astro', + buildScript: 'build', + buildOutputDir: 'dist', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + ], + }, + }, + { + name: 'next', + displayName: 'Next.js', + frameworkPackage: 'next', + focusedFramework: true, + starter: { + package: 'starter-next-js', + buildScript: 'build', + buildOutputDir: '.next', + devServerPort: 3000, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-next-js', + buildScript: 'build', + buildOutputDir: '.next', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + 'ssrRouterLinkLoad', + ], + }, + }, + { + name: 'nuxt', + displayName: 'Nuxt', + frameworkPackage: 'nuxt', + focusedFramework: true, + starter: { + package: 'starter-nuxt', + buildScript: 'build', + buildOutputDir: '.output', + devServerPort: 3000, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-nuxt', + buildScript: 'build', + buildOutputDir: '.output', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + 'ssrRouterLinkLoad', + ], + }, + }, + { + name: 'pracht', + displayName: 'Pracht', + frameworkPackage: '@pracht/core', + focusedFramework: false, + starter: { + package: 'starter-pracht', + buildScript: 'build', + buildOutputDir: 'dist', + devServerPort: 3000, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-pracht', + buildScript: 'build', + buildOutputDir: 'dist', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + 'ssrRouterLinkLoad', + ], + }, + }, + { + name: 'react-router', + displayName: 'React Router', + frameworkPackage: '@react-router/dev', + focusedFramework: true, + starter: { + package: 'starter-react-router', + buildScript: 'build', + buildOutputDir: 'build', + devServerPort: 5173, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-react-router', + buildScript: 'build', + buildOutputDir: 'build', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + 'ssrRouterLinkLoad', + ], + }, + }, + { + name: 'solid-start', + displayName: 'SolidStart', + frameworkPackage: '@solidjs/start', + focusedFramework: true, + starter: { + package: 'starter-solid-start', + buildScript: 'build', + buildOutputDir: '.output', + devServerPort: 3000, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-solid-start', + buildScript: 'build', + buildOutputDir: '.output', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + ], + }, + }, + { + name: 'sveltekit', + displayName: 'SvelteKit', + frameworkPackage: '@sveltejs/kit', + focusedFramework: true, + starter: { + package: 'starter-sveltekit', + buildScript: 'build', + buildOutputDir: 'build', + devServerPort: 5173, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-sveltekit', + buildScript: 'build', + buildOutputDir: 'build', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + ], + }, + }, + { + name: 'tanstack', + displayName: 'TanStack Start', + frameworkPackage: '@tanstack/react-start', + focusedFramework: true, + starter: { + package: 'starter-tanstack-start-react', + buildScript: 'build', + buildOutputDir: '.output', + devServerPort: 3000, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-tanstack-start-react', + buildScript: 'build', + buildOutputDir: '.output', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + 'ssrRouterLinkLoad', + ], + }, + }, +] satisfies FrameworkConfig[] diff --git a/packages/stats-generator/src/get-frameworks.ts b/packages/stats-generator/src/get-frameworks.ts index 0fbea3d75..18425bffa 100644 --- a/packages/stats-generator/src/get-frameworks.ts +++ b/packages/stats-generator/src/get-frameworks.ts @@ -1,10 +1,6 @@ -import { readFile } from 'node:fs/promises' -import { join } from 'node:path' -import { packagesDir } from './constants.ts' +import { frameworks } from './frameworks.ts' import type { FrameworkConfig } from './types.ts' export async function getFrameworks(): Promise { - const configPath = join(packagesDir, '..', '.github', 'frameworks.json') - const content = await readFile(configPath, 'utf-8') - return JSON.parse(content) as FrameworkConfig[] + return frameworks } diff --git a/packages/stats-generator/src/measurements.ts b/packages/stats-generator/src/measurements.ts new file mode 100644 index 000000000..4304d1b9b --- /dev/null +++ b/packages/stats-generator/src/measurements.ts @@ -0,0 +1,16 @@ +/** Shared defaults for every framework. Scans run once; benchmarks repeat. */ +export const measurements = { + install: { runs: 5 }, + build: { runs: 5 }, + devServer: { runs: 5 }, + dependencies: {}, + browserBaseline: {}, + nodeEngines: {}, + ssrRequestThroughput: { runs: 5 }, + ssrLoad: { runs: 3 }, + ssrRouterLinkLoad: { runs: 3 }, + clientSideRendered: { runs: 5 }, + serverSideRendered: { runs: 5 }, +} satisfies Record + +export type MeasurementName = keyof typeof measurements diff --git a/packages/stats-generator/src/print-frameworks.ts b/packages/stats-generator/src/print-frameworks.ts new file mode 100644 index 000000000..2a2148c0a --- /dev/null +++ b/packages/stats-generator/src/print-frameworks.ts @@ -0,0 +1,3 @@ +import { frameworks } from './frameworks.ts' + +process.stdout.write(JSON.stringify(frameworks)) diff --git a/packages/stats-generator/src/run-build-benchmark.ts b/packages/stats-generator/src/run-build-benchmark.ts index c9d4125e5..9df1f9570 100644 --- a/packages/stats-generator/src/run-build-benchmark.ts +++ b/packages/stats-generator/src/run-build-benchmark.ts @@ -1,3 +1,4 @@ +import { measurements } from './measurements.ts' import { execFileSync } from 'node:child_process' import { cpSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' @@ -49,7 +50,7 @@ async function main() { 'Usage: run-build-benchmark [run-frequency]\nExample: run-build-benchmark starter-astro 5', ) - const runFrequency = parseRunFrequency(args[0]) + const runFrequency = parseRunFrequency(args[0], measurements.build.runs) const { framework, testConfig } = await getFrameworkByPackage(packageName) diff --git a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts index 63883e884..bdfdd6aa7 100644 --- a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts @@ -1,3 +1,5 @@ +import { parseRunFrequency } from './benchmark-utils.ts' +import { measurements } from './measurements.ts' import { ClientSideRenderedStatsSchema } from './schemas.ts' import { join } from 'node:path' import { runClientSideRenderedBenchmark } from './clientSideRendered/index.ts' @@ -16,7 +18,7 @@ async function main() { const { packageName, args } = parseArgs( 'Usage: run-client-side-rendered-benchmark [runs]\nExample: run-client-side-rendered-benchmark app-astro 5', ) - const runs = args[0] ? Number.parseInt(args[0], 10) : 5 + const runs = parseRunFrequency(args[0], measurements.clientSideRendered.runs) console.info( `Running client-side rendered benchmark for ${packageName} (${runs} runs)...\n`, diff --git a/packages/stats-generator/src/run-dev-server-benchmark.ts b/packages/stats-generator/src/run-dev-server-benchmark.ts index 2d0423182..b56932d7b 100644 --- a/packages/stats-generator/src/run-dev-server-benchmark.ts +++ b/packages/stats-generator/src/run-dev-server-benchmark.ts @@ -1,3 +1,4 @@ +import { measurements } from './measurements.ts' import { rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -97,13 +98,13 @@ async function main() { 'Usage: run-dev-server-benchmark [run-frequency]\nExample: run-dev-server-benchmark starter-astro 5', ) - const runFrequency = parseRunFrequency(args[0]) + const runFrequency = parseRunFrequency(args[0], measurements.devServer.runs) const { framework, testConfig } = await getFrameworkByPackage(packageName) const port = testConfig.devServerPort if (port === undefined) { throw new Error( - `${packageName} has no devServerPort in .github/frameworks.json`, + `${packageName} has no devServerPort in packages/stats-generator/src/frameworks.ts`, ) } diff --git a/packages/stats-generator/src/run-install-benchmark.ts b/packages/stats-generator/src/run-install-benchmark.ts index e01047c7e..f5d62296e 100644 --- a/packages/stats-generator/src/run-install-benchmark.ts +++ b/packages/stats-generator/src/run-install-benchmark.ts @@ -1,3 +1,4 @@ +import { measurements } from './measurements.ts' import { summarizeMetric } from './sample-statistics.ts' import { execSync } from 'node:child_process' import { cpSync, mkdirSync, rmSync } from 'node:fs' @@ -65,7 +66,7 @@ async function main() { 'Usage: run-install-benchmark [run-frequency]\nExample: run-install-benchmark starter-astro 5', ) - const runFrequency = parseRunFrequency(args[0]) + const runFrequency = parseRunFrequency(args[0], measurements.install.runs) const { framework } = await getFrameworkByPackage(packageName) diff --git a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts index 77f639b5b..62db9d167 100644 --- a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts @@ -1,3 +1,5 @@ +import { parseRunFrequency } from './benchmark-utils.ts' +import { measurements } from './measurements.ts' import { ServerSideRenderedStatsSchema } from './schemas.ts' import { join } from 'node:path' import { runServerSideRenderedBenchmark } from './serverSideRendered/index.ts' @@ -16,7 +18,7 @@ async function main() { const { packageName, args } = parseArgs( 'Usage: run-server-side-rendered-benchmark [runs]\nExample: run-server-side-rendered-benchmark app-astro 5', ) - const runs = args[0] ? Number.parseInt(args[0], 10) : 5 + const runs = parseRunFrequency(args[0], measurements.serverSideRendered.runs) console.info( `Running server side rendered benchmark for ${packageName} (${runs} runs)...\n`, diff --git a/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts b/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts index 2f94e7526..bdf645cf6 100644 --- a/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts +++ b/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts @@ -1,3 +1,4 @@ +import { measurements } from './measurements.ts' import { parseRunFrequency } from './benchmark-utils.ts' import { RequestThroughputResultSchema } from './schemas.ts' import { join } from 'node:path' @@ -21,7 +22,10 @@ async function main() { 'Usage: run-ssr-request-throughput-benchmark [runs]\nExample: run-ssr-request-throughput-benchmark app-astro 5', ) - const runs = parseRunFrequency(args[0]) + const runs = parseRunFrequency( + args[0], + measurements.ssrRequestThroughput.runs, + ) console.info( `Running SSR request throughput benchmark for ${packageName} (${runs} runs)...\n`, diff --git a/packages/stats-generator/src/schemas.ts b/packages/stats-generator/src/schemas.ts index 3280ea28b..7b51e3e38 100644 --- a/packages/stats-generator/src/schemas.ts +++ b/packages/stats-generator/src/schemas.ts @@ -13,7 +13,7 @@ export const MetricBaseSchema = z.object({ }) export type MetricBase = z.infer -/** Framework identifiers configured in .github/frameworks.json. */ +/** Framework identifiers configured in packages/stats-generator/src/frameworks.ts. */ export const FrameworkSchema = z.enum([ 'baseline-html', 'astro', @@ -26,7 +26,7 @@ export const FrameworkSchema = z.enum([ 'tanstack', ]) -/** Tested package folders configured in .github/frameworks.json. */ +/** Tested package folders configured in packages/stats-generator/src/frameworks.ts. */ export const PackagePathSchema = z.enum([ 'packages/app-baseline-html', 'packages/starter-astro', diff --git a/packages/stats-generator/src/ssrLoad/run-load-test.ts b/packages/stats-generator/src/ssrLoad/run-load-test.ts index 089ff643d..f7d1e9bc2 100644 --- a/packages/stats-generator/src/ssrLoad/run-load-test.ts +++ b/packages/stats-generator/src/ssrLoad/run-load-test.ts @@ -1,3 +1,4 @@ +import { measurements } from '../measurements.ts' import { getLoadPeakStage, selectRepresentativeIteration } from './selection.ts' import { SSRLoadTestsSchema } from '../schemas.ts' import { summarizeMetric } from '../sample-statistics.ts' @@ -7,7 +8,7 @@ import autocannon from 'autocannon' import type { SSRLoadStageStats, SSRLoadSweep, SSRLoadTests } from './types.ts' const STAGE_DURATION_SECONDS = 5 -const LOAD_SWEEP_COUNT = 3 + const WARMUP_ATTEMPTS = 3 const RECOVERY_MS = 5000 const WORKER_STAGES = [1, 5, 10, 25, 50, 100, 200] as const @@ -50,21 +51,21 @@ export async function runLoadTest( pause: (ms: number) => Promise = delay, ): Promise { const samples: SSRLoadSweep[] = [] - for (let run = 1; run <= LOAD_SWEEP_COUNT; run++) { + for (let run = 1; run <= measurements[getSSRLoadKind()].runs; run++) { if (run > 1) { console.info( - ` Sweep ${run}/${LOAD_SWEEP_COUNT}: recovering for ${RECOVERY_MS / 1000}s`, + ` Sweep ${run}/${measurements[getSSRLoadKind()].runs}: recovering for ${RECOVERY_MS / 1000}s`, ) await pause(RECOVERY_MS) } for (let attempt = 1; attempt <= WARMUP_ATTEMPTS; attempt++) { console.info( - ` Sweep ${run}/${LOAD_SWEEP_COUNT}: warm-up ${attempt}/${WARMUP_ATTEMPTS} for ${STAGE_DURATION_SECONDS}s`, + ` Sweep ${run}/${measurements[getSSRLoadKind()].runs}: warm-up ${attempt}/${WARMUP_ATTEMPTS} for ${STAGE_DURATION_SECONDS}s`, ) const warmup = await measureStage(url, WORKER_STAGES[0]) if (warmup.errors === 0 && warmup.requests > 0) break - const details = `sweep ${run}/${LOAD_SWEEP_COUNT}, attempt ${attempt}/${WARMUP_ATTEMPTS}: ${warmup.requests} requests, ${warmup.errors} errors, ${warmup.elapsedTimeMs}ms` + const details = `sweep ${run}/${measurements[getSSRLoadKind()].runs}, attempt ${attempt}/${WARMUP_ATTEMPTS}: ${warmup.requests} requests, ${warmup.errors} errors, ${warmup.elapsedTimeMs}ms` if (attempt === WARMUP_ATTEMPTS) { throw new Error( `SSR load warm-up failed (${details}); expected requests without errors after recovery.`, diff --git a/packages/stats-generator/src/types.ts b/packages/stats-generator/src/types.ts index 0411731b3..4b1e7e1f9 100644 --- a/packages/stats-generator/src/types.ts +++ b/packages/stats-generator/src/types.ts @@ -16,30 +16,14 @@ import type { InstallResultSchema, } from './schemas.ts' import type { SSRLoadTests } from './ssrLoad/types.ts' -type MeasurementType = - | 'install' - | 'build' - | 'devServer' - | 'test' - | 'dependencies' - | 'browserBaseline' - | 'ssrRequestThroughput' - | 'ssrLoad' - | 'ssrRouterLinkLoad' - | 'clientSideRendered' - | 'serverSideRendered' - -interface MeasurementConfig { - type: MeasurementType - runFrequency?: number -} +import type { MeasurementName } from './measurements.ts' export interface TestConfig { package: string buildScript: string buildOutputDir: string devServerPort?: number - measurements: MeasurementConfig[] + measurements: MeasurementName[] } export interface InteractionTiming { diff --git a/packages/stats-generator/src/validate-stats.ts b/packages/stats-generator/src/validate-stats.ts index 3d7d8c3b2..60a475801 100644 --- a/packages/stats-generator/src/validate-stats.ts +++ b/packages/stats-generator/src/validate-stats.ts @@ -151,11 +151,7 @@ async function main() { const errors: string[] = [] for (const { type: benchType, file, schema } of STARTER_BENCHMARKS) { if (filterType && filterType !== benchType) continue - if ( - !framework.starter.measurements.some( - (measurement) => measurement.type === benchType, - ) - ) { + if (!framework.starter.measurements.includes(benchType)) { continue } @@ -185,11 +181,7 @@ async function main() { const errors: string[] = [] for (const { type: benchType, file, schema } of APP_BENCHMARKS) { if (filterType && filterType !== benchType) continue - if ( - !framework.app.measurements.some( - (measurement) => measurement.type === benchType, - ) - ) { + if (!framework.app.measurements.includes(benchType)) { continue }