diff --git a/.github/frameworks.json b/.github/frameworks.json deleted file mode 100644 index b218551b4..000000000 --- a/.github/frameworks.json +++ /dev/null @@ -1,444 +0,0 @@ -[ - { - "name": "baseline-html", - "displayName": "Baseline HTML", - "frameworkPackage": "node", - "focusedFramework": false, - "app": { - "package": "app-baseline-html", - "buildScript": "build", - "buildOutputDir": ".", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "ssrLoad" - } - ] - } - }, - { - "name": "astro", - "displayName": "Astro", - "frameworkPackage": "astro", - "focusedFramework": true, - "starter": { - "package": "starter-astro", - "buildScript": "build", - "buildOutputDir": "dist", - "devServerPort": 4321, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-astro", - "buildScript": "build", - "buildOutputDir": "dist", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - } - ] - } - }, - { - "name": "next", - "displayName": "Next.js", - "frameworkPackage": "next", - "focusedFramework": true, - "starter": { - "package": "starter-next-js", - "buildScript": "build", - "buildOutputDir": ".next", - "devServerPort": 3000, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-next-js", - "buildScript": "build", - "buildOutputDir": ".next", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - }, - { - "type": "ssrRouterLinkLoad" - } - ] - } - }, - { - "name": "nuxt", - "displayName": "Nuxt", - "frameworkPackage": "nuxt", - "focusedFramework": true, - "starter": { - "package": "starter-nuxt", - "buildScript": "build", - "buildOutputDir": ".output", - "devServerPort": 3000, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-nuxt", - "buildScript": "build", - "buildOutputDir": ".output", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - }, - { - "type": "ssrRouterLinkLoad" - } - ] - } - }, - { - "name": "pracht", - "displayName": "Pracht", - "frameworkPackage": "@pracht/core", - "focusedFramework": false, - "starter": { - "package": "starter-pracht", - "buildScript": "build", - "buildOutputDir": "dist", - "devServerPort": 3000, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-pracht", - "buildScript": "build", - "buildOutputDir": "dist", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - }, - { - "type": "ssrRouterLinkLoad" - } - ] - } - }, - { - "name": "react-router", - "displayName": "React Router", - "frameworkPackage": "@react-router/dev", - "focusedFramework": true, - "starter": { - "package": "starter-react-router", - "buildScript": "build", - "buildOutputDir": "build", - "devServerPort": 5173, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-react-router", - "buildScript": "build", - "buildOutputDir": "build", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - }, - { - "type": "ssrRouterLinkLoad" - } - ] - } - }, - { - "name": "solid-start", - "displayName": "SolidStart", - "frameworkPackage": "@solidjs/start", - "focusedFramework": true, - "starter": { - "package": "starter-solid-start", - "buildScript": "build", - "buildOutputDir": ".output", - "devServerPort": 3000, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-solid-start", - "buildScript": "build", - "buildOutputDir": ".output", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - } - ] - } - }, - { - "name": "sveltekit", - "displayName": "SvelteKit", - "frameworkPackage": "@sveltejs/kit", - "focusedFramework": true, - "starter": { - "package": "starter-sveltekit", - "buildScript": "build", - "buildOutputDir": "build", - "devServerPort": 5173, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-sveltekit", - "buildScript": "build", - "buildOutputDir": "build", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - } - ] - } - }, - { - "name": "tanstack", - "displayName": "TanStack Start", - "frameworkPackage": "@tanstack/react-start", - "focusedFramework": true, - "starter": { - "package": "starter-tanstack-start-react", - "buildScript": "build", - "buildOutputDir": ".output", - "devServerPort": 3000, - "measurements": [ - { - "type": "install", - "runFrequency": 5 - }, - { - "type": "build", - "runFrequency": 5 - }, - { - "type": "devServer", - "runFrequency": 5 - }, - { - "type": "dependencies" - }, - { - "type": "browserBaseline" - } - ] - }, - "app": { - "package": "app-tanstack-start-react", - "buildScript": "build", - "buildOutputDir": ".output", - "measurements": [ - { - "type": "ssrRequestThroughput" - }, - { - "type": "clientSideRendered" - }, - { - "type": "serverSideRendered" - }, - { - "type": "ssrLoad" - }, - { - "type": "ssrRouterLinkLoad" - } - ] - } - } -] diff --git a/.github/workflows/generate-stats.yml b/.github/workflows/generate-stats.yml index ddf417436..5467f954d 100644 --- a/.github/workflows/generate-stats.yml +++ b/.github/workflows/generate-stats.yml @@ -24,7 +24,7 @@ on: - 'packages/stats-generator/**' - '.github/workflows/generate-stats.yml' - '.github/workflows/measure-framework.yml' - - '.github/frameworks.json' + - 'packages/stats-generator/src/frameworks.ts' - 'package.json' jobs: @@ -44,19 +44,24 @@ jobs: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Set matrices from frameworks.json + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + + - name: Set matrices from TypeScript config id: set-matrix run: | - FRAMEWORKS=$(cat .github/frameworks.json) - echo "install=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "build=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "dev-server=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-router-link-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRouterLinkLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "deps=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT - echo "client-side-rendered=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["clientSideRendered"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "server-side-rendered=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["serverSideRendered"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "install=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "build=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "dev-server=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-router-link-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["ssrRouterLinkLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "deps=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT + echo "client-side-rendered=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["clientSideRendered"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "server-side-rendered=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | contains(["serverSideRendered"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT measure: needs: setup diff --git a/.github/workflows/measure-framework.yml b/.github/workflows/measure-framework.yml index e14a7e73c..0c166b6fe 100644 --- a/.github/workflows/measure-framework.yml +++ b/.github/workflows/measure-framework.yml @@ -74,8 +74,7 @@ jobs: - name: Run install benchmark run: | - RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "install") | .runFrequency') - pnpm --filter @framework-tracker/stats-generator run:install ${{ matrix.framework.package }} $RUN_FREQUENCY + pnpm --filter @framework-tracker/stats-generator run:install ${{ matrix.framework.package }} - name: Upload install stats uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -110,8 +109,7 @@ jobs: - name: Run build benchmark run: | - RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "build") | .runFrequency') - pnpm --filter @framework-tracker/stats-generator run:build ${{ matrix.framework.package }} $RUN_FREQUENCY + pnpm --filter @framework-tracker/stats-generator run:build ${{ matrix.framework.package }} - name: Scan build output for vendored core-js run: pnpm --filter @framework-tracker/stats-generator run:corejs ${{ matrix.framework.package }} @@ -171,8 +169,7 @@ jobs: - name: Run dev server benchmark run: | - RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "devServer") | .runFrequency') - pnpm --filter @framework-tracker/stats-generator run:dev-server ${{ matrix.framework.package }} $RUN_FREQUENCY + pnpm --filter @framework-tracker/stats-generator run:dev-server ${{ matrix.framework.package }} - name: Upload dev server stats uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/preview-stats.yml b/.github/workflows/preview-stats.yml index f5c272b8a..01144b28d 100644 --- a/.github/workflows/preview-stats.yml +++ b/.github/workflows/preview-stats.yml @@ -26,6 +26,11 @@ jobs: with: fetch-depth: 0 + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '24' + - name: Detect PR, changed frameworks, and build matrices id: detect env: @@ -41,7 +46,7 @@ jobs: echo "pr-number=$PR_NUMBER" >> $GITHUB_OUTPUT echo "Detected PR #$PR_NUMBER targeting $BASE_REF" - FRAMEWORKS=$(cat .github/frameworks.json) + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) CHANGED_FILES=$(git diff --name-only origin/$BASE_REF...HEAD -- 'packages/starter-*/package.json' 'packages/starter-*/pnpm-lock.yaml' 'packages/starter-*/first-party-dependencies.json' 'packages/app-*/package.json') @@ -75,13 +80,13 @@ jobs: echo "has-changes=true" >> $GITHUB_OUTPUT echo "framework-names=$NAMES_JSON" >> $GITHUB_OUTPUT - echo "install=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "build=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "dev-server=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | map(.type) | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "ssr-router-link-load=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRouterLinkLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT - echo "deps=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT + echo "install=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "build=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "dev-server=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "ssr-router-link-load=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | contains(["ssrRouterLinkLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT + echo "deps=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT measure: needs: setup diff --git a/.github/workflows/sync-version.yml b/.github/workflows/sync-version.yml index cec576611..724accc91 100644 --- a/.github/workflows/sync-version.yml +++ b/.github/workflows/sync-version.yml @@ -5,7 +5,7 @@ on: paths: - 'packages/starter-*/package.json' - 'packages/app-*/package.json' - - '.github/frameworks.json' + - 'packages/stats-generator/src/frameworks.ts' workflow_dispatch: # Allows manual triggering from GitHub Actions UI jobs: diff --git a/.github/workflows/validate-stats.yml b/.github/workflows/validate-stats.yml index bd14eddd1..25f75f2ac 100644 --- a/.github/workflows/validate-stats.yml +++ b/.github/workflows/validate-stats.yml @@ -7,7 +7,7 @@ on: - '.github/workflows/generate-stats.yml' - '.github/workflows/measure-framework.yml' - '.github/workflows/validate-stats.yml' - - '.github/frameworks.json' + - 'packages/stats-generator/src/frameworks.ts' - 'packages/app-*/**' - '!packages/app-*/ci-stats.json' - '!packages/app-*/stats/**' @@ -46,7 +46,7 @@ jobs: - name: Install all package dependencies run: | - FRAMEWORKS=$(cat .github/frameworks.json) + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) for PKG in $(echo "$FRAMEWORKS" | jq -r '.[].starter.package // empty, .[].app.package // empty'); do echo "Installing $PKG..." (cd packages/$PKG && pnpm install --frozen-lockfile) @@ -57,8 +57,8 @@ jobs: set -e echo "=== Running SSR request throughput benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRequestThroughput"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["ssrRequestThroughput"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.app.package') echo "Building $PKG..." (cd packages/$PKG && pnpm build) @@ -68,8 +68,8 @@ jobs: echo "" echo "=== Running SSR load benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrLoad"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["ssrLoad"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.app.package') echo "Building $PKG..." (cd packages/$PKG && pnpm build) @@ -79,8 +79,8 @@ jobs: echo "" echo "=== Running SSR router link load benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRouterLinkLoad"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["ssrRouterLinkLoad"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.app.package') echo "Building $PKG..." (cd packages/$PKG && pnpm build) @@ -90,23 +90,23 @@ jobs: echo "" echo "=== Running starter benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) echo "$FRAMEWORKS" | jq -c '.[] | select(.starter)' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.starter.package') - echo "Running install benchmark for $PKG (1 run)..." - pnpm --filter @framework-tracker/stats-generator run:install $PKG 1 + echo "Running install benchmark for $PKG..." + pnpm --filter @framework-tracker/stats-generator run:install $PKG echo "Running build benchmark for $PKG..." pnpm --filter @framework-tracker/stats-generator run:build $PKG - echo "Running dev server benchmark for $PKG (1 run)..." - pnpm --filter @framework-tracker/stats-generator run:dev-server $PKG 1 + echo "Running dev server benchmark for $PKG..." + pnpm --filter @framework-tracker/stats-generator run:dev-server $PKG echo "Running Node engines scan for $PKG..." pnpm --filter @framework-tracker/stats-generator run:node-engines $PKG done echo "" echo "=== Running dependency scans ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | contains(["dependencies"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.starter.package') echo "Running e18e analysis for $PKG..." (cd packages/$PKG && npx @e18e/cli@0.5.0 analyze --json > e18e-stats.json || true) @@ -120,8 +120,8 @@ jobs: echo "" echo "=== Running browser baseline scans ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["browserBaseline"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.starter) | select(.starter.measurements | contains(["browserBaseline"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.starter.package') echo "Scanning browser baseline requirements for $PKG..." pnpm --filter @framework-tracker/stats-generator run:browser-baseline $PKG @@ -129,8 +129,8 @@ jobs: echo "" echo "=== Running client-side rendered benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["clientSideRendered"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["clientSideRendered"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.app.package') echo "Building client-side rendered app for $PKG..." (cd packages/$PKG && pnpm build) @@ -140,8 +140,8 @@ jobs: echo "" echo "=== Running server-side rendered benchmarks ===" - FRAMEWORKS=$(cat .github/frameworks.json) - echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | map(.type) | contains(["serverSideRendered"]))' | while read -r framework; do + FRAMEWORKS=$(node packages/stats-generator/src/print-frameworks.ts) + echo "$FRAMEWORKS" | jq -c '.[] | select(.app) | select(.app.measurements | contains(["serverSideRendered"]))' | while read -r framework; do PKG=$(echo "$framework" | jq -r '.app.package') echo "Building server-side rendered app for $PKG..." (cd packages/$PKG && pnpm build) diff --git a/packages/cwv-stats/package.json b/packages/cwv-stats/package.json index 65fd9e430..09b5ee9e2 100644 --- a/packages/cwv-stats/package.json +++ b/packages/cwv-stats/package.json @@ -7,6 +7,6 @@ "@types/node": "^26.1.2" }, "dependencies": { - "zod": "^4.4.3" + "zod": "4.6.7" } } diff --git a/packages/docs/package.json b/packages/docs/package.json index 82fa9c662..cebb51da3 100644 --- a/packages/docs/package.json +++ b/packages/docs/package.json @@ -16,7 +16,8 @@ "@astrojs/starlight": "^0.41.7", "@tanstack/charts": "^1.1.0", "astro": "^7.4.0", - "chart.js": "^4.5.1" + "chart.js": "^4.5.1", + "@framework-tracker/results": "workspace:*" }, "devDependencies": { "@astrojs/check": "^0.9.10", diff --git a/packages/docs/src/content.config.ts b/packages/docs/src/content.config.ts index 4bc575e21..2be6d7036 100644 --- a/packages/docs/src/content.config.ts +++ b/packages/docs/src/content.config.ts @@ -1,4 +1,8 @@ -import { requestThroughputContentSchema } from './lib/request-throughput' +import { + devtimeContentSchema, + runtimeContentSchema, + runtimeVersionsContentSchema, +} from './lib/result-content' import { defineCollection } from 'astro:content' import { file, glob } from 'astro/loaders' import { docsLoader } from '@astrojs/starlight/loaders' @@ -6,396 +10,9 @@ import { docsSchema } from '@astrojs/starlight/schema' import { z } from 'astro/zod' import { readdirSync } from 'node:fs' -const timeSchema = z.object({ - avgMs: z.number(), - medianMs: z.number().optional(), - standardDeviationMs: z.number().nonnegative().optional(), - minMs: z.number(), - maxMs: z.number(), - samplesMs: z.array(z.number()).nonempty().optional(), -}) - -const buildTimeSchema = z.union([ - z - .object({ - name: z.enum(['coldBuild', 'warmBuild']), - runs: z.number().int().positive(), - iterations: z.array(z.object({ timeMs: z.number().nonnegative() })), - timeMs: z.object({ - mean: z.number(), - median: z.number(), - min: z.number(), - max: z.number(), - }), - }) - .refine((value) => value.iterations.length === value.runs) - .transform((value) => ({ - avgMs: value.timeMs.mean, - medianMs: value.timeMs.median, - minMs: value.timeMs.min, - maxMs: value.timeMs.max, - samplesMs: value.iterations.map((run) => run.timeMs), - })), - timeSchema, -]) - -const devServerTimeSchema = z.union([ - z - .object({ - name: z.literal('devServer'), - runs: z.number().int().positive(), - iterations: z.array(z.object({ timeMs: z.number().nonnegative() })), - timeMs: z.object({ - mean: z.number(), - median: z.number(), - min: z.number(), - max: z.number(), - }), - }) - .refine((value) => value.iterations.length === value.runs) - .transform((value) => ({ - avgMs: value.timeMs.mean, - medianMs: value.timeMs.median, - minMs: value.timeMs.min, - maxMs: value.timeMs.max, - samplesMs: value.iterations.map((run) => run.timeMs), - })), - timeSchema, -]) - -const interactionTimingSchema = z.object({ - interactionLatencyMs: z.number().nonnegative(), - inputDelayMs: z.number().nonnegative(), - processingDurationMs: z.number().nonnegative(), - presentationDelayMs: z.number().nonnegative(), -}) - -const renderedTestSampleSchema = z.object({ - firstPaintMs: z.number().positive(), - fcpMs: z.number().positive(), - interactionTests: interactionTimingSchema, -}) - -const metricSchema = z.object({ - mean: z.number(), - median: z.number(), - min: z.number(), - max: z.number(), -}) -const ClientRenderingBenchmarkSchema = z - .object({ - name: z.literal('clientSideRendered'), - runs: z.number().int().positive(), - iterations: z.array(renderedTestSampleSchema), - firstPaintMs: metricSchema, - fcpMs: metricSchema, - interactionTests: z.object({ - scenario: z.literal('first-row-detail-navigation'), - source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: metricSchema, - inputDelayMs: metricSchema, - processingDurationMs: metricSchema, - presentationDelayMs: metricSchema, - }), - }) - .refine((value) => value.iterations.length === value.runs) - -const ServerRenderingBenchmarkSchema = z - .object({ - name: z.literal('serverSideRendered'), - runs: z.number().int().positive(), - iterations: z.array(renderedTestSampleSchema), - firstPaintMs: metricSchema, - fcpMs: metricSchema, - interactionTests: z.object({ - scenario: z.literal('first-row-detail-navigation'), - source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: metricSchema, - inputDelayMs: metricSchema, - processingDurationMs: metricSchema, - presentationDelayMs: metricSchema, - }), - }) - .refine((value) => value.iterations.length === value.runs) - -const dependencyStatsSchema = z.object({ - prodDependencies: z.number().int().nonnegative(), - devDependencies: z.number().int().nonnegative(), - allDependencies: z.number().int().nonnegative(), -}) - -const firstPartyDependencyStatsSchema = dependencyStatsSchema.extend({ - duplicateDependencies: z.number().int().nonnegative().optional(), -}) - -const devtimeSchema = z.object({ - framework: z.string(), - packagePath: z.string(), - name: z.string(), - type: z.string(), - package: z.string(), - isFocused: z.boolean(), - order: z.number(), - prodDependencies: z.number(), - devDependencies: z.number(), - allDependencies: z.number(), - firstPartyDependencies: firstPartyDependencyStatsSchema, - frameworkDependencies: dependencyStatsSchema.optional(), - install: z.object({ - name: z.literal('install'), - runs: z.number().int().positive(), - iterations: z.array(z.object({ installTimeMs: z.number().nonnegative() })), - installTimeMs: z.object({ - mean: z.number(), - median: z.number(), - min: z.number(), - max: z.number(), - }), - nodeModulesSizeBytes: z.number().nonnegative(), - }), - coldBuildTime: buildTimeSchema, - warmBuildTime: buildTimeSchema, - devServerStartTime: devServerTimeSchema.optional(), - duplicateDependencies: z.number().optional(), - depInstallSize: z.number().optional(), - vendoredCoreJsSize: z.number().optional(), - vendoredCoreJsUnnecessaryModules: z.array(z.string()).optional(), - browserBaselineTests: z - .object({ - baselineStatus: z.union([ - z.literal('high'), - z.literal('low'), - z.literal(false), - z.null(), - ]), - baselineYear: z.number().nullable(), - baselineReason: z.string().nullable(), - baselineFeatureCount: z.number(), - }) - .optional(), - minimumNodeVersion: z.string().optional(), - minimumNodeVersionImposedBy: z.array(z.string()).optional(), - timingMeasuredAt: z.string(), - runner: z.string(), - browserVersion: z.string().optional(), - frameworkVersion: z.string().optional(), - releasedAt: z.string().optional(), -}) - -const ssrLoadStageSchema = z.object({ - workers: z.number().int().positive(), - elapsedTimeMs: z.number().positive(), - requests: z.number().int().nonnegative(), - errors: z.number().int().nonnegative(), - requestsPerSec: z.number().nonnegative(), - avgLatencyMs: z.number().nonnegative(), - p50LatencyMs: z.number().nonnegative(), - p75LatencyMs: z.number().nonnegative(), - p90LatencyMs: z.number().nonnegative(), - p99LatencyMs: z.number().nonnegative(), - maxLatencyMs: z.number().nonnegative(), - bytesPerSec: z.number().nonnegative(), -}) - -const ssrLoadSweepSchema = z.array(ssrLoadStageSchema).nonempty() - -const ssrLoadStageSummarySchema = z.object({ - connections: z.number().int().positive(), - requests: metricSchema, - errors: metricSchema, - requestsPerSec: metricSchema, - avgLatencyMs: metricSchema, - p50LatencyMs: metricSchema, - p75LatencyMs: metricSchema, - p90LatencyMs: metricSchema, - p99LatencyMs: metricSchema, - maxLatencyMs: metricSchema, - bytesPerSec: metricSchema, -}) - -const ssrLoadTestsSchema = z - .object({ - name: z.enum(['ssrLoad', 'ssrRouterLinkLoad']), - runs: z.number().int().positive(), - iterations: z.array(ssrLoadSweepSchema).nonempty(), - configuration: z.object({ - workerStages: z.array(z.number().int().positive()).nonempty(), - warmupDurationMs: z.number().positive().optional(), - recoveryMs: z.number().nonnegative().optional(), - warmupAttempts: z.number().int().positive().optional(), - stageDurationMs: z.number().positive().optional(), - }), - representativeIterationIndex: z.number().int().nonnegative(), - source: z.literal('autocannon'), - path: z.enum([ - '/server-side-rendered', - '/server-side-rendered-plain-links', - ]), - totalRequests: z.number().int().nonnegative(), - totalErrors: z.number().int().nonnegative(), - stage1: ssrLoadStageSummarySchema, - stage2: ssrLoadStageSummarySchema.optional(), - stage3: ssrLoadStageSummarySchema.optional(), - stage4: ssrLoadStageSummarySchema.optional(), - stage5: ssrLoadStageSummarySchema.optional(), - stage6: ssrLoadStageSummarySchema.optional(), - stage7: ssrLoadStageSummarySchema.optional(), - }) - .refine( - (value) => - value.iterations.length === value.runs && - value.representativeIterationIndex < value.iterations.length && - value.iterations[value.representativeIterationIndex]!.some( - (stage) => stage.errors === 0, - ), - ) - .transform((value) => { - const representativeSweep = - value.iterations[value.representativeIterationIndex]! - const representativePeak = representativeSweep - .filter((stage) => stage.errors === 0) - .reduce((best, stage) => - stage.requestsPerSec > best.requestsPerSec ? stage : best, - ) - return { - ...value, - representativeSweep, - representativePeak, - stages: [ - value.stage1, - value.stage2, - value.stage3, - value.stage4, - value.stage5, - value.stage6, - value.stage7, - ].flatMap((stage) => - stage - ? [ - { - workers: stage.connections, - requests: stage.requests.mean, - errors: stage.errors.mean, - requestsPerSec: stage.requestsPerSec.mean, - avgLatencyMs: stage.avgLatencyMs.mean, - p50LatencyMs: stage.p50LatencyMs.mean, - p75LatencyMs: stage.p75LatencyMs.mean, - p90LatencyMs: stage.p90LatencyMs.mean, - p99LatencyMs: stage.p99LatencyMs.mean, - maxLatencyMs: stage.maxLatencyMs.mean, - bytesPerSec: stage.bytesPerSec.mean, - }, - ] - : [], - ), - } - }) - -const runtimeSchema = z.object({ - name: z.string(), - type: z.string(), - package: z.string(), - isFocused: z.boolean(), - order: z.number(), - ssrRequestThroughputTests: requestThroughputContentSchema, - ssrLoadTests: ssrLoadTestsSchema - .refine((value) => value.name === 'ssrLoad') - .optional(), - ssrRouterLinkLoadTests: ssrLoadTestsSchema - .refine((value) => value.name === 'ssrRouterLinkLoad') - .optional(), - clientSideRenderedTests: z - .union([ - ClientRenderingBenchmarkSchema.transform((value) => ({ - ...value, - firstPaintMs: value.firstPaintMs.mean, - fcpMs: value.fcpMs.mean, - interactionTests: { - ...value.interactionTests, - interactionLatencyMs: - value.interactionTests.interactionLatencyMs.mean, - inputDelayMs: value.interactionTests.inputDelayMs.mean, - processingDurationMs: - value.interactionTests.processingDurationMs.mean, - presentationDelayMs: value.interactionTests.presentationDelayMs.mean, - }, - })), - z.object({ - firstPaintMs: z.number(), - fcpMs: z.number(), - inpMs: z.number().optional(), - interactionTests: z - .object({ - scenario: z.literal('first-row-detail-navigation'), - source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: z.number().positive(), - inputDelayMs: z.number().nonnegative(), - processingDurationMs: z.number().nonnegative(), - presentationDelayMs: z.number().nonnegative(), - }) - .optional(), - runs: z.number(), - standardDeviation: interactionTimingSchema - .extend({ - firstPaintMs: z.number().nonnegative(), - fcpMs: z.number().nonnegative(), - }) - .optional(), - samples: z.array(renderedTestSampleSchema).nonempty().optional(), - }), - ]) - .optional(), - serverSideRenderedTests: z - .union([ - ServerRenderingBenchmarkSchema.transform((value) => ({ - ...value, - firstPaintMs: value.firstPaintMs.mean, - fcpMs: value.fcpMs.mean, - interactionTests: { - ...value.interactionTests, - interactionLatencyMs: - value.interactionTests.interactionLatencyMs.mean, - inputDelayMs: value.interactionTests.inputDelayMs.mean, - processingDurationMs: - value.interactionTests.processingDurationMs.mean, - presentationDelayMs: value.interactionTests.presentationDelayMs.mean, - }, - })), - z.object({ - firstPaintMs: z.number(), - fcpMs: z.number(), - inpMs: z.number().optional(), - interactionTests: z - .object({ - scenario: z.literal('first-row-detail-navigation'), - source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: z.number().positive(), - inputDelayMs: z.number().nonnegative(), - processingDurationMs: z.number().nonnegative(), - presentationDelayMs: z.number().nonnegative(), - }) - .optional(), - runs: z.number(), - standardDeviation: interactionTimingSchema - .extend({ - firstPaintMs: z.number().nonnegative(), - fcpMs: z.number().nonnegative(), - }) - .optional(), - samples: z.array(renderedTestSampleSchema).nonempty().optional(), - }), - ]) - .optional(), - timingMeasuredAt: z.string().optional(), - runner: z.string().optional(), - browserVersion: z.string().optional(), - frameworkVersion: z.string().optional(), - releasedAt: z.string().optional(), -}) - const devtimeCollection = defineCollection({ loader: glob({ pattern: '*.json', base: './src/content/devtime' }), - schema: devtimeSchema, + schema: devtimeContentSchema, }) const devtimeVersionsCollection = defineCollection({ @@ -403,12 +20,12 @@ const devtimeVersionsCollection = defineCollection({ pattern: '**/*.json', base: './src/content/devtime/versions', }), - schema: devtimeSchema, + schema: devtimeContentSchema, }) const runtimeCollection = defineCollection({ loader: glob({ pattern: '*.json', base: './src/content/runtime' }), - schema: runtimeSchema, + schema: runtimeContentSchema, }) const runtimeVersionsCollection = defineCollection({ @@ -416,10 +33,7 @@ const runtimeVersionsCollection = defineCollection({ pattern: '**/*.json', base: './src/content/runtime/versions', }), - schema: runtimeSchema.extend({ - ssrRequestThroughputTests: - runtimeSchema.shape.ssrRequestThroughputTests.optional(), - }), + schema: runtimeVersionsContentSchema, }) const cwvCollection = defineCollection({ diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index 1c01436ed..0f506d037 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -266,7 +266,7 @@ Installed using the CLI with the following setup: response body is not read, because several dev servers stream the page and body time would measure page size rather than startup. - The port is the starter's default dev port, recorded as `devServerPort` in - `.github/frameworks.json`. The starters are not modified, so whatever they + `packages/stats-generator/src/frameworks.ts`. The starters are not modified, so whatever they ship is inside the measurement: Nuxt and TanStack Start enable devtools, and Astro and React Router generate types at boot. - The dev server runs with a minimal environment (`PATH`, `HOME`, `TMPDIR`, diff --git a/packages/docs/src/lib/request-throughput.ts b/packages/docs/src/lib/request-throughput.ts index 7a1f8854b..dd9e2e063 100644 --- a/packages/docs/src/lib/request-throughput.ts +++ b/packages/docs/src/lib/request-throughput.ts @@ -1,59 +1,18 @@ +import { + RequestThroughputBenchmarkSchema, + LegacySSRRequestThroughputSchema, +} from '@framework-tracker/results' import { z } from 'astro/zod' -const metricSchema = z.object({ - mean: z.number(), - median: z.number(), - min: z.number(), - max: z.number(), -}) - -const iterationSchema = z.object({ - opsPerSec: metricSchema, - latencyMs: metricSchema, - samples: z.number().int().positive(), -}) - export const requestThroughputContentSchema = z.union([ - z - .object({ - name: z.literal('ssrRequestThroughput'), - runs: z.number().int().positive(), - iterations: z.array(iterationSchema), - opsPerSec: metricSchema, - latencyMs: metricSchema, - samples: z.number().int().positive(), - bodySizeKb: z.number().positive(), - duplicationFactor: z.number().nonnegative(), - durationMs: z.number().positive(), - warmupDurationMs: z.number().nonnegative(), - minimumSamples: z.number().int().positive(), - warmupMinimumSamples: z.number().int().nonnegative(), - source: z.literal('tinybench'), - path: z.literal('/ssr-throughput'), - }) - .refine((value) => value.runs === value.iterations.length) - .refine( - (value) => - value.samples === - value.iterations.reduce( - (count, iteration) => count + iteration.samples, - 0, - ), - ) - .transform((value) => ({ - ...value, - opsPerSec: value.opsPerSec.mean, - latencyMs: value.latencyMs.mean, - })), + RequestThroughputBenchmarkSchema.transform((value) => ({ + ...value, + opsPerSec: value.opsPerSec.mean, + latencyMs: value.latencyMs.mean, + })), // Current checked-in records have rounded summaries and no recoverable samples. - z - .object({ - opsPerSec: z.number().positive(), - avgLatencyMs: z.number().nonnegative(), - medianLatencyMs: z.number().nonnegative(), - samples: z.number().positive(), - bodySizeKb: z.number().positive(), - duplicationFactor: z.number().nonnegative(), - }) - .transform((value) => ({ ...value, latencyMs: value.avgLatencyMs })), + LegacySSRRequestThroughputSchema.transform((value) => ({ + ...value, + latencyMs: value.avgLatencyMs, + })), ]) diff --git a/packages/docs/src/lib/result-content.ts b/packages/docs/src/lib/result-content.ts new file mode 100644 index 000000000..0f65cbd43 --- /dev/null +++ b/packages/docs/src/lib/result-content.ts @@ -0,0 +1,156 @@ +import { + FrameworkResultSchema, + type FrameworkResult, +} from '@framework-tracker/results' + +function formatTime( + value: NonNullable< + | FrameworkResult['coldBuildTime'] + | FrameworkResult['warmBuildTime'] + | FrameworkResult['devServerStartTime'] + >, +) { + return { + avgMs: value.timeMs.mean, + medianMs: value.timeMs.median, + minMs: value.timeMs.min, + maxMs: value.timeMs.max, + samplesMs: value.iterations.map((run) => run.timeMs), + } +} +function formatRendered( + value: NonNullable< + | FrameworkResult['clientSideRenderedTests'] + | FrameworkResult['serverSideRenderedTests'] + >, +) { + if (!('name' in value)) return value + return { + ...value, + firstPaintMs: value.firstPaintMs.mean, + fcpMs: value.fcpMs.mean, + interactionTests: { + ...value.interactionTests, + interactionLatencyMs: value.interactionTests.interactionLatencyMs.mean, + inputDelayMs: value.interactionTests.inputDelayMs.mean, + processingDurationMs: value.interactionTests.processingDurationMs.mean, + presentationDelayMs: value.interactionTests.presentationDelayMs.mean, + }, + } +} +function formatThroughput( + value: NonNullable, +) { + if ('name' in value) + return { + ...value, + opsPerSec: value.opsPerSec.mean, + latencyMs: value.latencyMs.mean, + } + return { ...value, latencyMs: value.avgLatencyMs } +} +function formatLoad(value: NonNullable) { + const representativeSweep = + value.iterations[value.representativeIterationIndex]! + const representativePeak = representativeSweep + .filter((stage) => stage.errors === 0) + .reduce((best, stage) => + stage.requestsPerSec > best.requestsPerSec ? stage : best, + ) + return { + ...value, + representativeSweep, + representativePeak, + stages: [ + value.stage1, + value.stage2, + value.stage3, + value.stage4, + value.stage5, + value.stage6, + value.stage7, + ].flatMap((stage) => + stage + ? [ + { + workers: stage.connections, + requests: stage.requests.mean, + errors: stage.errors.mean, + requestsPerSec: stage.requestsPerSec.mean, + avgLatencyMs: stage.avgLatencyMs.mean, + p50LatencyMs: stage.p50LatencyMs.mean, + p75LatencyMs: stage.p75LatencyMs.mean, + p90LatencyMs: stage.p90LatencyMs.mean, + p99LatencyMs: stage.p99LatencyMs.mean, + maxLatencyMs: stage.maxLatencyMs.mean, + bytesPerSec: stage.bytesPerSec.mean, + }, + ] + : [], + ), + } +} + +const displayMetadata = { + name: true, + type: true, + package: true, + isFocused: true, + order: true, +} as const + +export const devtimeContentSchema = FrameworkResultSchema.required({ + ...displayMetadata, + framework: true, + packagePath: true, + prodDependencies: true, + devDependencies: true, + allDependencies: true, + firstPartyDependencies: true, + install: true, + coldBuildTime: true, + warmBuildTime: true, + timingMeasuredAt: true, + runner: true, +}).transform((value) => ({ + ...formatRuntime(value), + coldBuildTime: formatTime(value.coldBuildTime), + warmBuildTime: formatTime(value.warmBuildTime), + devServerStartTime: + value.devServerStartTime && formatTime(value.devServerStartTime), +})) + +const runtimeInputSchema = FrameworkResultSchema.required(displayMetadata) +function formatRuntime>( + value: T, +) { + return { + ...value, + coldBuildTime: value.coldBuildTime && formatTime(value.coldBuildTime), + warmBuildTime: value.warmBuildTime && formatTime(value.warmBuildTime), + devServerStartTime: + value.devServerStartTime && formatTime(value.devServerStartTime), + ssrRequestThroughputTests: + value.ssrRequestThroughputTests && + formatThroughput(value.ssrRequestThroughputTests), + ssrLoadTests: value.ssrLoadTests && formatLoad(value.ssrLoadTests), + ssrRouterLinkLoadTests: + value.ssrRouterLinkLoadTests && formatLoad(value.ssrRouterLinkLoadTests), + clientSideRenderedTests: + value.clientSideRenderedTests && + formatRendered(value.clientSideRenderedTests), + serverSideRenderedTests: + value.serverSideRenderedTests && + formatRendered(value.serverSideRenderedTests), + } +} +export const runtimeContentSchema = runtimeInputSchema + .required({ ssrRequestThroughputTests: true }) + .transform((value) => ({ + ...formatRuntime(value), + ssrRequestThroughputTests: formatThroughput( + value.ssrRequestThroughputTests, + ), + })) +export const runtimeVersionsContentSchema = + runtimeInputSchema.transform(formatRuntime) diff --git a/packages/results/package.json b/packages/results/package.json new file mode 100644 index 000000000..9be9a5a00 --- /dev/null +++ b/packages/results/package.json @@ -0,0 +1,13 @@ +{ + "name": "@framework-tracker/results", + "version": "0.0.1", + "private": true, + "type": "module", + "exports": { + ".": "./src/schemas.ts", + "./selection": "./src/selection.ts" + }, + "dependencies": { + "zod": "4.6.7" + } +} diff --git a/packages/results/src/schemas.ts b/packages/results/src/schemas.ts new file mode 100644 index 000000000..d9c64b7e4 --- /dev/null +++ b/packages/results/src/schemas.ts @@ -0,0 +1,516 @@ +import { selectRepresentativeIteration } from './selection.ts' +import { z } from 'zod' + +export const MetricBaseSchema = z.object({ + mean: z.number(), + median: z.number(), + min: z.number(), + max: z.number(), +}) +export type MetricBase = z.infer + +export const FrameworkSchema = z.enum([ + 'baseline-html', + 'astro', + 'next', + 'nuxt', + 'pracht', + 'react-router', + 'solid-start', + 'sveltekit', + 'tanstack', +]) + +export const PackagePathSchema = z.enum([ + 'packages/app-baseline-html', + 'packages/starter-astro', + 'packages/app-astro', + 'packages/starter-next-js', + 'packages/app-next-js', + 'packages/starter-nuxt', + 'packages/app-nuxt', + 'packages/starter-pracht', + 'packages/app-pracht', + 'packages/starter-react-router', + 'packages/app-react-router', + 'packages/starter-solid-start', + 'packages/app-solid-start', + 'packages/starter-sveltekit', + 'packages/app-sveltekit', + 'packages/starter-tanstack-start-react', + 'packages/app-tanstack-start-react', +]) + +export const BaseResultSchema = z.object({ + framework: FrameworkSchema, + packagePath: PackagePathSchema, + timingMeasuredAt: z.iso.datetime(), + runner: z.string().min(1), +}) + +export const TestBaseSchema = z.object({ + /** Stable test identifier, such as install or ssrLoad. */ + name: z.string().min(1), +}) +export type TestBase = z.infer + +/** + * Repeated benchmark result with benchmark-specific raw iterations. + * Use safeExtend() to preserve the iteration-count check when adding fields. + */ +export function createBenchmarkBaseSchema(iteration: T) { + return TestBaseSchema.extend({ + /** Number of measured runs contributing to the metrics, excluding warmups. */ + runs: z.number().int().positive(), + /** Complete raw result for each measured run. */ + iterations: z.array(iteration), + }).refine((stats) => stats.iterations.length === stats.runs, { + message: 'Expected one iteration per run', + path: ['iterations'], + }) +} + +export const InstallIterationSchema = z.object({ + installTimeMs: z.number().nonnegative(), +}) + +export const InstallBenchmarkSchema = createBenchmarkBaseSchema( + InstallIterationSchema, +).safeExtend({ + name: z.literal('install'), + installTimeMs: MetricBaseSchema, + nodeModulesSizeBytes: z.number().nonnegative(), +}) + +/** Artifact emitted by the install runner, before overall result assembly. */ +export const InstallResultSchema = z.object({ + frameworkVersion: z.string().min(1), + install: InstallBenchmarkSchema, +}) + +export const BuildIterationSchema = z.object({ + timeMs: z.number().nonnegative(), +}) +export const BuildBenchmarkSchema = createBenchmarkBaseSchema( + BuildIterationSchema, +).safeExtend({ + name: z.enum(['coldBuild', 'warmBuild']), + timeMs: MetricBaseSchema, +}) +export const BuildStatsSchema = z.object({ + coldBuildTime: BuildBenchmarkSchema.safeExtend({ + name: z.literal('coldBuild'), + }), + warmBuildTime: BuildBenchmarkSchema.safeExtend({ + name: z.literal('warmBuild'), + }), +}) + +export const DevServerIterationSchema = z.object({ + timeMs: z.number().nonnegative(), +}) + +export const DevServerBenchmarkSchema = createBenchmarkBaseSchema( + DevServerIterationSchema, +).safeExtend({ + name: z.literal('devServer'), + timeMs: MetricBaseSchema, +}) + +export const DevServerStatsSchema = z.object({ + devServerStartTime: DevServerBenchmarkSchema, +}) + +export const BrowserBaselineStatsSchema = TestBaseSchema.extend({ + name: z.literal('browserBaseline'), + baselineStatus: z.union([ + z.literal('high'), + z.literal('low'), + z.literal(false), + z.null(), + ]), + baselineYear: z.number().nullable(), + baselineReason: z.string().nullable(), + baselineFeatureCount: z.number().nonnegative(), +}) + +export const NodeEnginesStatsSchema = TestBaseSchema.extend({ + name: z.literal('nodeEngines'), + minimumNodeVersion: z.string().nullable(), + imposedBy: z.array(z.string()), + packagesScanned: z.number().nonnegative(), + packagesDeclaringNodeEngine: z.number().nonnegative(), + unsatisfiableRanges: z.array(z.string()), +}) + +export const LegacySSRRequestThroughputSchema = z.object({ + opsPerSec: z.number().positive(), + avgLatencyMs: z.number().nonnegative(), + medianLatencyMs: z.number().nonnegative(), + samples: z.number().positive(), + bodySizeKb: z.number().positive(), + duplicationFactor: z.number().nonnegative(), +}) + +export const RequestThroughputIterationSchema = z.object({ + opsPerSec: MetricBaseSchema, + latencyMs: MetricBaseSchema, + samples: z.number().int().positive(), +}) +export const RequestThroughputBenchmarkSchema = createBenchmarkBaseSchema( + RequestThroughputIterationSchema, +) + .safeExtend({ + name: z.literal('ssrRequestThroughput'), + opsPerSec: MetricBaseSchema, + latencyMs: MetricBaseSchema, + samples: z.number().int().positive(), + bodySizeKb: z.number().positive(), + duplicationFactor: z.number().nonnegative(), + durationMs: z.number().positive(), + warmupDurationMs: z.number().nonnegative(), + minimumSamples: z.number().int().positive(), + warmupMinimumSamples: z.number().int().nonnegative(), + source: z.literal('tinybench'), + path: z.literal('/ssr-throughput'), + }) + .refine( + (value) => + value.samples === + value.iterations.reduce( + (count, iteration) => count + iteration.samples, + 0, + ), + { + message: 'Expected total HTTP sample count to match iterations', + path: ['samples'], + }, + ) + +export const RequestThroughputResultSchema = z.object({ + ssrRequestThroughputTests: RequestThroughputBenchmarkSchema, + frameworkVersion: z.string().optional(), + timingMeasuredAt: z.string().optional(), + runner: z.string().optional(), + browserVersion: z.string().optional(), +}) + +/** Compatibility for current checked-in summaries that predate raw retention. */ +export const SSRRequestThroughputStatsSchema = + RequestThroughputResultSchema.extend({ + ssrRequestThroughputTests: z.union([ + RequestThroughputBenchmarkSchema, + LegacySSRRequestThroughputSchema, + ]), + }) + +export const SSRLoadStageSchema = z.strictObject({ + workers: z.number().int().positive(), + elapsedTimeMs: z.number().positive(), + requests: z.number().int().nonnegative(), + errors: z.number().int().nonnegative(), + requestsPerSec: z.number().nonnegative(), + avgLatencyMs: z.number().nonnegative(), + p50LatencyMs: z.number().nonnegative(), + p75LatencyMs: z.number().nonnegative(), + p90LatencyMs: z.number().nonnegative(), + p99LatencyMs: z.number().nonnegative(), + maxLatencyMs: z.number().nonnegative(), + bytesPerSec: z.number().nonnegative(), +}) + +export const SSRLoadSweepSchema = z.array(SSRLoadStageSchema).nonempty() + +export const SSRLoadStageSummarySchema = z.object({ + connections: z.number().int().positive(), + requests: MetricBaseSchema, + errors: MetricBaseSchema, + requestsPerSec: MetricBaseSchema, + avgLatencyMs: MetricBaseSchema, + p50LatencyMs: MetricBaseSchema, + p75LatencyMs: MetricBaseSchema, + p90LatencyMs: MetricBaseSchema, + p99LatencyMs: MetricBaseSchema, + maxLatencyMs: MetricBaseSchema, + bytesPerSec: MetricBaseSchema, +}) + +export const SSRLoadTestsSchema = createBenchmarkBaseSchema(SSRLoadSweepSchema) + .safeExtend({ + name: z.enum(['ssrLoad', 'ssrRouterLinkLoad']), + configuration: z.object({ + workerStages: z.array(z.number().int().positive()).nonempty(), + warmupDurationMs: z.number().positive().optional(), + recoveryMs: z.number().nonnegative().optional(), + warmupAttempts: z.number().int().positive().optional(), + stageDurationMs: z.number().positive().optional(), + }), + representativeIterationIndex: z.number().int().nonnegative(), + source: z.literal('autocannon'), + path: z.enum([ + '/server-side-rendered', + '/server-side-rendered-plain-links', + ]), + totalRequests: z.number().int().nonnegative(), + totalErrors: z.number().int().nonnegative(), + stage1: SSRLoadStageSummarySchema, + stage2: SSRLoadStageSummarySchema.optional(), + stage3: SSRLoadStageSummarySchema.optional(), + stage4: SSRLoadStageSummarySchema.optional(), + stage5: SSRLoadStageSummarySchema.optional(), + stage6: SSRLoadStageSummarySchema.optional(), + stage7: SSRLoadStageSummarySchema.optional(), + }) + .strict() + .refine( + (value) => { + const workers = value.iterations[0]?.map((stage) => stage.workers) + return ( + workers !== undefined && + new Set(workers).size === workers.length && + JSON.stringify(value.configuration.workerStages) === + JSON.stringify(workers) && + [ + value.stage1, + value.stage2, + value.stage3, + value.stage4, + value.stage5, + value.stage6, + value.stage7, + ].every((stage, index) => + index < workers.length + ? stage?.connections === workers[index] + : stage === undefined, + ) && + workers.length <= 7 && + value.iterations.every( + (sweep) => + sweep.length === workers.length && + sweep.every((stage, index) => stage.workers === workers[index]), + ) + ) + }, + { message: 'Expected matching worker stages in every sweep' }, + ) + .refine( + (value) => { + try { + return ( + value.representativeIterationIndex === + selectRepresentativeIteration(value.iterations) + ) + } catch { + return false + } + }, + { message: 'Expected the actual median peak throughput sweep' }, + ) + +export const SSRLoadStatsSchema = z.object({ + ssrLoadTests: SSRLoadTestsSchema.refine((value) => value.name === 'ssrLoad'), + frameworkVersion: z.string().optional(), + timingMeasuredAt: z.string().optional(), + runner: z.string().optional(), + browserVersion: z.string().optional(), +}) + +export const SSRRouterLinkLoadStatsSchema = SSRLoadStatsSchema.omit({ + ssrLoadTests: true, +}).extend({ + ssrRouterLinkLoadTests: SSRLoadTestsSchema.refine( + (value) => value.name === 'ssrRouterLinkLoad', + ), +}) + +const InteractionTestsSchema = z.object({ + scenario: z.literal('first-row-detail-navigation'), + source: z.literal('lighthouse-inp-breakdown'), + interactionLatencyMs: z.number().positive(), + inputDelayMs: z.number().nonnegative(), + processingDurationMs: z.number().nonnegative(), + presentationDelayMs: z.number().nonnegative(), +}) + +export const InteractionTimingSchema = z.object({ + interactionLatencyMs: z.number().nonnegative(), + inputDelayMs: z.number().nonnegative(), + processingDurationMs: z.number().nonnegative(), + presentationDelayMs: z.number().nonnegative(), +}) + +export const RenderedTestSampleSchema = z.object({ + firstPaintMs: z.number().positive(), + fcpMs: z.number().positive(), + interactionTests: InteractionTimingSchema, +}) + +export const LegacyRenderedTestsSchema = z + .object({ + firstPaintMs: z.number().positive(), + fcpMs: z.number().positive(), + /** Historical only; new runs record interactionTests instead. */ + inpMs: z.number().optional(), + interactionTests: InteractionTestsSchema.optional(), + runs: z.number().int().positive(), + standardDeviation: InteractionTimingSchema.extend({ + firstPaintMs: z.number().nonnegative(), + fcpMs: z.number().nonnegative(), + }).optional(), + samples: z.array(RenderedTestSampleSchema).nonempty().optional(), + }) + .refine( + (stats) => + stats.inpMs !== undefined || stats.interactionTests !== undefined, + { + message: 'Expected historical inpMs or current interactionTests', + }, + ) + .refine( + (stats) => + stats.samples === undefined || stats.samples.length === stats.runs, + { message: 'Expected one browser sample per run' }, + ) + .refine( + (stats) => + (stats.standardDeviation === undefined) === (stats.samples === undefined), + { message: 'Expected standard deviation and samples together' }, + ) + +export const ClientRenderingIterationSchema = RenderedTestSampleSchema +export const ClientRenderingBenchmarkSchema = createBenchmarkBaseSchema( + ClientRenderingIterationSchema, +).safeExtend({ + name: z.literal('clientSideRendered'), + firstPaintMs: MetricBaseSchema, + fcpMs: MetricBaseSchema, + interactionTests: z.object({ + scenario: z.literal('first-row-detail-navigation'), + source: z.literal('lighthouse-inp-breakdown'), + interactionLatencyMs: MetricBaseSchema, + inputDelayMs: MetricBaseSchema, + processingDurationMs: MetricBaseSchema, + presentationDelayMs: MetricBaseSchema, + }), +}) + +export const ServerRenderingIterationSchema = RenderedTestSampleSchema +export const ServerRenderingBenchmarkSchema = createBenchmarkBaseSchema( + ServerRenderingIterationSchema, +).safeExtend({ + name: z.literal('serverSideRendered'), + firstPaintMs: MetricBaseSchema, + fcpMs: MetricBaseSchema, + interactionTests: z.object({ + scenario: z.literal('first-row-detail-navigation'), + source: z.literal('lighthouse-inp-breakdown'), + interactionLatencyMs: MetricBaseSchema, + inputDelayMs: MetricBaseSchema, + processingDurationMs: MetricBaseSchema, + presentationDelayMs: MetricBaseSchema, + }), +}) + +export const ClientSideRenderedStatsSchema = z.object({ + clientSideRenderedTests: ClientRenderingBenchmarkSchema, + browserVersion: z.string().min(1), +}) + +export const ServerSideRenderedStatsSchema = z.object({ + serverSideRenderedTests: ServerRenderingBenchmarkSchema, + browserVersion: z.string().min(1), +}) + +export const DependencyCountsSchema = z.object({ + prodDependencies: z.number().int().nonnegative(), + devDependencies: z.number().int().nonnegative(), + allDependencies: z.number().int().nonnegative(), +}) +export const DependencyScanSchema = TestBaseSchema.extend( + DependencyCountsSchema.shape, +).extend({ + name: z.literal('dependencies'), + duplicateDependencies: z.number().int().nonnegative().optional(), + depInstallSize: z.number().nonnegative().optional(), +}) +export const FrameworkDependencyScanSchema = TestBaseSchema.extend( + DependencyCountsSchema.shape, +).extend({ + name: z.literal('frameworkDependencies'), +}) +export const FirstPartyDependencyScanSchema = TestBaseSchema.extend( + DependencyCountsSchema.shape, +).extend({ + name: z.literal('firstPartyDependencies'), + duplicateDependencies: z.number().int().nonnegative().optional(), +}) +export const CoreJsStatsSchema = TestBaseSchema.extend({ + name: z.literal('corejs'), + totalVendoredBytes: z.number().int().nonnegative(), + unnecessaryModules: z.array(z.string()), +}) + +export const FrameworkResultSchema = z.object({ + framework: FrameworkSchema.optional(), + packagePath: PackagePathSchema.optional(), + name: z.string().optional(), + type: z.enum(['starter-kit', 'runtime-app']).optional(), + package: z.string().optional(), + isFocused: z.boolean().optional(), + order: z.number().optional(), + timingMeasuredAt: z.string().optional(), + runner: z.string().optional(), + browserVersion: z.string().optional(), + frameworkVersion: z.string().optional(), + releasedAt: z.string().optional(), + install: InstallBenchmarkSchema.optional(), + coldBuildTime: BuildStatsSchema.shape.coldBuildTime.optional(), + warmBuildTime: BuildStatsSchema.shape.warmBuildTime.optional(), + devServerStartTime: DevServerBenchmarkSchema.optional(), + ssrRequestThroughputTests: z + .union([RequestThroughputBenchmarkSchema, LegacySSRRequestThroughputSchema]) + .optional(), + ssrLoadTests: SSRLoadTestsSchema.refine( + (value) => value.name === 'ssrLoad', + ).optional(), + ssrRouterLinkLoadTests: SSRLoadTestsSchema.refine( + (value) => value.name === 'ssrRouterLinkLoad', + ).optional(), + clientSideRenderedTests: z + .union([ClientRenderingBenchmarkSchema, LegacyRenderedTestsSchema]) + .optional(), + serverSideRenderedTests: z + .union([ServerRenderingBenchmarkSchema, LegacyRenderedTestsSchema]) + .optional(), + corejs: CoreJsStatsSchema.optional(), + nodeEngines: NodeEnginesStatsSchema.partial() + .required({ name: true, minimumNodeVersion: true, imposedBy: true }) + .optional(), + dependencyScan: DependencyScanSchema.optional(), + prodDependencies: z.number().nonnegative().optional(), + devDependencies: z.number().nonnegative().optional(), + allDependencies: z.number().nonnegative().optional(), + frameworkDependencies: DependencyCountsSchema.extend({ + name: z.literal('frameworkDependencies').optional(), + }).optional(), + firstPartyDependencies: FirstPartyDependencyScanSchema.extend({ + name: z.literal('firstPartyDependencies').optional(), + }).optional(), + duplicateDependencies: z.number().nonnegative().optional(), + depInstallSize: z.number().nonnegative().optional(), + vendoredCoreJsSize: z.number().nonnegative().optional(), + vendoredCoreJsUnnecessaryModules: z.array(z.string()).optional(), + browserBaselineTests: BrowserBaselineStatsSchema.extend({ + name: z.literal('browserBaseline').optional(), + }).optional(), + minimumNodeVersion: z.string().optional(), + minimumNodeVersionImposedBy: z.array(z.string()).optional(), + testTimeMs: z.number().optional(), + packageJson: z + .object({ + dependencies: z.record(z.string(), z.string()).optional(), + devDependencies: z.record(z.string(), z.string()).optional(), + }) + .optional(), +}) +export type FrameworkResult = z.infer diff --git a/packages/results/src/selection.ts b/packages/results/src/selection.ts new file mode 100644 index 000000000..806ab0883 --- /dev/null +++ b/packages/results/src/selection.ts @@ -0,0 +1,36 @@ +interface PeakCandidate { + errors: number + requestsPerSec: number +} + +export function getLoadPeakStage( + stages: readonly T[], +): T { + const peak = stages.reduce( + (best, stage) => + stage.errors === 0 && + (best === undefined || stage.requestsPerSec > best.requestsPerSec) + ? stage + : best, + undefined, + ) + if (peak === undefined) + throw new Error( + 'No valid SSR load peak: no stage completed with zero errors.', + ) + return peak +} + +export function selectRepresentativeIteration( + iterations: readonly (readonly PeakCandidate[])[], +): number { + if (iterations.length === 0 || iterations.length % 2 === 0) { + throw new Error('Expected an odd, nonzero number of load sweeps.') + } + return iterations + .map((sweep, index) => ({ + index, + peak: getLoadPeakStage(sweep).requestsPerSec, + })) + .sort((a, b) => a.peak - b.peak)[Math.floor(iterations.length / 2)]!.index +} diff --git a/packages/stats-generator/package.json b/packages/stats-generator/package.json index 0c62f7572..ce76f421e 100644 --- a/packages/stats-generator/package.json +++ b/packages/stats-generator/package.json @@ -42,7 +42,8 @@ "tinyglobby": "^0.2.17", "verkit": "^0.4.0", "yaml": "2.9.0", - "zod": "^4.4.3" + "zod": "4.6.7", + "@framework-tracker/results": "workspace:*" }, "devDependencies": { "@types/autocannon": "^7.12.7", diff --git a/packages/stats-generator/src/collect-stats.ts b/packages/stats-generator/src/collect-stats.ts index 7559077a6..80ce250b7 100644 --- a/packages/stats-generator/src/collect-stats.ts +++ b/packages/stats-generator/src/collect-stats.ts @@ -38,7 +38,7 @@ async function processStarter(framework: FrameworkConfig, order: number) { const ciStats = (await getCIStats(pkgDir)) ?? {} - const hasDependencies = measurements.some((m) => m.type === 'dependencies') + const hasDependencies = measurements.includes('dependencies') const dependencyStats = hasDependencies && (ciStats.prodDependencies === undefined || @@ -46,9 +46,7 @@ async function processStarter(framework: FrameworkConfig, order: number) { ciStats.allDependencies === undefined) ? getDependencyCountsFromPackageMetadata(pkgDir) : {} - const hasBrowserBaseline = measurements.some( - (m) => m.type === 'browserBaseline', - ) + const hasBrowserBaseline = measurements.includes('browserBaseline') const browserBaselineStats = hasBrowserBaseline && ciStats.browserBaselineTests === undefined ? await getBrowserBaselineStats(pkgDir) diff --git a/packages/stats-generator/src/final-result.test.ts b/packages/stats-generator/src/final-result.test.ts new file mode 100644 index 000000000..4b67c4353 --- /dev/null +++ b/packages/stats-generator/src/final-result.test.ts @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict' +import { globSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { test } from 'node:test' +import { FrameworkResultSchema } from '@framework-tracker/results' +import { packagesDir } from './constants.ts' + +test('shared final contract accepts every persisted result and history', () => { + const files = [ + ...globSync( + [ + '*/ci-stats.json', + '*/stats/*.json', + 'docs/src/content/devtime/**/*.json', + 'docs/src/content/runtime/**/*.json', + ], + { cwd: packagesDir }, + ), + ] + assert.ok(files.length > 0) + for (const file of files) { + const result = FrameworkResultSchema.safeParse( + JSON.parse(readFileSync(join(packagesDir, file), 'utf8')), + ) + assert.ok(result.success, `${file}: ${result.error?.message}`) + } +}) + +test('final contract validates nested benchmarks and test identity', () => { + assert.equal( + FrameworkResultSchema.safeParse({ + install: { + name: 'install', + runs: 2, + iterations: [{ installTimeMs: 10 }], + installTimeMs: { mean: 10, median: 10, min: 10, max: 10 }, + nodeModulesSizeBytes: 100, + }, + }).success, + false, + ) + assert.equal( + FrameworkResultSchema.safeParse({ + ssrRouterLinkLoadTests: { name: 'ssrLoad' }, + }).success, + false, + ) +}) diff --git a/packages/stats-generator/src/frameworks.ts b/packages/stats-generator/src/frameworks.ts new file mode 100644 index 000000000..f88fe5268 --- /dev/null +++ b/packages/stats-generator/src/frameworks.ts @@ -0,0 +1,261 @@ +import type { FrameworkConfig } from './types.ts' + +export const frameworks = [ + { + name: 'baseline-html', + displayName: 'Baseline HTML', + frameworkPackage: 'node', + focusedFramework: false, + app: { + package: 'app-baseline-html', + buildScript: 'build', + buildOutputDir: '.', + measurements: ['ssrRequestThroughput', 'ssrLoad'], + }, + }, + { + name: 'astro', + displayName: 'Astro', + frameworkPackage: 'astro', + focusedFramework: true, + starter: { + package: 'starter-astro', + buildScript: 'build', + buildOutputDir: 'dist', + devServerPort: 4321, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-astro', + buildScript: 'build', + buildOutputDir: 'dist', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + ], + }, + }, + { + name: 'next', + displayName: 'Next.js', + frameworkPackage: 'next', + focusedFramework: true, + starter: { + package: 'starter-next-js', + buildScript: 'build', + buildOutputDir: '.next', + devServerPort: 3000, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-next-js', + buildScript: 'build', + buildOutputDir: '.next', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + 'ssrRouterLinkLoad', + ], + }, + }, + { + name: 'nuxt', + displayName: 'Nuxt', + frameworkPackage: 'nuxt', + focusedFramework: true, + starter: { + package: 'starter-nuxt', + buildScript: 'build', + buildOutputDir: '.output', + devServerPort: 3000, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-nuxt', + buildScript: 'build', + buildOutputDir: '.output', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + 'ssrRouterLinkLoad', + ], + }, + }, + { + name: 'pracht', + displayName: 'Pracht', + frameworkPackage: '@pracht/core', + focusedFramework: false, + starter: { + package: 'starter-pracht', + buildScript: 'build', + buildOutputDir: 'dist', + devServerPort: 3000, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-pracht', + buildScript: 'build', + buildOutputDir: 'dist', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + 'ssrRouterLinkLoad', + ], + }, + }, + { + name: 'react-router', + displayName: 'React Router', + frameworkPackage: '@react-router/dev', + focusedFramework: true, + starter: { + package: 'starter-react-router', + buildScript: 'build', + buildOutputDir: 'build', + devServerPort: 5173, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-react-router', + buildScript: 'build', + buildOutputDir: 'build', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + 'ssrRouterLinkLoad', + ], + }, + }, + { + name: 'solid-start', + displayName: 'SolidStart', + frameworkPackage: '@solidjs/start', + focusedFramework: true, + starter: { + package: 'starter-solid-start', + buildScript: 'build', + buildOutputDir: '.output', + devServerPort: 3000, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-solid-start', + buildScript: 'build', + buildOutputDir: '.output', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + ], + }, + }, + { + name: 'sveltekit', + displayName: 'SvelteKit', + frameworkPackage: '@sveltejs/kit', + focusedFramework: true, + starter: { + package: 'starter-sveltekit', + buildScript: 'build', + buildOutputDir: 'build', + devServerPort: 5173, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-sveltekit', + buildScript: 'build', + buildOutputDir: 'build', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + ], + }, + }, + { + name: 'tanstack', + displayName: 'TanStack Start', + frameworkPackage: '@tanstack/react-start', + focusedFramework: true, + starter: { + package: 'starter-tanstack-start-react', + buildScript: 'build', + buildOutputDir: '.output', + devServerPort: 3000, + measurements: [ + 'install', + 'build', + 'devServer', + 'dependencies', + 'browserBaseline', + ], + }, + app: { + package: 'app-tanstack-start-react', + buildScript: 'build', + buildOutputDir: '.output', + measurements: [ + 'ssrRequestThroughput', + 'clientSideRendered', + 'serverSideRendered', + 'ssrLoad', + 'ssrRouterLinkLoad', + ], + }, + }, +] satisfies FrameworkConfig[] diff --git a/packages/stats-generator/src/get-frameworks.ts b/packages/stats-generator/src/get-frameworks.ts index 0fbea3d75..18425bffa 100644 --- a/packages/stats-generator/src/get-frameworks.ts +++ b/packages/stats-generator/src/get-frameworks.ts @@ -1,10 +1,6 @@ -import { readFile } from 'node:fs/promises' -import { join } from 'node:path' -import { packagesDir } from './constants.ts' +import { frameworks } from './frameworks.ts' import type { FrameworkConfig } from './types.ts' export async function getFrameworks(): Promise { - const configPath = join(packagesDir, '..', '.github', 'frameworks.json') - const content = await readFile(configPath, 'utf-8') - return JSON.parse(content) as FrameworkConfig[] + return frameworks } diff --git a/packages/stats-generator/src/measurements.ts b/packages/stats-generator/src/measurements.ts new file mode 100644 index 000000000..4304d1b9b --- /dev/null +++ b/packages/stats-generator/src/measurements.ts @@ -0,0 +1,16 @@ +/** Shared defaults for every framework. Scans run once; benchmarks repeat. */ +export const measurements = { + install: { runs: 5 }, + build: { runs: 5 }, + devServer: { runs: 5 }, + dependencies: {}, + browserBaseline: {}, + nodeEngines: {}, + ssrRequestThroughput: { runs: 5 }, + ssrLoad: { runs: 3 }, + ssrRouterLinkLoad: { runs: 3 }, + clientSideRendered: { runs: 5 }, + serverSideRendered: { runs: 5 }, +} satisfies Record + +export type MeasurementName = keyof typeof measurements diff --git a/packages/stats-generator/src/print-frameworks.ts b/packages/stats-generator/src/print-frameworks.ts new file mode 100644 index 000000000..2a2148c0a --- /dev/null +++ b/packages/stats-generator/src/print-frameworks.ts @@ -0,0 +1,3 @@ +import { frameworks } from './frameworks.ts' + +process.stdout.write(JSON.stringify(frameworks)) diff --git a/packages/stats-generator/src/run-build-benchmark.ts b/packages/stats-generator/src/run-build-benchmark.ts index c9d4125e5..9df1f9570 100644 --- a/packages/stats-generator/src/run-build-benchmark.ts +++ b/packages/stats-generator/src/run-build-benchmark.ts @@ -1,3 +1,4 @@ +import { measurements } from './measurements.ts' import { execFileSync } from 'node:child_process' import { cpSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' @@ -49,7 +50,7 @@ async function main() { 'Usage: run-build-benchmark [run-frequency]\nExample: run-build-benchmark starter-astro 5', ) - const runFrequency = parseRunFrequency(args[0]) + const runFrequency = parseRunFrequency(args[0], measurements.build.runs) const { framework, testConfig } = await getFrameworkByPackage(packageName) diff --git a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts index 63883e884..bdfdd6aa7 100644 --- a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts @@ -1,3 +1,5 @@ +import { parseRunFrequency } from './benchmark-utils.ts' +import { measurements } from './measurements.ts' import { ClientSideRenderedStatsSchema } from './schemas.ts' import { join } from 'node:path' import { runClientSideRenderedBenchmark } from './clientSideRendered/index.ts' @@ -16,7 +18,7 @@ async function main() { const { packageName, args } = parseArgs( 'Usage: run-client-side-rendered-benchmark [runs]\nExample: run-client-side-rendered-benchmark app-astro 5', ) - const runs = args[0] ? Number.parseInt(args[0], 10) : 5 + const runs = parseRunFrequency(args[0], measurements.clientSideRendered.runs) console.info( `Running client-side rendered benchmark for ${packageName} (${runs} runs)...\n`, diff --git a/packages/stats-generator/src/run-dev-server-benchmark.ts b/packages/stats-generator/src/run-dev-server-benchmark.ts index 2d0423182..b56932d7b 100644 --- a/packages/stats-generator/src/run-dev-server-benchmark.ts +++ b/packages/stats-generator/src/run-dev-server-benchmark.ts @@ -1,3 +1,4 @@ +import { measurements } from './measurements.ts' import { rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -97,13 +98,13 @@ async function main() { 'Usage: run-dev-server-benchmark [run-frequency]\nExample: run-dev-server-benchmark starter-astro 5', ) - const runFrequency = parseRunFrequency(args[0]) + const runFrequency = parseRunFrequency(args[0], measurements.devServer.runs) const { framework, testConfig } = await getFrameworkByPackage(packageName) const port = testConfig.devServerPort if (port === undefined) { throw new Error( - `${packageName} has no devServerPort in .github/frameworks.json`, + `${packageName} has no devServerPort in packages/stats-generator/src/frameworks.ts`, ) } diff --git a/packages/stats-generator/src/run-install-benchmark.ts b/packages/stats-generator/src/run-install-benchmark.ts index e01047c7e..f5d62296e 100644 --- a/packages/stats-generator/src/run-install-benchmark.ts +++ b/packages/stats-generator/src/run-install-benchmark.ts @@ -1,3 +1,4 @@ +import { measurements } from './measurements.ts' import { summarizeMetric } from './sample-statistics.ts' import { execSync } from 'node:child_process' import { cpSync, mkdirSync, rmSync } from 'node:fs' @@ -65,7 +66,7 @@ async function main() { 'Usage: run-install-benchmark [run-frequency]\nExample: run-install-benchmark starter-astro 5', ) - const runFrequency = parseRunFrequency(args[0]) + const runFrequency = parseRunFrequency(args[0], measurements.install.runs) const { framework } = await getFrameworkByPackage(packageName) diff --git a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts index 77f639b5b..62db9d167 100644 --- a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts @@ -1,3 +1,5 @@ +import { parseRunFrequency } from './benchmark-utils.ts' +import { measurements } from './measurements.ts' import { ServerSideRenderedStatsSchema } from './schemas.ts' import { join } from 'node:path' import { runServerSideRenderedBenchmark } from './serverSideRendered/index.ts' @@ -16,7 +18,7 @@ async function main() { const { packageName, args } = parseArgs( 'Usage: run-server-side-rendered-benchmark [runs]\nExample: run-server-side-rendered-benchmark app-astro 5', ) - const runs = args[0] ? Number.parseInt(args[0], 10) : 5 + const runs = parseRunFrequency(args[0], measurements.serverSideRendered.runs) console.info( `Running server side rendered benchmark for ${packageName} (${runs} runs)...\n`, diff --git a/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts b/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts index 2f94e7526..bdf645cf6 100644 --- a/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts +++ b/packages/stats-generator/src/run-ssr-request-throughput-benchmark.ts @@ -1,3 +1,4 @@ +import { measurements } from './measurements.ts' import { parseRunFrequency } from './benchmark-utils.ts' import { RequestThroughputResultSchema } from './schemas.ts' import { join } from 'node:path' @@ -21,7 +22,10 @@ async function main() { 'Usage: run-ssr-request-throughput-benchmark [runs]\nExample: run-ssr-request-throughput-benchmark app-astro 5', ) - const runs = parseRunFrequency(args[0]) + const runs = parseRunFrequency( + args[0], + measurements.ssrRequestThroughput.runs, + ) console.info( `Running SSR request throughput benchmark for ${packageName} (${runs} runs)...\n`, diff --git a/packages/stats-generator/src/save-ci-stats.ts b/packages/stats-generator/src/save-ci-stats.ts index 3dce72a46..796ef9646 100644 --- a/packages/stats-generator/src/save-ci-stats.ts +++ b/packages/stats-generator/src/save-ci-stats.ts @@ -1,3 +1,4 @@ +import { FrameworkResultSchema } from '@framework-tracker/results' import { validateSSRLoadForPersistence } from './ssrLoad/persistence.ts' import { DependencyScanSchema, @@ -332,6 +333,7 @@ async function main() { const ciStatsPath = join(packagesDir, packageName, 'ci-stats.json') validateSSRLoadForPersistence(stats) validateRequestThroughputForPersistence(stats) + FrameworkResultSchema.parse(stats) writeJsonFile(ciStatsPath, stats) console.info(` ✓ Saved ${ciStatsPath}`) @@ -522,6 +524,7 @@ async function main() { const ciStatsPath = join(packagesDir, packageName, 'ci-stats.json') validateSSRLoadForPersistence(stats) validateRequestThroughputForPersistence(stats) + FrameworkResultSchema.parse(stats) writeJsonFile(ciStatsPath, stats) console.info(` ✓ Saved ${ciStatsPath}`) diff --git a/packages/stats-generator/src/save-stats.ts b/packages/stats-generator/src/save-stats.ts index c8738187a..76c3fdefa 100644 --- a/packages/stats-generator/src/save-stats.ts +++ b/packages/stats-generator/src/save-stats.ts @@ -1,3 +1,4 @@ +import { FrameworkResultSchema } from '@framework-tracker/results' import { validateSSRLoadForPersistence } from './ssrLoad/persistence.ts' import { validateRequestThroughputForPersistence } from './request-throughput.ts' import { access, mkdir, readFile, readdir, writeFile } from 'node:fs/promises' @@ -64,6 +65,7 @@ async function saveVersionedStats( ) validateSSRLoadForPersistence(fileStats) validateRequestThroughputForPersistence(fileStats) + FrameworkResultSchema.parse(fileStats) const outputPath = join(outputDir, fileName) const versionedStats = { ...metadata, @@ -122,6 +124,7 @@ export async function saveStats( validateRequestThroughputForPersistence(mergedStats) const docsStats = getDocsStats(mergedStats) + FrameworkResultSchema.parse(docsStats) await writeFile(filePath, `${JSON.stringify(docsStats, null, 2)}\n`, 'utf-8') await saveVersionedStats(packageName, docsStats, collection) diff --git a/packages/stats-generator/src/schemas.ts b/packages/stats-generator/src/schemas.ts index 3280ea28b..594533584 100644 --- a/packages/stats-generator/src/schemas.ts +++ b/packages/stats-generator/src/schemas.ts @@ -1,461 +1 @@ -import { selectRepresentativeIteration } from './ssrLoad/selection.ts' -import { z } from 'zod' - -/** Summary of one metric; units belong to the metric being measured. */ -export const MetricBaseSchema = z.object({ - /** Arithmetic mean of the measured values. */ - mean: z.number(), - median: z.number(), - /** Lowest measured value. */ - min: z.number(), - /** Highest measured value. */ - max: z.number(), -}) -export type MetricBase = z.infer - -/** Framework identifiers configured in .github/frameworks.json. */ -export const FrameworkSchema = z.enum([ - 'baseline-html', - 'astro', - 'next', - 'nuxt', - 'pracht', - 'react-router', - 'solid-start', - 'sveltekit', - 'tanstack', -]) - -/** Tested package folders configured in .github/frameworks.json. */ -export const PackagePathSchema = z.enum([ - 'packages/app-baseline-html', - 'packages/starter-astro', - 'packages/app-astro', - 'packages/starter-next-js', - 'packages/app-next-js', - 'packages/starter-nuxt', - 'packages/app-nuxt', - 'packages/starter-pracht', - 'packages/app-pracht', - 'packages/starter-react-router', - 'packages/app-react-router', - 'packages/starter-solid-start', - 'packages/app-solid-start', - 'packages/starter-sveltekit', - 'packages/app-sveltekit', - 'packages/starter-tanstack-start-react', - 'packages/app-tanstack-start-react', -]) - -/** Shared metadata for the overall result document. */ -export const BaseResultSchema = z.object({ - framework: FrameworkSchema, - packagePath: PackagePathSchema, - timingMeasuredAt: z.iso.datetime(), - runner: z.string().min(1), -}) - -/** Identity shared by benchmarks and exact-value scans. */ -export const TestBaseSchema = z.object({ - /** Stable test identifier, such as install or ssrLoad. */ - name: z.string().min(1), -}) -export type TestBase = z.infer - -/** - * Repeated benchmark result with benchmark-specific raw iterations. - * Use safeExtend() to preserve the iteration-count check when adding fields. - */ -export function createBenchmarkBaseSchema(iteration: T) { - return TestBaseSchema.extend({ - /** Number of measured runs contributing to the metrics, excluding warmups. */ - runs: z.number().int().positive(), - /** Complete raw result for each measured run. */ - iterations: z.array(iteration), - }).refine((stats) => stats.iterations.length === stats.runs, { - message: 'Expected one iteration per run', - path: ['iterations'], - }) -} - -export const InstallIterationSchema = z.object({ - installTimeMs: z.number().nonnegative(), -}) - -export const InstallBenchmarkSchema = createBenchmarkBaseSchema( - InstallIterationSchema, -).safeExtend({ - name: z.literal('install'), - installTimeMs: MetricBaseSchema, - nodeModulesSizeBytes: z.number().nonnegative(), -}) - -/** Artifact emitted by the install runner, before overall result assembly. */ -export const InstallResultSchema = z.object({ - frameworkVersion: z.string().min(1), - install: InstallBenchmarkSchema, -}) - -export const BuildIterationSchema = z.object({ - timeMs: z.number().nonnegative(), -}) -export const BuildBenchmarkSchema = createBenchmarkBaseSchema( - BuildIterationSchema, -).safeExtend({ - name: z.enum(['coldBuild', 'warmBuild']), - timeMs: MetricBaseSchema, -}) -export const BuildStatsSchema = z.object({ - coldBuildTime: BuildBenchmarkSchema.safeExtend({ - name: z.literal('coldBuild'), - }), - warmBuildTime: BuildBenchmarkSchema.safeExtend({ - name: z.literal('warmBuild'), - }), -}) - -export const DevServerIterationSchema = z.object({ - timeMs: z.number().nonnegative(), -}) - -export const DevServerBenchmarkSchema = createBenchmarkBaseSchema( - DevServerIterationSchema, -).safeExtend({ - name: z.literal('devServer'), - timeMs: MetricBaseSchema, -}) - -export const DevServerStatsSchema = z.object({ - devServerStartTime: DevServerBenchmarkSchema, -}) - -export const BrowserBaselineStatsSchema = TestBaseSchema.extend({ - name: z.literal('browserBaseline'), - baselineStatus: z.union([ - z.literal('high'), - z.literal('low'), - z.literal(false), - z.null(), - ]), - baselineYear: z.number().nullable(), - baselineReason: z.string().nullable(), - baselineFeatureCount: z.number().nonnegative(), -}) - -export const NodeEnginesStatsSchema = TestBaseSchema.extend({ - name: z.literal('nodeEngines'), - minimumNodeVersion: z.string().nullable(), - imposedBy: z.array(z.string()), - packagesScanned: z.number().nonnegative(), - packagesDeclaringNodeEngine: z.number().nonnegative(), - unsatisfiableRanges: z.array(z.string()), -}) - -export const LegacySSRRequestThroughputSchema = z.object({ - opsPerSec: z.number().positive(), - avgLatencyMs: z.number().nonnegative(), - medianLatencyMs: z.number().nonnegative(), - samples: z.number().positive(), - bodySizeKb: z.number().positive(), - duplicationFactor: z.number().nonnegative(), -}) - -export const RequestThroughputIterationSchema = z.object({ - opsPerSec: MetricBaseSchema, - latencyMs: MetricBaseSchema, - samples: z.number().int().positive(), -}) -export const RequestThroughputBenchmarkSchema = createBenchmarkBaseSchema( - RequestThroughputIterationSchema, -) - .safeExtend({ - name: z.literal('ssrRequestThroughput'), - opsPerSec: MetricBaseSchema, - latencyMs: MetricBaseSchema, - samples: z.number().int().positive(), - bodySizeKb: z.number().positive(), - duplicationFactor: z.number().nonnegative(), - durationMs: z.number().positive(), - warmupDurationMs: z.number().nonnegative(), - minimumSamples: z.number().int().positive(), - warmupMinimumSamples: z.number().int().nonnegative(), - source: z.literal('tinybench'), - path: z.literal('/ssr-throughput'), - }) - .refine( - (value) => - value.samples === - value.iterations.reduce( - (count, iteration) => count + iteration.samples, - 0, - ), - { - message: 'Expected total HTTP sample count to match iterations', - path: ['samples'], - }, - ) - -export const RequestThroughputResultSchema = z.object({ - ssrRequestThroughputTests: RequestThroughputBenchmarkSchema, - frameworkVersion: z.string().optional(), - timingMeasuredAt: z.string().optional(), - runner: z.string().optional(), - browserVersion: z.string().optional(), -}) - -/** Compatibility for current checked-in summaries that predate raw retention. */ -export const SSRRequestThroughputStatsSchema = - RequestThroughputResultSchema.extend({ - ssrRequestThroughputTests: z.union([ - RequestThroughputBenchmarkSchema, - LegacySSRRequestThroughputSchema, - ]), - }) - -export const SSRLoadStageSchema = z - .object({ - workers: z.number().int().positive(), - elapsedTimeMs: z.number().positive(), - requests: z.number().int().nonnegative(), - errors: z.number().int().nonnegative(), - requestsPerSec: z.number().nonnegative(), - avgLatencyMs: z.number().nonnegative(), - p50LatencyMs: z.number().nonnegative(), - p75LatencyMs: z.number().nonnegative(), - p90LatencyMs: z.number().nonnegative(), - p99LatencyMs: z.number().nonnegative(), - maxLatencyMs: z.number().nonnegative(), - bytesPerSec: z.number().nonnegative(), - }) - .strict() - -export const SSRLoadSweepSchema = z.array(SSRLoadStageSchema).nonempty() - -export const SSRLoadStageSummarySchema = z.object({ - connections: z.number().int().positive(), - requests: MetricBaseSchema, - errors: MetricBaseSchema, - requestsPerSec: MetricBaseSchema, - avgLatencyMs: MetricBaseSchema, - p50LatencyMs: MetricBaseSchema, - p75LatencyMs: MetricBaseSchema, - p90LatencyMs: MetricBaseSchema, - p99LatencyMs: MetricBaseSchema, - maxLatencyMs: MetricBaseSchema, - bytesPerSec: MetricBaseSchema, -}) - -export const SSRLoadTestsSchema = createBenchmarkBaseSchema(SSRLoadSweepSchema) - .safeExtend({ - name: z.enum(['ssrLoad', 'ssrRouterLinkLoad']), - configuration: z.object({ - workerStages: z.array(z.number().int().positive()).nonempty(), - warmupDurationMs: z.number().positive().optional(), - recoveryMs: z.number().nonnegative().optional(), - warmupAttempts: z.number().int().positive().optional(), - stageDurationMs: z.number().positive().optional(), - }), - representativeIterationIndex: z.number().int().nonnegative(), - source: z.literal('autocannon'), - path: z.enum([ - '/server-side-rendered', - '/server-side-rendered-plain-links', - ]), - totalRequests: z.number().int().nonnegative(), - totalErrors: z.number().int().nonnegative(), - stage1: SSRLoadStageSummarySchema, - stage2: SSRLoadStageSummarySchema.optional(), - stage3: SSRLoadStageSummarySchema.optional(), - stage4: SSRLoadStageSummarySchema.optional(), - stage5: SSRLoadStageSummarySchema.optional(), - stage6: SSRLoadStageSummarySchema.optional(), - stage7: SSRLoadStageSummarySchema.optional(), - }) - .strict() - .refine( - (value) => { - const workers = value.iterations[0]?.map((stage) => stage.workers) - return ( - workers !== undefined && - new Set(workers).size === workers.length && - JSON.stringify(value.configuration.workerStages) === - JSON.stringify(workers) && - [ - value.stage1, - value.stage2, - value.stage3, - value.stage4, - value.stage5, - value.stage6, - value.stage7, - ].every((stage, index) => - index < workers.length - ? stage?.connections === workers[index] - : stage === undefined, - ) && - workers.length <= 7 && - value.iterations.every( - (sweep) => - sweep.length === workers.length && - sweep.every((stage, index) => stage.workers === workers[index]), - ) - ) - }, - { message: 'Expected matching worker stages in every sweep' }, - ) - .refine( - (value) => { - try { - return ( - value.representativeIterationIndex === - selectRepresentativeIteration(value.iterations) - ) - } catch { - return false - } - }, - { message: 'Expected the actual median peak throughput sweep' }, - ) - -export const SSRLoadStatsSchema = z.object({ - ssrLoadTests: SSRLoadTestsSchema.refine((value) => value.name === 'ssrLoad'), - frameworkVersion: z.string().optional(), - timingMeasuredAt: z.string().optional(), - runner: z.string().optional(), - browserVersion: z.string().optional(), -}) - -export const SSRRouterLinkLoadStatsSchema = SSRLoadStatsSchema.omit({ - ssrLoadTests: true, -}).extend({ - ssrRouterLinkLoadTests: SSRLoadTestsSchema.refine( - (value) => value.name === 'ssrRouterLinkLoad', - ), -}) - -const InteractionTestsSchema = z.object({ - scenario: z.literal('first-row-detail-navigation'), - source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: z.number().positive(), - inputDelayMs: z.number().nonnegative(), - processingDurationMs: z.number().nonnegative(), - presentationDelayMs: z.number().nonnegative(), -}) - -const InteractionTimingSchema = z.object({ - interactionLatencyMs: z.number().nonnegative(), - inputDelayMs: z.number().nonnegative(), - processingDurationMs: z.number().nonnegative(), - presentationDelayMs: z.number().nonnegative(), -}) - -const RenderedTestSampleSchema = z.object({ - firstPaintMs: z.number().positive(), - fcpMs: z.number().positive(), - interactionTests: InteractionTimingSchema, -}) - -export const LegacyRenderedTestsSchema = z - .object({ - firstPaintMs: z.number().positive(), - fcpMs: z.number().positive(), - /** Historical only; new runs record interactionTests instead. */ - inpMs: z.number().optional(), - interactionTests: InteractionTestsSchema.optional(), - runs: z.number().int().positive(), - standardDeviation: InteractionTimingSchema.extend({ - firstPaintMs: z.number().nonnegative(), - fcpMs: z.number().nonnegative(), - }).optional(), - samples: z.array(RenderedTestSampleSchema).nonempty().optional(), - }) - .refine( - (stats) => - stats.inpMs !== undefined || stats.interactionTests !== undefined, - { - message: 'Expected historical inpMs or current interactionTests', - }, - ) - .refine( - (stats) => - stats.samples === undefined || stats.samples.length === stats.runs, - { message: 'Expected one browser sample per run' }, - ) - .refine( - (stats) => - (stats.standardDeviation === undefined) === (stats.samples === undefined), - { message: 'Expected standard deviation and samples together' }, - ) - -export const ClientRenderingIterationSchema = RenderedTestSampleSchema -export const ClientRenderingBenchmarkSchema = createBenchmarkBaseSchema( - ClientRenderingIterationSchema, -).safeExtend({ - name: z.literal('clientSideRendered'), - firstPaintMs: MetricBaseSchema, - fcpMs: MetricBaseSchema, - interactionTests: z.object({ - scenario: z.literal('first-row-detail-navigation'), - source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: MetricBaseSchema, - inputDelayMs: MetricBaseSchema, - processingDurationMs: MetricBaseSchema, - presentationDelayMs: MetricBaseSchema, - }), -}) - -export const ServerRenderingIterationSchema = RenderedTestSampleSchema -export const ServerRenderingBenchmarkSchema = createBenchmarkBaseSchema( - ServerRenderingIterationSchema, -).safeExtend({ - name: z.literal('serverSideRendered'), - firstPaintMs: MetricBaseSchema, - fcpMs: MetricBaseSchema, - interactionTests: z.object({ - scenario: z.literal('first-row-detail-navigation'), - source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: MetricBaseSchema, - inputDelayMs: MetricBaseSchema, - processingDurationMs: MetricBaseSchema, - presentationDelayMs: MetricBaseSchema, - }), -}) - -export const ClientSideRenderedStatsSchema = z.object({ - clientSideRenderedTests: ClientRenderingBenchmarkSchema, - browserVersion: z.string().min(1), -}) - -export const ServerSideRenderedStatsSchema = z.object({ - serverSideRenderedTests: ServerRenderingBenchmarkSchema, - browserVersion: z.string().min(1), -}) - -export const DependencyCountsSchema = z.object({ - prodDependencies: z.number().int().nonnegative(), - devDependencies: z.number().int().nonnegative(), - allDependencies: z.number().int().nonnegative(), -}) -export const DependencyScanSchema = TestBaseSchema.extend( - DependencyCountsSchema.shape, -).extend({ - name: z.literal('dependencies'), - duplicateDependencies: z.number().int().nonnegative().optional(), - depInstallSize: z.number().nonnegative().optional(), -}) -export const FrameworkDependencyScanSchema = TestBaseSchema.extend( - DependencyCountsSchema.shape, -).extend({ - name: z.literal('frameworkDependencies'), -}) -export const FirstPartyDependencyScanSchema = TestBaseSchema.extend( - DependencyCountsSchema.shape, -).extend({ - name: z.literal('firstPartyDependencies'), - duplicateDependencies: z.number().int().nonnegative().optional(), -}) -export const CoreJsStatsSchema = TestBaseSchema.extend({ - name: z.literal('corejs'), - totalVendoredBytes: z.number().int().nonnegative(), - unnecessaryModules: z.array(z.string()), -}) +export * from '@framework-tracker/results' diff --git a/packages/stats-generator/src/ssrLoad/run-load-test.ts b/packages/stats-generator/src/ssrLoad/run-load-test.ts index 089ff643d..f7d1e9bc2 100644 --- a/packages/stats-generator/src/ssrLoad/run-load-test.ts +++ b/packages/stats-generator/src/ssrLoad/run-load-test.ts @@ -1,3 +1,4 @@ +import { measurements } from '../measurements.ts' import { getLoadPeakStage, selectRepresentativeIteration } from './selection.ts' import { SSRLoadTestsSchema } from '../schemas.ts' import { summarizeMetric } from '../sample-statistics.ts' @@ -7,7 +8,7 @@ import autocannon from 'autocannon' import type { SSRLoadStageStats, SSRLoadSweep, SSRLoadTests } from './types.ts' const STAGE_DURATION_SECONDS = 5 -const LOAD_SWEEP_COUNT = 3 + const WARMUP_ATTEMPTS = 3 const RECOVERY_MS = 5000 const WORKER_STAGES = [1, 5, 10, 25, 50, 100, 200] as const @@ -50,21 +51,21 @@ export async function runLoadTest( pause: (ms: number) => Promise = delay, ): Promise { const samples: SSRLoadSweep[] = [] - for (let run = 1; run <= LOAD_SWEEP_COUNT; run++) { + for (let run = 1; run <= measurements[getSSRLoadKind()].runs; run++) { if (run > 1) { console.info( - ` Sweep ${run}/${LOAD_SWEEP_COUNT}: recovering for ${RECOVERY_MS / 1000}s`, + ` Sweep ${run}/${measurements[getSSRLoadKind()].runs}: recovering for ${RECOVERY_MS / 1000}s`, ) await pause(RECOVERY_MS) } for (let attempt = 1; attempt <= WARMUP_ATTEMPTS; attempt++) { console.info( - ` Sweep ${run}/${LOAD_SWEEP_COUNT}: warm-up ${attempt}/${WARMUP_ATTEMPTS} for ${STAGE_DURATION_SECONDS}s`, + ` Sweep ${run}/${measurements[getSSRLoadKind()].runs}: warm-up ${attempt}/${WARMUP_ATTEMPTS} for ${STAGE_DURATION_SECONDS}s`, ) const warmup = await measureStage(url, WORKER_STAGES[0]) if (warmup.errors === 0 && warmup.requests > 0) break - const details = `sweep ${run}/${LOAD_SWEEP_COUNT}, attempt ${attempt}/${WARMUP_ATTEMPTS}: ${warmup.requests} requests, ${warmup.errors} errors, ${warmup.elapsedTimeMs}ms` + const details = `sweep ${run}/${measurements[getSSRLoadKind()].runs}, attempt ${attempt}/${WARMUP_ATTEMPTS}: ${warmup.requests} requests, ${warmup.errors} errors, ${warmup.elapsedTimeMs}ms` if (attempt === WARMUP_ATTEMPTS) { throw new Error( `SSR load warm-up failed (${details}); expected requests without errors after recovery.`, diff --git a/packages/stats-generator/src/ssrLoad/selection.ts b/packages/stats-generator/src/ssrLoad/selection.ts index 806ab0883..3e31a2fc7 100644 --- a/packages/stats-generator/src/ssrLoad/selection.ts +++ b/packages/stats-generator/src/ssrLoad/selection.ts @@ -1,36 +1,4 @@ -interface PeakCandidate { - errors: number - requestsPerSec: number -} - -export function getLoadPeakStage( - stages: readonly T[], -): T { - const peak = stages.reduce( - (best, stage) => - stage.errors === 0 && - (best === undefined || stage.requestsPerSec > best.requestsPerSec) - ? stage - : best, - undefined, - ) - if (peak === undefined) - throw new Error( - 'No valid SSR load peak: no stage completed with zero errors.', - ) - return peak -} - -export function selectRepresentativeIteration( - iterations: readonly (readonly PeakCandidate[])[], -): number { - if (iterations.length === 0 || iterations.length % 2 === 0) { - throw new Error('Expected an odd, nonzero number of load sweeps.') - } - return iterations - .map((sweep, index) => ({ - index, - peak: getLoadPeakStage(sweep).requestsPerSec, - })) - .sort((a, b) => a.peak - b.peak)[Math.floor(iterations.length / 2)]!.index -} +export { + getLoadPeakStage, + selectRepresentativeIteration, +} from '@framework-tracker/results/selection' diff --git a/packages/stats-generator/src/types.ts b/packages/stats-generator/src/types.ts index 0411731b3..4b1e7e1f9 100644 --- a/packages/stats-generator/src/types.ts +++ b/packages/stats-generator/src/types.ts @@ -16,30 +16,14 @@ import type { InstallResultSchema, } from './schemas.ts' import type { SSRLoadTests } from './ssrLoad/types.ts' -type MeasurementType = - | 'install' - | 'build' - | 'devServer' - | 'test' - | 'dependencies' - | 'browserBaseline' - | 'ssrRequestThroughput' - | 'ssrLoad' - | 'ssrRouterLinkLoad' - | 'clientSideRendered' - | 'serverSideRendered' - -interface MeasurementConfig { - type: MeasurementType - runFrequency?: number -} +import type { MeasurementName } from './measurements.ts' export interface TestConfig { package: string buildScript: string buildOutputDir: string devServerPort?: number - measurements: MeasurementConfig[] + measurements: MeasurementName[] } export interface InteractionTiming { diff --git a/packages/stats-generator/src/validate-stats.ts b/packages/stats-generator/src/validate-stats.ts index 3d7d8c3b2..794315f67 100644 --- a/packages/stats-generator/src/validate-stats.ts +++ b/packages/stats-generator/src/validate-stats.ts @@ -37,7 +37,7 @@ type BenchmarkType = interface BenchmarkConfig { type: BenchmarkType file: string - schema: z.ZodSchema + schema: z.ZodType } const STARTER_BENCHMARKS: BenchmarkConfig[] = [ @@ -104,7 +104,7 @@ const APP_BENCHMARKS: BenchmarkConfig[] = [ }, ] -function validateFile(filePath: string, schema: z.ZodSchema): string[] { +function validateFile(filePath: string, schema: z.ZodType): string[] { const data = readJsonFile(filePath) if (!data) { @@ -151,11 +151,7 @@ async function main() { const errors: string[] = [] for (const { type: benchType, file, schema } of STARTER_BENCHMARKS) { if (filterType && filterType !== benchType) continue - if ( - !framework.starter.measurements.some( - (measurement) => measurement.type === benchType, - ) - ) { + if (!framework.starter.measurements.includes(benchType)) { continue } @@ -185,11 +181,7 @@ async function main() { const errors: string[] = [] for (const { type: benchType, file, schema } of APP_BENCHMARKS) { if (filterType && filterType !== benchType) continue - if ( - !framework.app.measurements.some( - (measurement) => measurement.type === benchType, - ) - ) { + if (!framework.app.measurements.includes(benchType)) { continue } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index df90ce1fc..57088662a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,9 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + zod@>=4: 4.6.7 + importers: .: @@ -39,8 +42,8 @@ importers: packages/cwv-stats: dependencies: zod: - specifier: ^4.4.3 - version: 4.4.3 + specifier: 4.6.7 + version: 4.6.7 devDependencies: '@types/node': specifier: ^26.1.2 @@ -54,6 +57,9 @@ importers: '@astrojs/starlight': specifier: ^0.41.7 version: 0.41.7(astro@7.4.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.1))(typescript@6.0.3) + '@framework-tracker/results': + specifier: workspace:* + version: link:../results '@tanstack/charts': specifier: ^1.1.0 version: 1.1.0 @@ -77,8 +83,17 @@ importers: specifier: ^2.1.1 version: 2.1.1(@typescript-eslint/parser@8.62.1(eslint@10.6.0(jiti@2.7.0))(typescript@6.0.3))(eslint@10.6.0(jiti@2.7.0)) + packages/results: + dependencies: + zod: + specifier: 4.6.7 + version: 4.6.7 + packages/stats-generator: dependencies: + '@framework-tracker/results': + specifier: workspace:* + version: link:../results autocannon: specifier: ^8.0.0 version: 8.0.0 @@ -107,8 +122,8 @@ importers: specifier: 2.9.0 version: 2.9.0 zod: - specifier: ^4.4.3 - version: 4.4.3 + specifier: 4.6.7 + version: 4.6.7 devDependencies: '@types/autocannon': specifier: ^7.12.7 @@ -4419,9 +4434,6 @@ packages: zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} - zod@4.4.3: - resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} - zod@4.6.7: resolution: {integrity: sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==} @@ -4680,7 +4692,7 @@ snapshots: dependencies: sitemap: 9.0.1 stream-replace-string: 2.0.0 - zod: 4.4.3 + zod: 4.6.7 '@astrojs/starlight@0.41.7(astro@7.4.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(yaml@2.9.1))(typescript@6.0.3)': dependencies: @@ -7323,7 +7335,7 @@ snapshots: tinyglobby: 0.2.17 unbash: 4.0.11 yaml: 2.9.0 - zod: 4.4.3 + zod: 4.6.7 legacy-javascript@0.0.3: {} @@ -9124,8 +9136,6 @@ snapshots: zod@3.25.76: {} - zod@4.4.3: {} - zod@4.6.7: {} zwitch@2.0.4: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index dc6a2e90e..5981d0ddd 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,5 @@ packages: + - 'packages/results' - 'packages/docs' - 'packages/stats-generator' - 'packages/cwv-stats' @@ -10,3 +11,7 @@ allowBuilds: minimumReleaseAgeExclude: - baseline-detector@0.4.1 + +# Keep shared schemas and Astro on the same Zod 4 implementation. +overrides: + 'zod@>=4': 4.6.7