feat(devtime): add dev server startup time metric

.github/frameworks.json

@@ -27,6 +27,7 @@
       "package": "starter-astro",
       "buildScript": "build",
       "buildOutputDir": "dist",
+      "devServerPort": 4321,
       "measurements": [
         {
           "type": "install",
@@ -36,6 +37,10 @@
           "type": "build",
           "runFrequency": 5
         },
+        {
+          "type": "devServer",
+          "runFrequency": 5
+        },
         {
           "type": "dependencies"
         },
@@ -73,6 +78,7 @@
       "package": "starter-mastro",
       "buildScript": "build",
       "buildOutputDir": "generated",
+      "devServerPort": 8000,
       "measurements": [
         {
           "type": "install",
@@ -82,6 +88,10 @@
           "type": "build",
           "runFrequency": 5
         },
+        {
+          "type": "devServer",
+          "runFrequency": 5
+        },
         {
           "type": "dependencies"
         },
@@ -116,6 +126,7 @@
       "package": "starter-next-js",
       "buildScript": "build",
       "buildOutputDir": ".next",
+      "devServerPort": 3000,
       "measurements": [
         {
           "type": "install",
@@ -125,6 +136,10 @@
           "type": "build",
           "runFrequency": 5
         },
+        {
+          "type": "devServer",
+          "runFrequency": 5
+        },
         {
           "type": "dependencies"
         },
@@ -162,6 +177,7 @@
       "package": "starter-nuxt",
       "buildScript": "build",
       "buildOutputDir": ".output",
+      "devServerPort": 3000,
       "measurements": [
         {
           "type": "install",
@@ -171,6 +187,10 @@
           "type": "build",
           "runFrequency": 5
         },
+        {
+          "type": "devServer",
+          "runFrequency": 5
+        },
         {
           "type": "dependencies"
         },
@@ -208,6 +228,7 @@
       "package": "starter-react-router",
       "buildScript": "build",
       "buildOutputDir": "build",
+      "devServerPort": 5173,
       "measurements": [
         {
           "type": "install",
@@ -217,6 +238,10 @@
           "type": "build",
           "runFrequency": 5
         },
+        {
+          "type": "devServer",
+          "runFrequency": 5
+        },
         {
           "type": "dependencies"
         },
@@ -254,6 +279,7 @@
       "package": "starter-solid-start",
       "buildScript": "build",
       "buildOutputDir": ".output",
+      "devServerPort": 3000,
       "measurements": [
         {
           "type": "install",
@@ -263,6 +289,10 @@
           "type": "build",
           "runFrequency": 5
         },
+        {
+          "type": "devServer",
+          "runFrequency": 5
+        },
         {
           "type": "dependencies"
         },
@@ -300,6 +330,7 @@
       "package": "starter-sveltekit",
       "buildScript": "build",
       "buildOutputDir": "build",
+      "devServerPort": 5173,
       "measurements": [
         {
           "type": "install",
@@ -309,6 +340,10 @@
           "type": "build",
           "runFrequency": 5
         },
+        {
+          "type": "devServer",
+          "runFrequency": 5
+        },
         {
           "type": "dependencies"
         },
@@ -346,6 +381,7 @@
       "package": "starter-tanstack-start-react",
       "buildScript": "build",
       "buildOutputDir": ".output",
+      "devServerPort": 3000,
       "measurements": [
         {
           "type": "install",
@@ -355,6 +391,10 @@
           "type": "build",
           "runFrequency": 5
         },
+        {
+          "type": "devServer",
+          "runFrequency": 5
+        },
         {
           "type": "dependencies"
         },

.github/workflows/generate-stats.yml

@@ -13,6 +13,7 @@ on:
       - '!packages/starter-*/stats/**'
       - '!packages/starter-*/install-stats.json'
       - '!packages/starter-*/build-stats.json'
+      - '!packages/starter-*/dev-server-stats.json'
       - '!packages/starter-*/corejs-stats.json'
       - '!packages/starter-*/browser-baseline-stats.json'
       - '!packages/starter-*/e18e-stats.json'
@@ -32,6 +33,7 @@ jobs:
     outputs:
       install-matrix: ${{ steps.set-matrix.outputs.install }}
       build-matrix: ${{ steps.set-matrix.outputs.build }}
+      dev-server-matrix: ${{ steps.set-matrix.outputs.dev-server }}
       ssr-request-throughput-matrix: ${{ steps.set-matrix.outputs.ssr-request-throughput }}
       ssr-load-matrix: ${{ steps.set-matrix.outputs.ssr-load }}
       deps-matrix: ${{ steps.set-matrix.outputs.deps }}
@@ -47,6 +49,7 @@ jobs:
           FRAMEWORKS=$(cat .github/frameworks.json)
           echo "install=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT
           echo "build=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT
+          echo "dev-server=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT
           echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT
           echo "ssr-load=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.app) | select(.app.measurements | map(.type) | contains(["ssrLoad"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT
           echo "deps=$(echo "$FRAMEWORKS" | jq -c '[.[] | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT
@@ -59,6 +62,7 @@ jobs:
     with:
       install-matrix: ${{ needs.setup.outputs.install-matrix }}
       build-matrix: ${{ needs.setup.outputs.build-matrix }}
+      dev-server-matrix: ${{ needs.setup.outputs.dev-server-matrix }}
       ssr-request-throughput-matrix: ${{ needs.setup.outputs.ssr-request-throughput-matrix }}
       ssr-load-matrix: ${{ needs.setup.outputs.ssr-load-matrix }}
       deps-matrix: ${{ needs.setup.outputs.deps-matrix }}

.github/workflows/measure-framework.yml

@@ -14,6 +14,11 @@ on:
         description: 'JSON array of frameworks to measure build time'
         type: string
         required: true
+      dev-server-matrix:
+        description: 'JSON array of frameworks to measure dev server startup time'
+        type: string
+        required: false
+        default: '[]'
       ssr-request-throughput-matrix:
         description: 'JSON array of frameworks to measure SSR request throughput performance'
         type: string
@@ -135,6 +140,43 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
+  measure-dev-server:
+    if: inputs.dev-server-matrix != '[]'
+    runs-on: depot-ubuntu-24.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        framework: ${{ fromJson(inputs.dev-server-matrix) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5
+
+      - name: Setup Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: '24'
+          cache: 'pnpm'
+
+      - name: Install workspace dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run dev server benchmark
+        run: |
+          RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "devServer") | .runFrequency')
+          pnpm --filter @framework-tracker/stats-generator run:dev-server ${{ matrix.framework.package }} $RUN_FREQUENCY
+
+      - name: Upload dev server stats
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: dev-server-stats-${{ matrix.framework.name }}
+          path: packages/${{ matrix.framework.package }}/dev-server-stats.json
+          retention-days: 1
+          if-no-files-found: error
+
   measure-ssr-request-throughput:
     if: inputs.ssr-request-throughput-matrix != '[]'
     runs-on: depot-ubuntu-24.04

.github/workflows/preview-stats.yml

@@ -12,6 +12,7 @@ jobs:
     outputs:
       install-matrix: ${{ steps.detect.outputs.install }}
       build-matrix: ${{ steps.detect.outputs.build }}
+      dev-server-matrix: ${{ steps.detect.outputs.dev-server }}
       ssr-request-throughput-matrix: ${{ steps.detect.outputs.ssr-request-throughput }}
       deps-matrix: ${{ steps.detect.outputs.deps }}
       framework-names: ${{ steps.detect.outputs.framework-names }}
@@ -48,6 +49,7 @@ jobs:
             echo "framework-names=[]" >> $GITHUB_OUTPUT
             echo "install=[]" >> $GITHUB_OUTPUT
             echo "build=[]" >> $GITHUB_OUTPUT
+            echo "dev-server=[]" >> $GITHUB_OUTPUT
             echo "ssr-request-throughput=[]" >> $GITHUB_OUTPUT
             echo "deps=[]" >> $GITHUB_OUTPUT
             exit 0
@@ -71,6 +73,7 @@ jobs:
 
           echo "install=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["install"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT
           echo "build=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["build"])) | {name, displayName, package: .starter.package, buildScript: .starter.buildScript, buildOutputDir: .starter.buildOutputDir, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT
+          echo "dev-server=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["devServer"])) | {name, displayName, package: .starter.package, devServerPort: .starter.devServerPort, measurements: .starter.measurements}]')" >> $GITHUB_OUTPUT
           echo "ssr-request-throughput=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.app) | select(.app.measurements | map(.type) | contains(["ssrRequestThroughput"])) | {name, displayName, package: .app.package, buildScript: .app.buildScript, buildOutputDir: .app.buildOutputDir, measurements: .app.measurements}]')" >> $GITHUB_OUTPUT
           echo "deps=$(echo "$FRAMEWORKS" | jq -c --argjson names "$NAMES_JSON" '[.[] | select(.name as $n | $names | contains([$n])) | select(.starter) | select(.starter.measurements | map(.type) | contains(["dependencies"])) | {name, displayName, package: .starter.package}]')" >> $GITHUB_OUTPUT
 
@@ -81,6 +84,7 @@ jobs:
     with:
       install-matrix: ${{ needs.setup.outputs.install-matrix }}
       build-matrix: ${{ needs.setup.outputs.build-matrix }}
+      dev-server-matrix: ${{ needs.setup.outputs.dev-server-matrix }}
       ssr-request-throughput-matrix: ${{ needs.setup.outputs.ssr-request-throughput-matrix }}
       deps-matrix: ${{ needs.setup.outputs.deps-matrix }}
 

.github/workflows/validate-stats.yml

@@ -16,6 +16,7 @@ on:
       - '!packages/starter-*/stats/**'
       - '!packages/starter-*/install-stats.json'
       - '!packages/starter-*/build-stats.json'
+      - '!packages/starter-*/dev-server-stats.json'
       - '!packages/starter-*/corejs-stats.json'
       - '!packages/starter-*/browser-baseline-stats.json'
       - '!packages/starter-*/e18e-stats.json'
@@ -82,6 +83,8 @@ jobs:
             pnpm --filter @framework-tracker/stats-generator run:install $PKG 1
             echo "Running build benchmark for $PKG..."
             pnpm --filter @framework-tracker/stats-generator run:build $PKG
+            echo "Running dev server benchmark for $PKG (1 run)..."
+            pnpm --filter @framework-tracker/stats-generator run:dev-server $PKG 1
             echo "Running Node engines scan for $PKG..."
             pnpm --filter @framework-tracker/stats-generator run:node-engines $PKG
           done

.gitignore

@@ -70,6 +70,7 @@ coverage/
 /artifacts
 packages/*/install-stats.json
 packages/*/build-stats.json
+packages/*/dev-server-stats.json
 packages/*/corejs-stats.json
 packages/*/e18e-stats.json
 packages/*/framework-dependency-stats.json

packages/docs/src/components/DevServerStartChart.astro

@@ -0,0 +1,23 @@
+---
+import { starterStats } from '../lib/collections'
+import ComparisonBarChart from './ComparisonBarChart.astro'
+
+const validEntries = starterStats.filter(
+  (f) => f?.name != null && Number.isFinite(f.devServerStartTime?.avgMs),
+)
+
+const data = validEntries
+  .map((f) => ({
+    name: f.name,
+    value: (f.devServerStartTime?.avgMs ?? 0) / 1000,
+    focused: f.isFocused,
+  }))
+  .sort((a, b) => a.value - b.value || a.name.localeCompare(b.name))
+---
+
+<ComparisonBarChart
+  title="Avg dev server startup time"
+  data={data}
+  valueFormat="s"
+  yAxisLabel="Seconds"
+/>

packages/docs/src/components/DevServerStartTable.astro

@@ -0,0 +1,24 @@
+---
+import { devServerStartData } from '../lib/collections'
+import { getFrameworkSlug } from '../lib/utils'
+import StatsTable from './StatsTable.astro'
+
+const columns = [
+  {
+    key: 'name',
+    header: 'Framework',
+    nameCell: true,
+    href: (row: Record<string, unknown>) =>
+      `/framework/${getFrameworkSlug(row.package as string)}`,
+  },
+  { key: 'avgStart', header: 'Avg Start' },
+  { key: 'minStart', header: 'Min Start' },
+  { key: 'maxStart', header: 'Max Start' },
+]
+---
+
+<StatsTable
+  label="Dev server startup time by framework"
+  columns={columns}
+  data={devServerStartData}
+/>

packages/docs/src/components/FrameworkDevServerVersionCharts.astro

@@ -0,0 +1,18 @@
+---
+import VersionLineChart from './VersionLineChart.astro'
+
+interface Props {
+  versions: unknown[]
+}
+
+const { versions } = Astro.props
+---
+
+<VersionLineChart
+  title="Average Dev Server Startup Time"
+  versions={versions}
+  metric="devServerStartTime.avgMs"
+  transform="msToSeconds"
+  valueFormat="s"
+  yAxisLabel="Seconds"
+/>

packages/docs/src/content.config.ts

@@ -44,6 +44,7 @@ const devtimeSchema = z.object({
   installTime: timeSchema,
   coldBuildTime: timeSchema,
   warmBuildTime: timeSchema,
+  devServerStartTime: timeSchema.optional(),
   buildOutputSize: z.number(),
   nodeModulesSize: z.number(),
   duplicateDependencies: z.number().optional(),

packages/docs/src/content/docs/all-frameworks.mdx

@@ -14,6 +14,8 @@ import CoreWebVitalsTable from '../../components/CoreWebVitalsTable.astro'
 import DependencyCharts from '../../components/DependencyCharts.astro'
 import DependencyCountText from '../../components/DependencyCountText.astro'
 import DependencyStatsTable from '../../components/DependencyStatsTable.astro'
+import DevServerStartChart from '../../components/DevServerStartChart.astro'
+import DevServerStartTable from '../../components/DevServerStartTable.astro'
 import FrameworkDependencyCharts from '../../components/FrameworkDependencyCharts.astro'
 import FrameworkDependencyStatsTable from '../../components/FrameworkDependencyStatsTable.astro'
 import MinimumNodeVersionTable from '../../components/MinimumNodeVersionTable.astro'
@@ -57,6 +59,13 @@ Methodology: [Build and Install Times](/methodology/#build-and-install-times).
 <BuildInstallStatsTable />
 <BuildSizeCharts />
 
+## Dev Server Startup
+
+Methodology: [Dev Server Startup](/methodology/#dev-server-startup).
+
+<DevServerStartTable />
+<DevServerStartChart />
+
 ## Core-JS Polyfills
 
 Methodology: [Core-JS Polyfills](/methodology/#core-js-polyfills).