654 update test schema ssr load

packages/docs/src/components/SSRLoadStatsTable.astro

@@ -17,12 +17,12 @@ const columns = [
         ? `/framework/${getFrameworkSlug(row.package as string)}`
         : null,
   },
-  { key: 'peakRequestsPerSec', header: 'Peak req/s' },
-  { key: 'peakWorkers', header: 'Peak Connections' },
-  { key: 'worker25P99LatencyMs', header: 'P99 @ 25' },
-  { key: 'worker50P99LatencyMs', header: 'P99 @ 50' },
-  { key: 'worker100P99LatencyMs', header: 'P99 @ 100' },
-  { key: 'totalRequests', header: 'Total Req.' },
+  { key: 'peakRequestsPerSec', header: 'Representative peak req/s' },
+  { key: 'peakWorkers', header: 'Representative peak connections' },
+  { key: 'worker25P99LatencyMs', header: 'Mean sweep P99 @ 25' },
+  { key: 'worker50P99LatencyMs', header: 'Mean sweep P99 @ 50' },
+  { key: 'worker100P99LatencyMs', header: 'Mean sweep P99 @ 100' },
+  { key: 'totalRequests', header: 'Total measured requests' },
 ]
 ---
 

packages/docs/src/content.config.ts

@@ -26,7 +26,6 @@ const buildTimeSchema = z.union([
         median: z.number(),
         min: z.number(),
         max: z.number(),
-        variance: z.number().nonnegative(),
       }),
     })
     .refine((value) => value.iterations.length === value.runs)
@@ -35,7 +34,6 @@ const buildTimeSchema = z.union([
       medianMs: value.timeMs.median,
       minMs: value.timeMs.min,
       maxMs: value.timeMs.max,
-      standardDeviationMs: Math.sqrt(value.timeMs.variance),
       samplesMs: value.iterations.map((run) => run.timeMs),
     })),
   timeSchema,
@@ -52,7 +50,6 @@ const devServerTimeSchema = z.union([
         median: z.number(),
         min: z.number(),
         max: z.number(),
-        variance: z.number().nonnegative(),
       }),
     })
     .refine((value) => value.iterations.length === value.runs)
@@ -61,7 +58,6 @@ const devServerTimeSchema = z.union([
       medianMs: value.timeMs.median,
       minMs: value.timeMs.min,
       maxMs: value.timeMs.max,
-      standardDeviationMs: Math.sqrt(value.timeMs.variance),
       samplesMs: value.iterations.map((run) => run.timeMs),
     })),
   timeSchema,
@@ -85,7 +81,6 @@ const metricSchema = z.object({
   median: z.number(),
   min: z.number(),
   max: z.number(),
-  variance: z.number().nonnegative(),
 })
 const ClientRenderingBenchmarkSchema = z
   .object({
@@ -155,7 +150,6 @@ const devtimeSchema = z.object({
       median: z.number(),
       min: z.number(),
       max: z.number(),
-      variance: z.number().nonnegative(),
     }),
     nodeModulesSizeBytes: z.number().nonnegative(),
   }),
@@ -188,50 +182,128 @@ const devtimeSchema = z.object({
   releasedAt: z.string().optional(),
 })
 
-const ssrLoadSweepSchema = z.object({
-  peakWorkers: z.number(),
-  peakRequestsPerSec: z.number(),
-  peakAvgLatencyMs: z.number(),
-  peakP50LatencyMs: z.number(),
-  peakP75LatencyMs: z.number(),
-  peakP90LatencyMs: z.number(),
-  peakP99LatencyMs: z.number(),
-  totalRequests: z.number(),
-  totalErrors: z.number(),
-  stages: z.array(
-    z.object({
-      workers: z.number(),
-      durationMs: z.number(),
-      requests: z.number(),
-      errors: z.number(),
-      requestsPerSec: z.number(),
-      avgLatencyMs: z.number(),
-      medianLatencyMs: z.number(),
-      p50LatencyMs: z.number(),
-      p75LatencyMs: z.number(),
-      p90LatencyMs: z.number(),
-      p99LatencyMs: z.number(),
-      maxLatencyMs: z.number(),
-      bytesPerSec: z.number(),
-    }),
-  ),
+const ssrLoadStageSchema = z.object({
+  workers: z.number().int().positive(),
+  elapsedTimeMs: z.number().positive(),
+  requests: z.number().int().nonnegative(),
+  errors: z.number().int().nonnegative(),
+  requestsPerSec: z.number().nonnegative(),
+  avgLatencyMs: z.number().nonnegative(),
+  p50LatencyMs: z.number().nonnegative(),
+  p75LatencyMs: z.number().nonnegative(),
+  p90LatencyMs: z.number().nonnegative(),
+  p99LatencyMs: z.number().nonnegative(),
+  maxLatencyMs: z.number().nonnegative(),
+  bytesPerSec: z.number().nonnegative(),
 })
 
-const ssrLoadTestsSchema = ssrLoadSweepSchema.extend({
-  runs: z.number().int().positive().optional(),
-  samples: z.array(ssrLoadSweepSchema).nonempty().optional(),
-  warmupDurationMs: z.number().positive().optional(),
+const ssrLoadSweepSchema = z.array(ssrLoadStageSchema).nonempty()
+
+const ssrLoadStageSummarySchema = z.object({
+  connections: z.number().int().positive(),
+  requests: metricSchema,
+  errors: metricSchema,
+  requestsPerSec: metricSchema,
+  avgLatencyMs: metricSchema,
+  p50LatencyMs: metricSchema,
+  p75LatencyMs: metricSchema,
+  p90LatencyMs: metricSchema,
+  p99LatencyMs: metricSchema,
+  maxLatencyMs: metricSchema,
+  bytesPerSec: metricSchema,
 })
 
+const ssrLoadTestsSchema = z
+  .object({
+    name: z.enum(['ssrLoad', 'ssrRouterLinkLoad']),
+    runs: z.number().int().positive(),
+    iterations: z.array(ssrLoadSweepSchema).nonempty(),
+    configuration: z.object({
+      workerStages: z.array(z.number().int().positive()).nonempty(),
+      warmupDurationMs: z.number().positive().optional(),
+      recoveryMs: z.number().nonnegative().optional(),
+      warmupAttempts: z.number().int().positive().optional(),
+      stageDurationMs: z.number().positive().optional(),
+    }),
+    representativeIterationIndex: z.number().int().nonnegative(),
+    source: z.literal('autocannon'),
+    path: z.enum([
+      '/server-side-rendered',
+      '/server-side-rendered-plain-links',
+    ]),
+    totalRequests: z.number().int().nonnegative(),
+    totalErrors: z.number().int().nonnegative(),
+    stage1: ssrLoadStageSummarySchema,
+    stage2: ssrLoadStageSummarySchema.optional(),
+    stage3: ssrLoadStageSummarySchema.optional(),
+    stage4: ssrLoadStageSummarySchema.optional(),
+    stage5: ssrLoadStageSummarySchema.optional(),
+    stage6: ssrLoadStageSummarySchema.optional(),
+    stage7: ssrLoadStageSummarySchema.optional(),
+  })
+  .refine(
+    (value) =>
+      value.iterations.length === value.runs &&
+      value.representativeIterationIndex < value.iterations.length &&
+      value.iterations[value.representativeIterationIndex]!.some(
+        (stage) => stage.errors === 0,
+      ),
+  )
+  .transform((value) => {
+    const representativeSweep =
+      value.iterations[value.representativeIterationIndex]!
+    const representativePeak = representativeSweep
+      .filter((stage) => stage.errors === 0)
+      .reduce((best, stage) =>
+        stage.requestsPerSec > best.requestsPerSec ? stage : best,
+      )
+    return {
+      ...value,
+      representativeSweep,
+      representativePeak,
+      stages: [
+        value.stage1,
+        value.stage2,
+        value.stage3,
+        value.stage4,
+        value.stage5,
+        value.stage6,
+        value.stage7,
+      ].flatMap((stage) =>
+        stage
+          ? [
+              {
+                workers: stage.connections,
+                requests: stage.requests.mean,
+                errors: stage.errors.mean,
+                requestsPerSec: stage.requestsPerSec.mean,
+                avgLatencyMs: stage.avgLatencyMs.mean,
+                p50LatencyMs: stage.p50LatencyMs.mean,
+                p75LatencyMs: stage.p75LatencyMs.mean,
+                p90LatencyMs: stage.p90LatencyMs.mean,
+                p99LatencyMs: stage.p99LatencyMs.mean,
+                maxLatencyMs: stage.maxLatencyMs.mean,
+                bytesPerSec: stage.bytesPerSec.mean,
+              },
+            ]
+          : [],
+      ),
+    }
+  })
+
 const runtimeSchema = z.object({
   name: z.string(),
   type: z.string(),
   package: z.string(),
   isFocused: z.boolean(),
   order: z.number(),
   ssrRequestThroughputTests: requestThroughputContentSchema,
-  ssrLoadTests: ssrLoadTestsSchema.optional(),
-  ssrRouterLinkLoadTests: ssrLoadTestsSchema.optional(),
+  ssrLoadTests: ssrLoadTestsSchema
+    .refine((value) => value.name === 'ssrLoad')
+    .optional(),
+  ssrRouterLinkLoadTests: ssrLoadTestsSchema
+    .refine((value) => value.name === 'ssrRouterLinkLoad')
+    .optional(),
   clientSideRenderedTests: z
     .union([
       ClientRenderingBenchmarkSchema.transform((value) => ({

packages/docs/src/content/devtime/starter-astro.json

@@ -36,8 +36,7 @@
       "mean": 1122.6,
       "median": 1190,
       "min": 967,
-      "max": 1283,
-      "variance": 19628.3
+      "max": 1283
     }
   },
   "warmBuildTime": {
@@ -64,8 +63,7 @@
       "mean": 1158.2,
       "median": 1194,
       "min": 939,
-      "max": 1446,
-      "variance": 46260.2
+      "max": 1446
     }
   },
   "order": 1,
@@ -116,8 +114,7 @@
       "mean": 1439,
       "median": 1409,
       "min": 1379,
-      "max": 1590,
-      "variance": 7615.5
+      "max": 1590
     }
   },
   "firstPartyDependencies": {
@@ -152,8 +149,7 @@
       "mean": 3549.2,
       "median": 4182,
       "min": 1783,
-      "max": 4206,
-      "variance": 1102950.7
+      "max": 4206
     },
     "nodeModulesSizeBytes": 214360064
   },

packages/docs/src/content/devtime/starter-next-js.json

@@ -36,8 +36,7 @@
       "mean": 6105.8,
       "median": 6030,
       "min": 5759,
-      "max": 6638,
-      "variance": 104152.7
+      "max": 6638
     }
   },
   "warmBuildTime": {
@@ -64,8 +63,7 @@
       "mean": 2347,
       "median": 2331,
       "min": 2311,
-      "max": 2408,
-      "variance": 1481.5
+      "max": 2408
     }
   },
   "order": 2,
@@ -415,8 +413,7 @@
       "mean": 2863.8,
       "median": 2768,
       "min": 2443,
-      "max": 3563,
-      "variance": 184731.7
+      "max": 3563
     }
   },
   "firstPartyDependencies": {
@@ -451,8 +448,7 @@
       "mean": 5892.8,
       "median": 6690,
       "min": 2955,
-      "max": 6897,
-      "variance": 2758639.7
+      "max": 6897
     },
     "nodeModulesSizeBytes": 486125568
   },

packages/docs/src/content/devtime/starter-nuxt.json

@@ -36,8 +36,7 @@
       "mean": 4251.8,
       "median": 4256,
       "min": 3831,
-      "max": 4539,
-      "variance": 90422.70000000001
+      "max": 4539
     }
   },
   "warmBuildTime": {
@@ -64,8 +63,7 @@
       "mean": 4029.2,
       "median": 3961,
       "min": 3884,
-      "max": 4390,
-      "variance": 41714.7
+      "max": 4390
     }
   },
   "order": 3,
@@ -119,8 +117,7 @@
       "mean": 2479.6,
       "median": 2468,
       "min": 2409,
-      "max": 2570,
-      "variance": 3467.3
+      "max": 2570
     }
   },
   "firstPartyDependencies": {
@@ -155,8 +152,7 @@
       "mean": 4668.2,
       "median": 4963,
       "min": 2857,
-      "max": 5365,
-      "variance": 1087801.2
+      "max": 5365
     },
     "nodeModulesSizeBytes": 233607168
   },

packages/docs/src/content/devtime/starter-pracht.json

@@ -31,8 +31,7 @@
       "mean": 705.2,
       "median": 697,
       "min": 695,
-      "max": 738,
-      "variance": 342.19999999999993
+      "max": 738
     }
   },
   "warmBuildTime": {
@@ -59,8 +58,7 @@
       "mean": 699.6,
       "median": 687,
       "min": 681,
-      "max": 740,
-      "variance": 576.8000000000001
+      "max": 740
     }
   },
   "devServerStartTime": {
@@ -87,8 +85,7 @@
       "mean": 933.2,
       "median": 939,
       "min": 899,
-      "max": 967,
-      "variance": 668.1999999999999
+      "max": 967
     }
   },
   "vendoredCoreJsSize": 0,
@@ -152,8 +149,7 @@
       "mean": 1889,
       "median": 1807,
       "min": 1539,
-      "max": 2270,
-      "variance": 92357.5
+      "max": 2270
     },
     "nodeModulesSizeBytes": 138997760
   },