Fix missing timing summary penalties and dataset coverage

web/lib/summary.test.ts

@@ -327,6 +327,90 @@ describe('timing summaries (shared ranking model)', () => {
     expect(byName.get('partial')?.totalRuntime).toBe(0);
   });
 
+  it('counts datasets with complementary incomplete formats in each open mode', async () => {
+    query.mockResolvedValueOnce({
+      rows: [
+        { bucket: 'taxi', series: 'lance', open_mode: 'cached', value: 100_000 },
+        { bucket: 'taxi', series: 'vortex', open_mode: 'cached', value: 200_000 },
+        {
+          bucket: 'feature-vectors/correlated',
+          series: 'lance',
+          open_mode: 'cached',
+          value: 50_000,
+        },
+        {
+          bucket: 'feature-vectors/uniform',
+          series: 'vortex',
+          open_mode: 'cached',
+          value: 300_000,
+        },
+        { bucket: 'taxi', series: 'lance', open_mode: 'reopen', value: 400_000 },
+        { bucket: 'taxi', series: 'vortex', open_mode: 'reopen', value: 200_000 },
+        {
+          bucket: 'nested-structs/correlated',
+          series: 'lance',
+          open_mode: 'reopen',
+          value: 100_000,
+        },
+        { bucket: 'nested-structs/uniform', series: 'vortex', open_mode: 'reopen', value: 500_000 },
+        { bucket: 'feature-vectors', series: 'lance', open_mode: 'reopen', value: 200_000 },
+        { bucket: 'feature-vectors', series: 'vortex', open_mode: 'reopen', value: 200_000 },
+      ],
+    });
+
+    const summary = await collectGroupSummary({ k: 'RandomAccessGroup' });
+    if (summary === null || summary.type !== 'randomAccess') {
+      throw new Error('expected a randomAccess summary');
+    }
+
+    expect(
+      summary.hotRankings.map((ranking) => [
+        ranking.name,
+        ranking.measured,
+        ranking.total,
+        ranking.totalRuntime,
+      ]),
+    ).toEqual([
+      ['lance', 1, 2, 100_000],
+      ['vortex', 1, 2, 200_000],
+    ]);
+    expect(summary.hotRankings[0].score).toBeCloseTo(Math.sqrt(2), 6);
+    expect(summary.hotRankings[1].score).toBeCloseTo(Math.sqrt(2 * (200_010 / 100_010)), 6);
+    expect(
+      summary.coldRankings.map((ranking) => [
+        ranking.name,
+        ranking.measured,
+        ranking.total,
+        ranking.totalRuntime,
+      ]),
+    ).toEqual([
+      ['vortex', 2, 3, 200_000],
+      ['lance', 2, 3, 300_000],
+    ]);
+    expect(summary.coldRankings[0].score).toBeCloseTo(Math.cbrt(2), 6);
+    expect(summary.coldRankings[1].score).toBeCloseTo(Math.cbrt(2 * (400_010 / 200_010)), 6);
+  });
+
+  it('reports zero coverage when no random-access dataset is complete', async () => {
+    query.mockResolvedValueOnce({
+      rows: [
+        { bucket: 'taxi/correlated', series: 'lance', value: 100_000 },
+        { bucket: 'taxi/uniform', series: 'vortex', value: 200_000 },
+      ],
+    });
+
+    const summary = await collectGroupSummary({ k: 'RandomAccessGroup' });
+    if (summary === null || summary.type !== 'randomAccess') {
+      throw new Error('expected a randomAccess summary');
+    }
+
+    expect(summary.hotRankings).toEqual([
+      { name: 'lance', score: 2, measured: 0, total: 1, totalRuntime: 0 },
+      { name: 'vortex', score: 2, measured: 0, total: 1, totalRuntime: 0 },
+    ]);
+    expect(summary.coldRankings).toEqual([]);
+  });
+
   it('does not reward a series for skipping a slow bucket', async () => {
     query.mockResolvedValueOnce({
       rows: [
@@ -396,6 +480,51 @@ describe('timing summaries (shared ranking model)', () => {
     expect(summary.rankings[1].totalRuntime).toBeCloseTo(10_000, 6);
   });
 
+  it.each([
+    { name: 'fast', missingQueryRuntime: 50_000, missingRatio: 600_010 / 50_010 },
+    { name: 'slow', missingQueryRuntime: 100_000_000, missingRatio: 2 },
+  ])(
+    'penalizes a missing $name query without scoring it faster than the observed best',
+    async ({ missingQueryRuntime, missingRatio }) => {
+      query.mockResolvedValueOnce({
+        rows: [
+          { query_idx: 1, series: 'datafusion:partial', value_ns: 100_000 },
+          { query_idx: 1, series: 'datafusion:complete', value_ns: 110_000 },
+          { query_idx: 2, series: 'datafusion:complete', value_ns: missingQueryRuntime },
+        ],
+      });
+
+      const summary = await collectGroupSummary({
+        k: 'QueryGroup',
+        dataset: 'tpch',
+        dataset_variant: null,
+        scale_factor: null,
+        storage: 'nvme',
+      });
+      if (summary === null || summary.type !== 'queryBenchmark') {
+        throw new Error('expected a queryBenchmark summary');
+      }
+      const byName = new Map(summary.rankings.map((ranking) => [ranking.name, ranking]));
+
+      expect(summary.rankings.map((ranking) => ranking.name)).toEqual([
+        'datafusion:complete',
+        'datafusion:partial',
+      ]);
+      expect(byName.get('datafusion:partial')?.score).toBeCloseTo(Math.sqrt(missingRatio), 6);
+      expect(byName.get('datafusion:partial')).toMatchObject({
+        measured: 1,
+        total: 2,
+        totalRuntime: 100_000,
+      });
+      expect(byName.get('datafusion:complete')?.score).toBeCloseTo(Math.sqrt(110_010 / 100_010), 6);
+      expect(byName.get('datafusion:complete')).toMatchObject({
+        measured: 2,
+        total: 2,
+        totalRuntime: 110_000 + missingQueryRuntime,
+      });
+    },
+  );
+
   it('summarizes every query group, with no dataset allowlist', async () => {
     // `spatialbench` (and every other suite outside the retired v2 five) used to
     // fall through to `null` and render no card at all.

web/lib/summary.ts

@@ -240,19 +240,19 @@ interface SeriesSample<K> {
  *
  * The `(10 + value) / (10 + best)` ratio (rather than `value / best`) is v2's,
  * damping sub-10ns noise; it is preserved because the shipped query scores are
- * pinned to it. Random-access and vector-search summaries also set a 2x floor
- * for a missing bucket. The floor prevents a penalty derived from a fast bucket
- * from beating a real measurement on a slower bucket. Query summaries keep a
- * zero floor to preserve the shipped v2 scores.
+ * pinned to it. All timing summaries set a 2x floor for a missing bucket. The floor prevents
+ * a penalty derived from a fast bucket from beating the observed best on a slower bucket.
+ * A known bucket with no complete measurement contributes the same floor to every series.
  */
 function rankSeries<K>(
   samples: readonly SeriesSample<K>[],
   compareBuckets: (a: K, b: K) => number,
   penaltyFloorNs: number,
   missingRatioFloor: number,
   knownSeries: readonly string[] = [],
+  knownBuckets: readonly K[] = [],
 ): SeriesRanking[] {
-  const buckets = new Map<string, K>();
+  const buckets = new Map<string, K>(knownBuckets.map((bucket) => [String(bucket), bucket]));
   const valuesBySeries = new Map<string, Map<string, number>>();
   for (const series of knownSeries) {
     valuesBySeries.set(series, new Map<string, number>());
@@ -315,6 +315,7 @@ function rankSeries<K>(
     for (const [bucketKey] of sortedBuckets) {
       const base = bestByBucket.get(bucketKey);
       if (base === undefined) {
+        ratios.push(missingRatioFloor);
         continue;
       }
       const measuredValue = bucketValues.get(bucketKey);
@@ -345,11 +346,13 @@ function rankSeries<K>(
  * Correlated and uniform charts contribute to one dataset total. The legacy
  * `taxi` chart contributes to the same total as `taxi/correlated` and
  * `taxi/uniform`. A format must cover every chart in a dataset before that
- * dataset contributes to its score. Coverage describes complete datasets.
+ * dataset contributes a measured time. Incomplete datasets remain in the bucket universe
+ * for scoring and coverage, even when no format completes them.
  */
-function groupRandomAccessSamples(
-  samples: readonly SeriesSample<string>[],
-): SeriesSample<string>[] {
+function groupRandomAccessSamples(samples: readonly SeriesSample<string>[]): {
+  samples: SeriesSample<string>[];
+  datasets: string[];
+} {
   const chartsByDataset = new Map<string, Set<string>>();
   const groupsBySeries = new Map<string, Map<string, { value: number; charts: Set<string> }>>();
 
@@ -388,7 +391,7 @@ function groupRandomAccessSamples(
       }
     }
   }
-  return grouped;
+  return { samples: grouped, datasets: [...chartsByDataset.keys()] };
 }
 
 /**
@@ -444,11 +447,13 @@ async function collectRandomAccessSummary(): Promise<Summary | null> {
     const modeRows = rows.filter((row) => (row.open_mode ?? 'cached') === openMode);
     const grouped = groupRandomAccessSamples(modeRows);
     const knownSeries = [...new Set(modeRows.map((row) => row.series))];
-    return rankSeries(grouped, compareCodeUnits, 0, 2, knownSeries).map((ranking) => ({
-      ...ranking,
-      totalRuntime:
-        ranking.measured > 0 ? ranking.totalRuntime / ranking.measured : ranking.totalRuntime,
-    }));
+    return rankSeries(grouped.samples, compareCodeUnits, 0, 2, knownSeries, grouped.datasets).map(
+      (ranking) => ({
+        ...ranking,
+        totalRuntime:
+          ranking.measured > 0 ? ranking.totalRuntime / ranking.measured : ranking.totalRuntime,
+      }),
+    );
   };
   const hotRankings = rankingsFor('cached');
   const coldRankings = rankingsFor('reopen');
@@ -1021,7 +1026,7 @@ async function collectQuerySummary(
     rows.map((row) => ({ series: row.series, bucket: row.query_idx, value: row.value_ns })),
     (a, b) => a - b,
     QUERY_PENALTY_FLOOR_NS,
-    0,
+    2,
   );
   if (rankings.length === 0) {
     return null;