From df9041365450464c5c16cff2b42773f306fd550a Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Tue, 8 Sep 2026 11:38:32 -0400 Subject: [PATCH] Fix missing timing summary penalties and dataset coverage Signed-off-by: Connor Tsui --- web/lib/summary.test.ts | 129 ++++++++++++++++++++++++++++++++++++++++ web/lib/summary.ts | 37 +++++++----- 2 files changed, 150 insertions(+), 16 deletions(-) diff --git a/web/lib/summary.test.ts b/web/lib/summary.test.ts index f563482..9ab93e8 100644 --- a/web/lib/summary.test.ts +++ b/web/lib/summary.test.ts @@ -327,6 +327,90 @@ describe('timing summaries (shared ranking model)', () => { expect(byName.get('partial')?.totalRuntime).toBe(0); }); + it('counts datasets with complementary incomplete formats in each open mode', async () => { + query.mockResolvedValueOnce({ + rows: [ + { bucket: 'taxi', series: 'lance', open_mode: 'cached', value: 100_000 }, + { bucket: 'taxi', series: 'vortex', open_mode: 'cached', value: 200_000 }, + { + bucket: 'feature-vectors/correlated', + series: 'lance', + open_mode: 'cached', + value: 50_000, + }, + { + bucket: 'feature-vectors/uniform', + series: 'vortex', + open_mode: 'cached', + value: 300_000, + }, + { bucket: 'taxi', series: 'lance', open_mode: 'reopen', value: 400_000 }, + { bucket: 'taxi', series: 'vortex', open_mode: 'reopen', value: 200_000 }, + { + bucket: 'nested-structs/correlated', + series: 'lance', + open_mode: 'reopen', + value: 100_000, + }, + { bucket: 'nested-structs/uniform', series: 'vortex', open_mode: 'reopen', value: 500_000 }, + { bucket: 'feature-vectors', series: 'lance', open_mode: 'reopen', value: 200_000 }, + { bucket: 'feature-vectors', series: 'vortex', open_mode: 'reopen', value: 200_000 }, + ], + }); + + const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }); + if (summary === null || summary.type !== 'randomAccess') { + throw new Error('expected a randomAccess summary'); + } + + expect( + summary.hotRankings.map((ranking) => [ + ranking.name, + ranking.measured, + ranking.total, + ranking.totalRuntime, + ]), + ).toEqual([ + ['lance', 1, 2, 100_000], + ['vortex', 1, 2, 200_000], + ]); + expect(summary.hotRankings[0].score).toBeCloseTo(Math.sqrt(2), 6); + expect(summary.hotRankings[1].score).toBeCloseTo(Math.sqrt(2 * (200_010 / 100_010)), 6); + expect( + summary.coldRankings.map((ranking) => [ + ranking.name, + ranking.measured, + ranking.total, + ranking.totalRuntime, + ]), + ).toEqual([ + ['vortex', 2, 3, 200_000], + ['lance', 2, 3, 300_000], + ]); + expect(summary.coldRankings[0].score).toBeCloseTo(Math.cbrt(2), 6); + expect(summary.coldRankings[1].score).toBeCloseTo(Math.cbrt(2 * (400_010 / 200_010)), 6); + }); + + it('reports zero coverage when no random-access dataset is complete', async () => { + query.mockResolvedValueOnce({ + rows: [ + { bucket: 'taxi/correlated', series: 'lance', value: 100_000 }, + { bucket: 'taxi/uniform', series: 'vortex', value: 200_000 }, + ], + }); + + const summary = await collectGroupSummary({ k: 'RandomAccessGroup' }); + if (summary === null || summary.type !== 'randomAccess') { + throw new Error('expected a randomAccess summary'); + } + + expect(summary.hotRankings).toEqual([ + { name: 'lance', score: 2, measured: 0, total: 1, totalRuntime: 0 }, + { name: 'vortex', score: 2, measured: 0, total: 1, totalRuntime: 0 }, + ]); + expect(summary.coldRankings).toEqual([]); + }); + it('does not reward a series for skipping a slow bucket', async () => { query.mockResolvedValueOnce({ rows: [ @@ -396,6 +480,51 @@ describe('timing summaries (shared ranking model)', () => { expect(summary.rankings[1].totalRuntime).toBeCloseTo(10_000, 6); }); + it.each([ + { name: 'fast', missingQueryRuntime: 50_000, missingRatio: 600_010 / 50_010 }, + { name: 'slow', missingQueryRuntime: 100_000_000, missingRatio: 2 }, + ])( + 'penalizes a missing $name query without scoring it faster than the observed best', + async ({ missingQueryRuntime, missingRatio }) => { + query.mockResolvedValueOnce({ + rows: [ + { query_idx: 1, series: 'datafusion:partial', value_ns: 100_000 }, + { query_idx: 1, series: 'datafusion:complete', value_ns: 110_000 }, + { query_idx: 2, series: 'datafusion:complete', value_ns: missingQueryRuntime }, + ], + }); + + const summary = await collectGroupSummary({ + k: 'QueryGroup', + dataset: 'tpch', + dataset_variant: null, + scale_factor: null, + storage: 'nvme', + }); + if (summary === null || summary.type !== 'queryBenchmark') { + throw new Error('expected a queryBenchmark summary'); + } + const byName = new Map(summary.rankings.map((ranking) => [ranking.name, ranking])); + + expect(summary.rankings.map((ranking) => ranking.name)).toEqual([ + 'datafusion:complete', + 'datafusion:partial', + ]); + expect(byName.get('datafusion:partial')?.score).toBeCloseTo(Math.sqrt(missingRatio), 6); + expect(byName.get('datafusion:partial')).toMatchObject({ + measured: 1, + total: 2, + totalRuntime: 100_000, + }); + expect(byName.get('datafusion:complete')?.score).toBeCloseTo(Math.sqrt(110_010 / 100_010), 6); + expect(byName.get('datafusion:complete')).toMatchObject({ + measured: 2, + total: 2, + totalRuntime: 110_000 + missingQueryRuntime, + }); + }, + ); + it('summarizes every query group, with no dataset allowlist', async () => { // `spatialbench` (and every other suite outside the retired v2 five) used to // fall through to `null` and render no card at all. diff --git a/web/lib/summary.ts b/web/lib/summary.ts index 6f53aca..2e3323d 100644 --- a/web/lib/summary.ts +++ b/web/lib/summary.ts @@ -240,10 +240,9 @@ interface SeriesSample { * * The `(10 + value) / (10 + best)` ratio (rather than `value / best`) is v2's, * damping sub-10ns noise; it is preserved because the shipped query scores are - * pinned to it. Random-access and vector-search summaries also set a 2x floor - * for a missing bucket. The floor prevents a penalty derived from a fast bucket - * from beating a real measurement on a slower bucket. Query summaries keep a - * zero floor to preserve the shipped v2 scores. + * pinned to it. All timing summaries set a 2x floor for a missing bucket. The floor prevents + * a penalty derived from a fast bucket from beating the observed best on a slower bucket. + * A known bucket with no complete measurement contributes the same floor to every series. */ function rankSeries( samples: readonly SeriesSample[], @@ -251,8 +250,9 @@ function rankSeries( penaltyFloorNs: number, missingRatioFloor: number, knownSeries: readonly string[] = [], + knownBuckets: readonly K[] = [], ): SeriesRanking[] { - const buckets = new Map(); + const buckets = new Map(knownBuckets.map((bucket) => [String(bucket), bucket])); const valuesBySeries = new Map>(); for (const series of knownSeries) { valuesBySeries.set(series, new Map()); @@ -315,6 +315,7 @@ function rankSeries( for (const [bucketKey] of sortedBuckets) { const base = bestByBucket.get(bucketKey); if (base === undefined) { + ratios.push(missingRatioFloor); continue; } const measuredValue = bucketValues.get(bucketKey); @@ -345,11 +346,13 @@ function rankSeries( * Correlated and uniform charts contribute to one dataset total. The legacy * `taxi` chart contributes to the same total as `taxi/correlated` and * `taxi/uniform`. A format must cover every chart in a dataset before that - * dataset contributes to its score. Coverage describes complete datasets. + * dataset contributes a measured time. Incomplete datasets remain in the bucket universe + * for scoring and coverage, even when no format completes them. */ -function groupRandomAccessSamples( - samples: readonly SeriesSample[], -): SeriesSample[] { +function groupRandomAccessSamples(samples: readonly SeriesSample[]): { + samples: SeriesSample[]; + datasets: string[]; +} { const chartsByDataset = new Map>(); const groupsBySeries = new Map }>>(); @@ -388,7 +391,7 @@ function groupRandomAccessSamples( } } } - return grouped; + return { samples: grouped, datasets: [...chartsByDataset.keys()] }; } /** @@ -444,11 +447,13 @@ async function collectRandomAccessSummary(): Promise { const modeRows = rows.filter((row) => (row.open_mode ?? 'cached') === openMode); const grouped = groupRandomAccessSamples(modeRows); const knownSeries = [...new Set(modeRows.map((row) => row.series))]; - return rankSeries(grouped, compareCodeUnits, 0, 2, knownSeries).map((ranking) => ({ - ...ranking, - totalRuntime: - ranking.measured > 0 ? ranking.totalRuntime / ranking.measured : ranking.totalRuntime, - })); + return rankSeries(grouped.samples, compareCodeUnits, 0, 2, knownSeries, grouped.datasets).map( + (ranking) => ({ + ...ranking, + totalRuntime: + ranking.measured > 0 ? ranking.totalRuntime / ranking.measured : ranking.totalRuntime, + }), + ); }; const hotRankings = rankingsFor('cached'); const coldRankings = rankingsFor('reopen'); @@ -1021,7 +1026,7 @@ async function collectQuerySummary( rows.map((row) => ({ series: row.series, bucket: row.query_idx, value: row.value_ns })), (a, b) => a - b, QUERY_PENALTY_FLOOR_NS, - 0, + 2, ); if (rankings.length === 0) { return null;