From bcfef280f8c97a5578c273d62ac0e904bbc2dbd2 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Tue, 22 Sep 2026 17:04:52 -0400 Subject: [PATCH] Corgi reduce: merge unit-valued runs without a sort `merge_present` merges presented chunk runs straight into the bridge when they arrive in `((key, value_id), time)` order, and otherwise consolidates. It recognized leaf values as ordered but not `Unit` values, though all unit rows share one id. Unit-valued presentations (`distinct`'s output, and unit-valued inputs such as reach's `join(edges, ($2 ;)) | distinct`) paid for a sort. The merge also reserves its bridge up front, as the fallback's `extend` does, rather than growing it by doubling. The `ordered_ids` line is the same as in #870, which predates the `IdPool` refactor. Same-plan harness, 1 worker, median of 3 (before -> after): reach 2M edges, 1000 x 100 load 767 -> 679 ms churn 85.2 -> 76.4 ms/round scc 200k edges, 100 x 50 load 674 -> 666 ms churn 35.6 -> 35.4 distinct 1M rows, 1000 x 500 load 87.5 -> 84.9 ms churn 2.51 -> 2.45 count 1M rows, 1000 x 500 load 88.5 -> 85.7 ms churn 3.62 -> 3.54 Peak RSS: a 4M-row count or distinct load 1289 -> 1145 MiB (the reserve); reach 609 -> 640 MiB, not traced. Co-Authored-By: Claude Opus 5.5 (1M context) --- interactive/src/corgi/reduce.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/interactive/src/corgi/reduce.rs b/interactive/src/corgi/reduce.rs index 893202f64..ff539a2c4 100644 --- a/interactive/src/corgi/reduce.rs +++ b/interactive/src/corgi/reduce.rs @@ -280,8 +280,8 @@ where (keys_col, vals_col, khs, times, diffs, run_ends) } -/// Merge already-ordered selected chunk runs directly into an empty proxy bridge. Leaf values -/// preserve value-id order. Keys may either be identity-id leaves or carried-hash columns, provided +/// Merge already-ordered selected chunk runs directly into an empty proxy bridge. Leaf and unit +/// values preserve value-id order. Keys may either be identity-id leaves or carried-hash columns, provided /// no one chunk run contains two real keys under the same hash; in the latter case the real-key /// tie-break would interrupt proxy `(key_id, value_id, time)` order, so we fall back to ordinary /// consolidation. A debug assertion audits the inferred order. Returns false when the inference @@ -302,10 +302,12 @@ fn merge_present( one_real_key_per_id }) && start == khs.len() }; - let ordered_ids = ordered_keys && corgi::arrange::leaf_slice(vals_col).is_some(); + let ordered_ids = ordered_keys && (matches!(vals_col, CValue::Unit(_)) || corgi::arrange::leaf_slice(vals_col).is_some()); if !ordered_ids || !bridge.is_empty() { return false; } + // At most one record per presented row, reserved up front rather than grown by doubling. + bridge.reserve(khs.len()); debug_assert!({ let mut start = 0usize; @@ -832,6 +834,18 @@ mod tests { } } + /// Unit values all share one id, so the runs merge directly, here cancelling across runs. + #[test] + fn merge_present_accepts_unit_values() { + let vals = CValue::Unit(4); + let vids = ids(&vals); + let mut bridge = Vec::new(); + assert!(merge_present( + &CValue::u64(vec![1, 3, 1, 2]), &vals, &[1, 3, 1, 2], &vids, &mut [0u64, 0, 0, 0], &[1, 2, -1, 5], &[2, 4], &mut bridge, + )); + assert_eq!(bridge, vec![((2, vids[0]), 0, 5), ((3, vids[0]), 0, 2)]); + } + #[test] fn merge_present_rejects_a_compound_hash_collision_within_a_run() { let keys = compound_keys(vec![1, 1], vec![7, 8]);