From 1f2fad0ffb091317e3ef2de0f8ff725f920b9dbb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 13:07:52 +0000 Subject: [PATCH 1/5] =?UTF-8?q?mask-risc:=20absolute=20execution=20extent?= =?UTF-8?q?=20=E2=80=94=20execute=5Fextent=20over=20[lo,=20hi)=20without?= =?UTF-8?q?=20rebasing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit execute_into is now execute_extent over 0..n_rows. A partial extent is an outer restriction in the same absolute row coordinates as Planes: only the words it touches are visited (extent_tiles), a cut word is restricted in a register at the terminal, lanes and masks are never copied or rebased, and the #1268 fused fold intersects the program's range with the extent. Partial extents accept Count/Any/All/MaskedSum/Min/Max (shipped merge laws) and Keep (writes only in-extent bits of its absolute Out::Mask); anything else is ExtentUnsupported. Foreign planes are key-addressed and never sliced. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GXUahz73MZxtxWcfpHp9dG --- crates/lance-graph-mask-risc/src/exec.rs | 225 ++++++- crates/lance-graph-mask-risc/src/ir.rs | 9 +- crates/lance-graph-mask-risc/src/lib.rs | 3 +- crates/lance-graph-mask-risc/src/value.rs | 12 + crates/lance-graph-mask-risc/tests/extent.rs | 636 +++++++++++++++++++ 5 files changed, 863 insertions(+), 22 deletions(-) create mode 100644 crates/lance-graph-mask-risc/tests/extent.rs diff --git a/crates/lance-graph-mask-risc/src/exec.rs b/crates/lance-graph-mask-risc/src/exec.rs index 474812c43..919c72bbe 100644 --- a/crates/lance-graph-mask-risc/src/exec.rs +++ b/crates/lance-graph-mask-risc/src/exec.rs @@ -38,13 +38,14 @@ use ndarray::simd::{ }; use crate::ir::{ - touched_words, Foreign, FusedFold, FusedTerminal, GroupFold, GroupKey, LaneRef, MaskOp, - Operand, Planes, Pred, Program, Terminal, MAX_SCRATCH_SLOTS, + span_words, touched_words, Foreign, FusedFold, FusedTerminal, GroupFold, GroupKey, LaneRef, + MaskOp, Operand, Planes, Pred, Program, Terminal, MAX_SCRATCH_SLOTS, }; use crate::reference::{out_shape, validate}; use crate::ternlog_dispatch::{ternlog_dispatch, ternlog_dispatch_assign}; use crate::value::{ExecError, Out, Value}; use crate::words_for; +use core::ops::Range; /// Where a [`Scratch`]'s words live: owned by the arena, or borrowed from a /// buffer the caller grows and keeps. @@ -657,11 +658,12 @@ fn slots_needed(program: &Program) -> usize { } } -/// The bits of word `w` that fall inside `[lo, hi)`. -fn edge_mask(w: usize, lo: u32, hi: u32) -> u64 { +/// The bits of word `w` that fall inside `[lo, hi)` (absolute rows; `w` must +/// be a word the range touches, so `hi > w * 64`). +fn edge_mask(w: usize, lo: usize, hi: usize) -> u64 { let base = w * 64; - let from = (lo as usize).saturating_sub(base).min(64); - let to = (hi as usize - base).min(64); + let from = lo.saturating_sub(base).min(64); + let to = (hi - base).min(64); let upper = if to == 64 { u64::MAX } else { (1u64 << to) - 1 }; upper & (u64::MAX << from) } @@ -688,8 +690,9 @@ fn run_fused(f: FusedTerminal, planes: &Planes<'_>) -> Value { } let plane = planes.masks[usize::from(p)]; let (first, last) = (span.start, span.end - 1); - let head = [plane[first] & edge_mask(first, f.lo, f.hi)]; - let tail = [plane[last] & edge_mask(last, f.lo, f.hi)]; + let (lo, hi) = (f.lo as usize, f.hi as usize); + let head = [plane[first] & edge_mask(first, lo, hi)]; + let tail = [plane[last] & edge_mask(last, lo, hi)]; let interior = if last > first + 1 { &plane[first + 1..last] } else { @@ -709,6 +712,86 @@ fn run_fused(f: FusedTerminal, planes: &Planes<'_>) -> Value { } } +/// The tiles an execution over the ABSOLUTE row extent `[lo, hi)` visits, +/// each as `(word range, edge)`. +/// +/// The extent is an outer restriction in the same row coordinates as +/// [`Planes`]: tile word `w` is word `w` of every resident mask, and its rows +/// are `w * 64 ..` of every lane — nothing is rebased, copied, or renumbered. +/// Only the words [`span_words`] says the extent touches are visited. A word +/// the extent cuts (an unaligned `lo`, or an unaligned `hi` short of +/// `n_rows`) is its own one-word tile carrying `Some(edge)`, the in-extent +/// bits the terminal is restricted to; every other tile is at most +/// `tile_words` whole words and carries `None`. For the whole population +/// `[0, n_rows)` there is no edge, and the tiles are exactly the ones +/// whole-population execution has always walked. +/// +/// Public so the structural claim — work scales with the extent, not with +/// `n_rows` — is checkable against the plan the executor actually iterates. +pub fn extent_tiles(n_rows: usize, tile_words: usize, extent: Range) -> ExtentTiles { + let span = span_words(extent.start, extent.end); + ExtentTiles { + first: span.start, + cur: span.start, + end: span.end, + lo: extent.start, + hi: extent.end, + head_edge: extent.start % 64 != 0, + tail_edge: extent.end % 64 != 0 && extent.end < n_rows, + tile_words: tile_words.max(1), + } +} + +/// Iterator returned by [`extent_tiles`]. +#[derive(Debug, Clone)] +pub struct ExtentTiles { + first: usize, + cur: usize, + end: usize, + lo: usize, + hi: usize, + head_edge: bool, + tail_edge: bool, + tile_words: usize, +} + +impl Iterator for ExtentTiles { + type Item = (Range, Option); + + fn next(&mut self) -> Option { + if self.cur >= self.end { + return None; + } + let w = self.cur; + if (w == self.first && self.head_edge) || (w + 1 == self.end && self.tail_edge) { + self.cur += 1; + return Some((w..w + 1, Some(edge_mask(w, self.lo, self.hi)))); + } + let stop = if self.tail_edge { + self.end - 1 + } else { + self.end + }; + let n = self.tile_words.min(stop - w); + self.cur += n; + Some((w..w + n, None)) + } +} + +/// A terminal's mask on an edge tile, restricted to the extent. `fill` +/// reads the out-of-extent rows as SET — the identity of `All` — every other +/// terminal reads them as CLEAR. A one-word register temporary; nothing +/// resident is written. +fn clip<'a>(m: &'a [u64], edge: Option, buf: &'a mut [u64; 1], fill: bool) -> &'a [u64] { + match edge { + None => m, + Some(e) => { + buf[0] = if fill { m[0] | !e } else { m[0] & e }; + &buf[..] + } + } +} + /// Fold one tile's `Option` reduction into the running one. fn fold_opt(acc: Option, tile: Option, f: fn(i32, i32) -> i32) -> Option { match (acc, tile) { @@ -743,11 +826,47 @@ fn fold_opt(acc: Option, tile: Option, f: fn(i32, i32) -> i32) -> Opti /// from its slot with `out: Out::None`; under a narrower scratch `Keep` /// requires `Out::Mask` (`TerminalNeedsOut`). pub fn execute_into( + program: &Program, + planes: &Planes<'_>, + foreign: &Foreign<'_>, + scratch: &mut Scratch<'_>, + out: Out<'_>, +) -> Result { + execute_extent(program, planes, foreign, scratch, out, 0..planes.n_rows) +} + +/// [`execute_into`] restricted to the ABSOLUTE row extent `[lo, hi)`. +/// +/// **The extent is an outer restriction, never a rebasing.** It lives in the +/// same row coordinates as `planes`: a `Pred::Range { lo: 1000, hi: 2000 }` +/// executed over the extent `1500..1700` means `[1000, 2000) ∩ [1500, 1700)`, +/// and lane element `r` is row `r` whatever the extent. The program's +/// meaning is unchanged; only the population it is evaluated over shrinks. +/// +/// Work is proportional to the extent: only the tiles [`extent_tiles`] +/// yields are visited, and a word the extent cuts is restricted in a +/// register at the terminal. No lane or mask is copied or rebased. +/// +/// `execute_into` is this call with `0..n_rows`, which accepts every +/// terminal. A partial extent accepts the terminals whose per-extent results +/// merge by a shipped law — `Count` (sum), `Any` (or), `All` (and), +/// `MaskedSumI32` (sum), `MaskedMinI32` / `MaskedMaxI32` (min / max) — plus +/// `Keep`, which writes only the in-extent bits of its population-addressed +/// [`Out::Mask`] and leaves every other bit as the caller holds it (so +/// disjoint extents compose into one buffer in any order). Anything else is +/// [`ExecError::ExtentUnsupported`]; `lo > hi` or `hi > n_rows` is +/// [`ExecError::ExtentOutOfRange`]. Both are refused before execution. +/// +/// Foreign planes and lanes ([`MaskOp::Gather`], `GroupKey::Via`) are +/// addressed by KEY, not by this table's rows, so the extent never slices +/// them. +pub fn execute_extent( program: &Program, planes: &Planes<'_>, foreign: &Foreign<'_>, scratch: &mut Scratch<'_>, mut out: Out<'_>, + extent: Range, ) -> Result { // BEFORE the capacity check, not after: an over-declared count is a lie // about the PROGRAM, and the caller's buffer is irrelevant to it. Checked @@ -761,6 +880,36 @@ pub fn execute_into( declared: program.scratch_slots, }); } + let (elo, ehi) = (extent.start, extent.end); + if elo > ehi || ehi > planes.n_rows { + return Err(ExecError::ExtentOutOfRange { + lo: elo, + hi: ehi, + n_rows: planes.n_rows, + }); + } + let whole = elo == 0 && ehi == planes.n_rows; + if !whole { + let refused = match program.terminal { + Terminal::Count { .. } + | Terminal::Any { .. } + | Terminal::All { .. } + | Terminal::MaskedSumI32 { .. } + | Terminal::MaskedMinI32 { .. } + | Terminal::MaskedMaxI32 { .. } + | Terminal::Keep { .. } => None, + Terminal::BlendI32 { .. } => Some("BlendI32"), + Terminal::ScatterOrU32 { .. } => Some("ScatterOrU32"), + Terminal::ScatterCountU32 { .. } => Some("ScatterCountU32"), + Terminal::CountKeyRunsU32 { .. } => Some("CountKeyRunsU32"), + Terminal::GroupSumI32 { .. } => Some("GroupSumI32"), + Terminal::GroupSumViaI32 { .. } => Some("GroupSumViaI32"), + Terminal::GroupReduce { .. } => Some("GroupReduce"), + }; + if let Some(what) = refused { + return Err(ExecError::ExtentUnsupported { what }); + } + } // A fused program folds from its operands: it reads no slot and writes no // membership bit, so the scratch capacity checks below do not apply to it. // Validation stays total — the one declared slot is tracked in a local @@ -768,6 +917,18 @@ pub fn execute_into( if let Some(f) = program.fused_terminal() { let mut written = [0u64; 1]; validate(program, planes, foreign, out_shape(&out), &mut written)?; + // The extent composes with the program's own range by intersection, + // in absolute rows — the #1268 fold, over a narrower span. + let (a, b) = ((f.lo as usize).max(elo), (f.hi as usize).min(ehi)); + let f = if a < b { + FusedTerminal { + lo: a as u32, + hi: b as u32, + ..f + } + } else { + FusedTerminal { hi: f.lo, ..f } + }; return Ok(run_fused(f, planes)); } if scratch.slots() < program.scratch_slots as usize { @@ -797,7 +958,7 @@ pub fn execute_into( // for every terminal) would return `Value::Mask` over a partial result. if matches!(program.terminal, Terminal::Keep { .. }) && !matches!(out, Out::Mask(_)) - && tw < words + && (tw < words || !whole) { return Err(ExecError::TerminalNeedsOut { what: "Keep" }); } @@ -824,9 +985,8 @@ pub fn execute_into( let mut run_carry = KeyRunCarry::default(); let mut runs = 0usize; - let mut w0 = 0usize; - while w0 < words { - let tws = tw.min(words - w0); + for (span, edge) in extent_tiles(n_rows, tw, elo..ehi) { + let (w0, tws) = (span.start, span.len()); let r0 = w0 * 64; let t = Tile { w0, @@ -834,6 +994,8 @@ pub fn execute_into( r0, rows: (n_rows - r0).min(tws * 64), }; + // The register an edge tile's terminal mask is restricted in. + let mut eb = [0u64; 1]; for op in &program.ops { match *op { MaskOp::Pred { pred, under, dst } => { @@ -968,24 +1130,42 @@ pub fn execute_into( let slots = scratch.all(); match program.terminal { Terminal::Count { mask } => { - count += popcount_batch_u64(read(planes, &slots, mask, t)) as usize; + count += + popcount_batch_u64(clip(read(planes, &slots, mask, t), edge, &mut eb, false)) + as usize; + } + Terminal::Any { mask } => { + any |= mask_any(clip(read(planes, &slots, mask, t), edge, &mut eb, false)) + } + Terminal::All { mask } => { + all &= mask_all( + clip(read(planes, &slots, mask, t), edge, &mut eb, true), + t.rows, + ) } - Terminal::Any { mask } => any |= mask_any(read(planes, &slots, mask, t)), - Terminal::All { mask } => all &= mask_all(read(planes, &slots, mask, t), t.rows), Terminal::MaskedSumI32 { mask, lane } => { - sum += masked_sum_i32(lane_i32(planes, lane, t), read(planes, &slots, mask, t)); + sum += masked_sum_i32( + lane_i32(planes, lane, t), + clip(read(planes, &slots, mask, t), edge, &mut eb, false), + ); } Terminal::MaskedMinI32 { mask, lane } => { min = fold_opt( min, - masked_min_i32(lane_i32(planes, lane, t), read(planes, &slots, mask, t)), + masked_min_i32( + lane_i32(planes, lane, t), + clip(read(planes, &slots, mask, t), edge, &mut eb, false), + ), i32::min, ); } Terminal::MaskedMaxI32 { mask, lane } => { max = fold_opt( max, - masked_max_i32(lane_i32(planes, lane, t), read(planes, &slots, mask, t)), + masked_max_i32( + lane_i32(planes, lane, t), + clip(read(planes, &slots, mask, t), edge, &mut eb, false), + ), i32::max, ); } @@ -1134,11 +1314,16 @@ pub fn execute_into( // scratch is single-tile (checked above) and the slot IS the // result. if let Out::Mask(o) = &mut out { - o[t.w0..t.w0 + t.words].copy_from_slice(read(planes, &slots, mask, t)); + let m = read(planes, &slots, mask, t); + match edge { + None => o[t.w0..t.w0 + t.words].copy_from_slice(m), + // An edge word writes only its in-extent bits: the + // neighbour extent's bits in the same word survive. + Some(e) => o[t.w0] = (o[t.w0] & !e) | (m[0] & e), + } } } } - w0 += tws; } Ok(match program.terminal { diff --git a/crates/lance-graph-mask-risc/src/ir.rs b/crates/lance-graph-mask-risc/src/ir.rs index c18173367..31d84e062 100644 --- a/crates/lance-graph-mask-risc/src/ir.rs +++ b/crates/lance-graph-mask-risc/src/ir.rs @@ -691,10 +691,17 @@ pub enum FusedFold { /// otherwise `floor(lo / 64) ..= floor((hi - 1) / 64)`. One spelling, used by /// the fused executor and by the tests that pin the touched-word law. pub fn touched_words(lo: u32, hi: u32) -> core::ops::Range { + span_words(lo as usize, hi as usize) +} + +/// [`touched_words`] over `usize` rows — the SAME law, for an execution +/// extent, whose bounds are `Planes::n_rows`-typed rather than `Pred::Range`- +/// typed. `touched_words` delegates here, so there is one spelling. +pub(crate) fn span_words(lo: usize, hi: usize) -> core::ops::Range { if lo >= hi { return 0..0; } - (lo as usize / 64)..((hi as usize - 1) / 64 + 1) + (lo / 64)..((hi - 1) / 64 + 1) } /// Per-kind op counts of a program. diff --git a/crates/lance-graph-mask-risc/src/lib.rs b/crates/lance-graph-mask-risc/src/lib.rs index 8fd51bb2c..5e13343d5 100644 --- a/crates/lance-graph-mask-risc/src/lib.rs +++ b/crates/lance-graph-mask-risc/src/lib.rs @@ -119,7 +119,8 @@ pub mod ternlog_dispatch; pub mod value; pub use exec::{ - execute, execute_into, materialize_rows, scratch_words_for, tile_words_for, Scratch, TILE_WORDS, + execute, execute_extent, execute_into, extent_tiles, materialize_rows, scratch_words_for, + tile_words_for, ExtentTiles, Scratch, TILE_WORDS, }; pub use fuse::{fuse, fuse_program, ternlog_imm, BoolExpr, FuseError, Fused}; pub use ir::{ diff --git a/crates/lance-graph-mask-risc/src/value.rs b/crates/lance-graph-mask-risc/src/value.rs index 42fd40871..5e134f3be 100644 --- a/crates/lance-graph-mask-risc/src/value.rs +++ b/crates/lance-graph-mask-risc/src/value.rs @@ -170,4 +170,16 @@ pub enum ExecError { /// kept apart because `BlendI32` predates `Out` and every existing /// caller already matches on the old variant. TerminalNeedsOut { what: &'static str }, + /// An execution extent with `lo > hi` or `hi > n_rows`. The extent is an + /// outer restriction in the SAME absolute row coordinates as + /// [`crate::Planes`]; it can narrow the population, never name rows that + /// do not exist. Refused before any execution. + ExtentOutOfRange { lo: usize, hi: usize, n_rows: usize }, + /// A partial execution extent over a terminal whose per-extent results + /// have no shipped merge law here (or whose sink is not a disjoint + /// write). The whole-population extent `[0, n_rows)` accepts every + /// terminal; a partial one accepts `Count`, `Any`, `All`, + /// `MaskedSumI32`, `MaskedMinI32`, `MaskedMaxI32` and `Keep`. `what` + /// names the refused terminal. + ExtentUnsupported { what: &'static str }, } diff --git a/crates/lance-graph-mask-risc/tests/extent.rs b/crates/lance-graph-mask-risc/tests/extent.rs new file mode 100644 index 000000000..3874efddc --- /dev/null +++ b/crates/lance-graph-mask-risc/tests/extent.rs @@ -0,0 +1,636 @@ +//! Absolute execution extent. +//! +//! `execute_extent(.., lo..hi)` evaluates the SAME program over the rows of +//! `[lo, hi)`, in the same absolute row coordinates as `Planes`. It is an +//! outer restriction: a program `Range [1000, 2000)` over the extent +//! `[1500, 1700)` means `[1000, 2000) ∩ [1500, 1700)`, and lane element `r` +//! is row `r` whatever the extent. +//! +//! The gates: +//! - the edge matrix: every result equals a scalar oracle that knows nothing +//! about tiles, words or edges; +//! - split composition: whole == the merge of any partition, in any order; +//! - the non-rebasing falsifier: an extent that does not start at row 0 must +//! read the absolute rows, and the rebased reading is shown to differ; +//! - the structural gate: the tiles visited scale with the extent, not with +//! `n_rows`; +//! - foreign planes are addressed by key and are never sliced by the extent. + +use lance_graph_mask_risc::exec::{execute_extent, execute_into, Scratch}; +use lance_graph_mask_risc::{ + extent_tiles, touched_words, ExecError, Foreign, ForeignPlane, LaneRef, MaskOp, Operand, Out, + Planes, Pred, Program, Terminal, Value, TILE_WORDS, +}; + +fn lcg(seed: &mut u64) -> u64 { + *seed = seed + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + *seed >> 11 +} + +fn words(n: usize) -> usize { + n.div_ceil(64) +} + +fn plane(n: usize, set: impl Fn(usize) -> bool) -> Vec { + let mut w = vec![0u64; words(n)]; + for r in (0..n).filter(|&r| set(r)) { + w[r / 64] |= 1 << (r % 64); + } + w +} + +fn bit(p: &[u64], r: usize) -> bool { + p[r / 64] >> (r % 64) & 1 == 1 +} + +/// The two physical shapes a `Range ∩ plane` program can take: the #1268 +/// fused fold, and the tiled path that writes the relation into scratch. +#[derive(Clone, Copy, Debug)] +enum Shape { + Fused, + Tiled, +} + +#[derive(Clone, Copy, Debug)] +enum Term { + Count, + Any, + All, + Keep, +} + +fn program(shape: Shape, lo: u32, hi: u32, term: Term) -> Program { + let (ops, m) = match shape { + Shape::Fused => ( + vec![MaskOp::Pred { + pred: Pred::Range { lo, hi }, + under: Some(Operand::Plane(0)), + dst: 0, + }], + Operand::Scratch(0), + ), + Shape::Tiled => ( + vec![ + MaskOp::Pred { + pred: Pred::Range { lo, hi }, + under: None, + dst: 0, + }, + MaskOp::And { + a: Operand::Scratch(0), + b: Operand::Plane(0), + dst: 1, + }, + ], + Operand::Scratch(1), + ), + }; + let terminal = match term { + Term::Count => Terminal::Count { mask: m }, + Term::Any => Terminal::Any { mask: m }, + Term::All => Terminal::All { mask: m }, + Term::Keep => Terminal::Keep { mask: m }, + }; + Program::new(ops, terminal) +} + +const POISON: u64 = 0xA5A5_5A5A_C3C3_3C3C; + +/// Run `p` over `ext`. `Keep` writes into a poisoned buffer, which is +/// returned so the caller can check which bits were touched. +fn run(p: &Program, planes: &Planes<'_>, ext: std::ops::Range) -> (Value, Option>) { + let n = planes.n_rows; + let mut s = Scratch::for_program(p, n).expect("scratch"); + if matches!(p.terminal, Terminal::Keep { .. }) { + let mut o = vec![POISON; words(n)]; + let v = execute_extent(p, planes, &Foreign::NONE, &mut s, Out::Mask(&mut o), ext) + .expect("keep"); + (v, Some(o)) + } else { + let v = execute_extent(p, planes, &Foreign::NONE, &mut s, Out::None, ext).expect("execute"); + (v, None) + } +} + +/// Check one (program range, extent, plane, shape, terminal) case against a +/// scalar oracle over absolute rows. +fn check(pl: &[u64], n: usize, prog: (u32, u32), ext: (usize, usize), shape: Shape, term: Term) { + let masks: [&[u64]; 1] = [pl]; + let planes = Planes { + n_rows: n, + masks: &masks, + lanes: &[], + }; + let p = program(shape, prog.0, prog.1, term); + let member = |r: usize| (prog.0 as usize..prog.1 as usize).contains(&r) && bit(pl, r); + let in_ext = |r: usize| (ext.0..ext.1).contains(&r); + let tag = format!("{shape:?} {term:?} n={n} prog={prog:?} ext={ext:?}"); + let (v, out) = run(&p, &planes, ext.0..ext.1); + match term { + Term::Count => { + let want = (ext.0..ext.1).filter(|&r| member(r)).count(); + assert_eq!(v, Value::Count(want), "{tag}"); + } + Term::Any => { + let want = (ext.0..ext.1).any(member); + assert_eq!(v, Value::Bool(want), "{tag}"); + } + Term::All => { + let want = (ext.0..ext.1).all(member); + assert_eq!(v, Value::Bool(want), "{tag}"); + } + Term::Keep => { + let o = out.expect("keep writes out"); + for r in 0..n { + let want = if in_ext(r) { + member(r) + } else { + // Outside the extent the caller's bits survive. + POISON >> (r % 64) & 1 == 1 + }; + assert_eq!(bit(&o, r), want, "{tag} row {r}"); + } + } + } +} + +#[test] +fn the_edge_matrix_agrees_with_the_scalar_oracle() { + let n = 1317; // not a multiple of 64, three 8-word tiles + let mut seed = 0xE17E_u64; + let scattered: Vec = (0..n).map(|_| lcg(&mut seed) % 3 == 0).collect(); + let planes: [(&str, Vec); 3] = [ + ("zero", plane(n, |_| false)), + ("ones", plane(n, |_| true)), + ("scattered", plane(n, |r| scattered[r])), + ]; + let extents: &[(usize, usize)] = &[ + (0, 0), // empty at zero + (65, 65), // empty mid-word + (0, n), // whole + (5, 6), // one row + (3, 64), // unaligned start, aligned end + (0, 70), // aligned start, unaligned end + (3, 70), // both unaligned, cross-word + (70, 75), // single word + (64, 128), // one aligned word pair + (1, 511), // inside one tile + (5, 600), // cross-tile + (n - 30, n), // tail extent, N % 64 != 0 + (n - 1, n), // last row + (300, 500), // the named program-relation cases below + (100, 900), + ]; + let progs: &[(u32, u32)] = &[ + (0, n as u32), // program contains every extent + (100, 900), // contains (300,500); partial vs most + (100, 200), // disjoint from (300,500) + (300, 500), // equal to one extent, inside (100,900) + (250, 1300), // partial overlap with (100,900) + (65, 65), // empty program range + ]; + let mut cases = 0usize; + for (_, pl) in &planes { + for &prog in progs { + for &ext in extents { + for shape in [Shape::Fused, Shape::Tiled] { + let terms: &[Term] = match shape { + // The fused seam is Count/Any only; All and Keep + // stay on the tiled path by construction. + Shape::Fused => &[Term::Count, Term::Any], + Shape::Tiled => &[Term::Count, Term::Any, Term::All, Term::Keep], + }; + for &term in terms { + check(pl, n, prog, ext, shape, term); + cases += 1; + } + } + } + } + } + assert_eq!(cases, 3 * 6 * 15 * 6); +} + +#[test] +fn a_tiny_extent_in_a_million_rows_reads_absolute_rows() { + let n = 1 << 20; + let pl = plane(n, |r| r % 5 != 0); + for shape in [Shape::Fused, Shape::Tiled] { + for ext in [(500_001, 500_002), (500_001, 500_065), (777_000, 777_512)] { + for prog in [(0, n as u32), (500_000, 600_000), (1, 500_001)] { + check(&pl, n, prog, ext, shape, Term::Count); + check(&pl, n, prog, ext, shape, Term::Any); + } + } + } +} + +/// Program `Range [1000, 2000)` over the extent `[1500, 1700)` is exactly +/// rows `1500..1700`. Both rebased readings — the program's range taken +/// relative to the extent start, or the extent taken relative to the +/// program's start — select a different set and are shown to. +#[test] +fn the_extent_restricts_and_never_rebases_the_program() { + let n = 4133; + let ones = plane(n, |_| true); + let masks: [&[u64]; 1] = [&ones]; + let planes = Planes { + n_rows: n, + masks: &masks, + lanes: &[], + }; + let absolute = (1500..1700).filter(|r| (1000..2000).contains(r)).count(); + let program_rebased = (1500..1700).filter(|r| (2500..3500).contains(r)).count(); + let extent_rebased = (1000..1200).filter(|r| (1000..2000).contains(r)).count(); + assert_eq!(absolute, 200); + assert_ne!(program_rebased, absolute); + for shape in [Shape::Fused, Shape::Tiled] { + let (v, _) = run( + &program(shape, 1000, 2000, Term::Count), + &planes, + 1500..1700, + ); + assert_eq!(v, Value::Count(absolute), "{shape:?}"); + } + // Keep: exactly rows 1500..1700 carry the relation, in absolute words. + let (_, out) = run( + &program(Shape::Tiled, 1000, 2000, Term::Keep), + &planes, + 1500..1700, + ); + let o = out.expect("keep"); + for r in 0..n { + let want = if (1500..1700).contains(&r) { + true + } else { + POISON >> (r % 64) & 1 == 1 + }; + assert_eq!(bit(&o, r), want, "row {r}"); + } + // The extent-rebased reading would put the selected bits at 1000..1200. + assert_eq!(extent_rebased, 200); + assert!( + (1000..1200).any(|r| bit(&o, r) != ((1000..1200).contains(&r))), + "the Keep result must not look like the extent-rebased reading" + ); +} + +/// Distinct lane values at the word seams: an extent starting at 65 or 129 +/// must sum those ABSOLUTE rows. A worker-local reading (lane element 0 is +/// the extent's first row) sums a different window, and the test proves the +/// two differ, so a rebasing executor cannot pass. +#[test] +fn values_are_read_at_absolute_rows_not_worker_local_ones() { + let n = 1317; + let mut lane: Vec = (0..n as i32).map(|r| r % 7).collect(); + for (i, r) in [63usize, 64, 65, 127, 128, 129].into_iter().enumerate() { + lane[r] = 1000 * (i as i32 + 1); + } + let ones = plane(n, |_| true); + let masks: [&[u64]; 1] = [&ones]; + let lanes = [LaneRef::I32(&lane)]; + let planes = Planes { + n_rows: n, + masks: &masks, + lanes: &lanes, + }; + let p = Program::new( + vec![MaskOp::Pred { + pred: Pred::Range { + lo: 0, + hi: n as u32, + }, + under: Some(Operand::Plane(0)), + dst: 0, + }], + Terminal::MaskedSumI32 { + mask: Operand::Scratch(0), + lane: 0, + }, + ); + let sum = |r: std::ops::Range| lane[r].iter().map(|&v| i64::from(v)).sum::(); + for (lo, hi) in [(65usize, 200usize), (129, 300), (63, 66), (127, 130)] { + let absolute = sum(lo..hi); + let worker_local = sum(0..hi - lo); + assert_ne!( + absolute, worker_local, + "the falsifier must separate the readings at {lo}" + ); + let mut s = Scratch::for_program(&p, n).expect("scratch"); + let v = + execute_extent(&p, &planes, &Foreign::NONE, &mut s, Out::None, lo..hi).expect("sum"); + assert_eq!(v, Value::SumI64(absolute), "extent {lo}..{hi}"); + } +} + +/// Merge the partial results of a partition with the terminal's shipped law. +fn merge(a: Value, b: Value) -> Value { + match (a, b) { + (Value::Count(x), Value::Count(y)) => Value::Count(x + y), + (Value::SumI64(x), Value::SumI64(y)) => Value::SumI64(x + y), + _ => panic!("merge: use merge_bool / merge_opt for {a:?} {b:?}"), + } +} + +#[derive(Clone, Copy)] +enum Law { + Add, + Or, + And, + Min, + Max, +} + +fn merge_with(law: Law, a: Value, b: Value) -> Value { + match (law, a, b) { + (Law::Add, a, b) => merge(a, b), + (Law::Or, Value::Bool(x), Value::Bool(y)) => Value::Bool(x || y), + (Law::And, Value::Bool(x), Value::Bool(y)) => Value::Bool(x && y), + (Law::Min, Value::OptI32(x), Value::OptI32(y)) => Value::OptI32(match (x, y) { + (Some(x), Some(y)) => Some(x.min(y)), + (x, None) => x, + (None, y) => y, + }), + (Law::Max, Value::OptI32(x), Value::OptI32(y)) => Value::OptI32(match (x, y) { + (Some(x), Some(y)) => Some(x.max(y)), + (x, None) => x, + (None, y) => y, + }), + _ => panic!("law/value mismatch"), + } +} + +#[test] +fn whole_execution_equals_the_merge_of_any_partition_in_any_order() { + for n in [1317usize, 4096 + 37] { + let mut seed = 0x5_1117 ^ n as u64; + let scattered: Vec = (0..n).map(|_| lcg(&mut seed) % 4 == 0).collect(); + let pl = plane(n, |r| scattered[r]); + let vals: Vec = (0..n) + .map(|_| (lcg(&mut seed) % 2001) as i32 - 1000) + .collect(); + let masks: [&[u64]; 1] = [&pl]; + let lanes = [LaneRef::I32(&vals)]; + let planes = Planes { + n_rows: n, + masks: &masks, + lanes: &lanes, + }; + let (plo, phi) = (37u32, n as u32 - 11); + let tiled_on = |t: Terminal| { + let mut p = program(Shape::Tiled, plo, phi, Term::Count); + p.terminal = t; + p + }; + let m = Operand::Scratch(1); + let programs: Vec<(Program, Law)> = vec![ + (program(Shape::Fused, plo, phi, Term::Count), Law::Add), + (program(Shape::Fused, plo, phi, Term::Any), Law::Or), + (program(Shape::Tiled, plo, phi, Term::Count), Law::Add), + (program(Shape::Tiled, plo, phi, Term::Any), Law::Or), + (program(Shape::Tiled, plo, phi, Term::All), Law::And), + ( + tiled_on(Terminal::MaskedSumI32 { mask: m, lane: 0 }), + Law::Add, + ), + ( + tiled_on(Terminal::MaskedMinI32 { mask: m, lane: 0 }), + Law::Min, + ), + ( + tiled_on(Terminal::MaskedMaxI32 { mask: m, lane: 0 }), + Law::Max, + ), + ]; + // Two-way splits at the named seams; three-way splits at random. + let mut partitions: Vec> = [0, 1, 63, 64, 65, 127, 128, 129, n / 2, n - 1, n] + .into_iter() + .map(|k| vec![0, k, n]) + .collect(); + for _ in 0..40 { + let a = (lcg(&mut seed) as usize) % (n + 1); + let b = (lcg(&mut seed) as usize) % (n + 1); + let (a, b) = (a.min(b), a.max(b)); + partitions.push(vec![0, a, b, n]); + } + for (p, law) in &programs { + let (whole, _) = run(p, &planes, 0..n); + for cuts in &partitions { + let parts: Vec = cuts + .windows(2) + .map(|w| run(p, &planes, w[0]..w[1]).0) + .collect(); + // Forward, reverse and a rotated order all merge to `whole`. + for order in [ + (0..parts.len()).collect::>(), + (0..parts.len()).rev().collect(), + (0..parts.len()).map(|i| (i + 1) % parts.len()).collect(), + ] { + let merged = order + .iter() + .map(|&i| parts[i]) + .reduce(|a, b| merge_with(*law, a, b)) + .expect("non-empty"); + assert_eq!(merged, whole, "n={n} cuts={cuts:?} order={order:?}"); + } + } + } + // Keep: disjoint extents write disjoint bits of ONE absolute buffer; + // any order reproduces the whole-population Keep. + let keep = program(Shape::Tiled, plo, phi, Term::Keep); + let mut s = Scratch::for_program(&keep, n).expect("scratch"); + let mut whole = vec![0u64; words(n)]; + execute_into( + &keep, + &planes, + &Foreign::NONE, + &mut s, + Out::Mask(&mut whole), + ) + .expect("whole keep"); + for cuts in &partitions { + let spans: Vec<(usize, usize)> = cuts.windows(2).map(|w| (w[0], w[1])).collect(); + for order in [ + (0..spans.len()).collect::>(), + (0..spans.len()).rev().collect(), + ] { + let mut o = vec![0u64; words(n)]; + for &i in &order { + let (lo, hi) = spans[i]; + execute_extent( + &keep, + &planes, + &Foreign::NONE, + &mut s, + Out::Mask(&mut o), + lo..hi, + ) + .expect("part keep"); + } + assert_eq!(o, whole, "keep n={n} cuts={cuts:?} order={order:?}"); + } + } + } +} + +/// The plan the executor iterates is proportional to the extent. A tiny +/// extent in a million rows is one one-word tile; the whole population is +/// exactly the `TILE_WORDS` chunking whole-population execution always used. +#[test] +fn the_tiles_visited_scale_with_the_extent_not_the_population() { + let n = 1 << 20; + let tile_count = |lo: usize, hi: usize| extent_tiles(n, TILE_WORDS, lo..hi).count(); + let words_visited = |lo: usize, hi: usize| { + extent_tiles(n, TILE_WORDS, lo..hi) + .map(|(w, _)| w.len()) + .sum::() + }; + assert_eq!(tile_count(500_001, 500_002), 1); + assert_eq!(words_visited(500_001, 500_002), 1); + // 64 rows across a word seam: two one-word edge tiles; aligned: one. + assert_eq!(tile_count(500_001, 500_065), 2); + assert_eq!(tile_count(500_032, 500_096), 1); // 500_032 = 64 * 7813 + assert_eq!(words_visited(512_000, 512_512), 8); + assert_eq!(tile_count(512_000, 512_512), 1); + assert_eq!(tile_count(0, 0), 0); + let whole: Vec<_> = extent_tiles(n, TILE_WORDS, 0..n).collect(); + assert_eq!(whole.len(), n / 64 / TILE_WORDS); + for (i, (w, edge)) in whole.iter().enumerate() { + assert_eq!(*w, i * TILE_WORDS..(i + 1) * TILE_WORDS); + assert!(edge.is_none()); + } + // Every extent: tiles are contiguous, disjoint, cover exactly the touched + // words, and only a word the extent cuts carries an edge. + let n = 1317; + for lo in [0usize, 1, 63, 64, 65, 500, 1300] { + for hi in [lo, lo + 1, lo + 63, lo + 64, lo + 700, n] { + let hi = hi.min(n); + if hi < lo { + continue; + } + let tiles: Vec<_> = extent_tiles(n, TILE_WORDS, lo..hi).collect(); + let covered: Vec = tiles.iter().flat_map(|(w, _)| w.clone()).collect(); + let want: Vec = touched_words(lo as u32, hi as u32).collect(); + assert_eq!(covered, want, "extent {lo}..{hi}"); + for (w, edge) in &tiles { + if edge.is_some() { + assert_eq!(w.len(), 1, "an edge tile is one word"); + let cut_lo = w.start == lo / 64 && lo % 64 != 0; + let cut_hi = w.start == (hi - 1) / 64 && hi % 64 != 0 && hi < n; + assert!(cut_lo || cut_hi, "edge on a word the extent does not cut"); + } + } + } + } +} + +#[test] +fn bad_extents_and_unmergeable_terminals_are_refused_before_execution() { + let n = 200; + let pl = plane(n, |r| r % 2 == 0); + let keys: Vec = (0..n as u32).map(|r| r % 4).collect(); + let vals: Vec = (0..n as i32).collect(); + let masks: [&[u64]; 1] = [&pl]; + let lanes = [LaneRef::U32(&keys), LaneRef::I32(&vals)]; + let planes = Planes { + n_rows: n, + masks: &masks, + lanes: &lanes, + }; + let count = program(Shape::Tiled, 0, n as u32, Term::Count); + let mut s = Scratch::for_program(&count, n).expect("scratch"); + for (lo, hi) in [(10usize, 9usize), (0, n + 1), (n + 1, n + 2)] { + #[allow(clippy::reversed_empty_ranges)] + let r = execute_extent(&count, &planes, &Foreign::NONE, &mut s, Out::None, lo..hi); + assert_eq!(r, Err(ExecError::ExtentOutOfRange { lo, hi, n_rows: n })); + } + let group = Program::new( + vec![], + Terminal::GroupSumI32 { + mask: Operand::Plane(0), + key: 0, + val: 1, + }, + ); + let mut sink = vec![0i64; 4]; + let mut s = Scratch::for_program(&group, n).expect("scratch"); + assert_eq!( + execute_extent( + &group, + &planes, + &Foreign::NONE, + &mut s, + Out::I64(&mut sink), + 10..20 + ), + Err(ExecError::ExtentUnsupported { + what: "GroupSumI32" + }) + ); + assert_eq!(sink, vec![0; 4], "a refusal writes nothing"); + // The whole extent accepts every terminal: it IS `execute_into`. + assert_eq!( + execute_extent( + &group, + &planes, + &Foreign::NONE, + &mut s, + Out::I64(&mut sink), + 0..n + ), + Ok(Value::GroupSummed) + ); + // A partial Keep needs its population-addressed sink. + let keep = program(Shape::Tiled, 0, n as u32, Term::Keep); + let mut s = Scratch::for_program(&keep, n).expect("scratch"); + assert_eq!( + execute_extent(&keep, &planes, &Foreign::NONE, &mut s, Out::None, 10..20), + Err(ExecError::TerminalNeedsOut { what: "Keep" }) + ); +} + +/// `Gather` reads its foreign plane by KEY. Slicing the local table must not +/// slice the foreign one: here every local row of the extent names a foreign +/// key below the extent's own row numbers, so a foreign plane restricted to +/// the extent would see none of them. +#[test] +fn the_extent_never_slices_a_foreign_plane() { + let n = 1317; + let foreign_rows = 100; + let fwords = plane(foreign_rows, |k| k % 3 == 0); + let fk: Vec = (0..n as u32) + .map(|r| (r * 7) % foreign_rows as u32) + .collect(); + let lanes = [LaneRef::U32(&fk)]; + let planes = Planes { + n_rows: n, + masks: &[], + lanes: &lanes, + }; + let fplanes = [ForeignPlane { + words: &fwords, + rows: foreign_rows, + }]; + let foreign = Foreign { + planes: &fplanes, + lanes: &[], + }; + let p = Program::new( + vec![MaskOp::Gather { + lane: 0, + foreign: 0, + dst: 0, + }], + Terminal::Count { + mask: Operand::Scratch(0), + }, + ); + let (lo, hi) = (600usize, 900usize); + let want = (lo..hi).filter(|&r| bit(&fwords, fk[r] as usize)).count(); + assert!(want > 0 && want < hi - lo, "the case must be non-trivial"); + let mut s = Scratch::for_program(&p, n).expect("scratch"); + let v = execute_extent(&p, &planes, &foreign, &mut s, Out::None, lo..hi).expect("gather"); + assert_eq!(v, Value::Count(want)); +} From f47155c07e91969b2a5944c7ed66e1d6be046283 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 13:08:03 +0000 Subject: [PATCH 2/5] mask-risc: clippy is_multiple_of in extent_tiles Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GXUahz73MZxtxWcfpHp9dG --- crates/lance-graph-mask-risc/src/exec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/lance-graph-mask-risc/src/exec.rs b/crates/lance-graph-mask-risc/src/exec.rs index 919c72bbe..dde80d4af 100644 --- a/crates/lance-graph-mask-risc/src/exec.rs +++ b/crates/lance-graph-mask-risc/src/exec.rs @@ -736,8 +736,8 @@ pub fn extent_tiles(n_rows: usize, tile_words: usize, extent: Range) -> E end: span.end, lo: extent.start, hi: extent.end, - head_edge: extent.start % 64 != 0, - tail_edge: extent.end % 64 != 0 && extent.end < n_rows, + head_edge: !extent.start.is_multiple_of(64), + tail_edge: !extent.end.is_multiple_of(64) && extent.end < n_rows, tile_words: tile_words.max(1), } } From 4b2b9708a8633189f324e12de4099e55c9d23829 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 13:08:13 +0000 Subject: [PATCH 3/5] mask-risc: clippy is_multiple_of in extent tests Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GXUahz73MZxtxWcfpHp9dG --- crates/lance-graph-mask-risc/tests/extent.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/lance-graph-mask-risc/tests/extent.rs b/crates/lance-graph-mask-risc/tests/extent.rs index 3874efddc..bddb94aec 100644 --- a/crates/lance-graph-mask-risc/tests/extent.rs +++ b/crates/lance-graph-mask-risc/tests/extent.rs @@ -160,7 +160,7 @@ fn check(pl: &[u64], n: usize, prog: (u32, u32), ext: (usize, usize), shape: Sha fn the_edge_matrix_agrees_with_the_scalar_oracle() { let n = 1317; // not a multiple of 64, three 8-word tiles let mut seed = 0xE17E_u64; - let scattered: Vec = (0..n).map(|_| lcg(&mut seed) % 3 == 0).collect(); + let scattered: Vec = (0..n).map(|_| lcg(&mut seed).is_multiple_of(3)).collect(); let planes: [(&str, Vec); 3] = [ ("zero", plane(n, |_| false)), ("ones", plane(n, |_| true)), @@ -366,7 +366,7 @@ fn merge_with(law: Law, a: Value, b: Value) -> Value { fn whole_execution_equals_the_merge_of_any_partition_in_any_order() { for n in [1317usize, 4096 + 37] { let mut seed = 0x5_1117 ^ n as u64; - let scattered: Vec = (0..n).map(|_| lcg(&mut seed) % 4 == 0).collect(); + let scattered: Vec = (0..n).map(|_| lcg(&mut seed).is_multiple_of(4)).collect(); let pl = plane(n, |r| scattered[r]); let vals: Vec = (0..n) .map(|_| (lcg(&mut seed) % 2001) as i32 - 1000) From c02c2b8e6c49c488336cb2d57952f124ee14193a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 13:11:17 +0000 Subject: [PATCH 4/5] mask-risc: extent_probe benchmark + board entry for the absolute execution extent Adds the N=1M dispatch-locality probe (fused / tiled / lane shapes over 1 row .. whole extents, oracle-checked), the D-WFL-EXTENT status row, and an append-only correction of #1267's 'rebasing every mask and lane' wording: the required property is correct absolute execution over an unaligned extent, met with word bounds and register edge masks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GXUahz73MZxtxWcfpHp9dG --- .claude/board/STATUS_BOARD.md | 1 + .../2026-09-23-absolute-execution-extent.md | 114 ++++++++++++ .claude/board/entries/README.md | 3 +- .../examples/extent_probe.rs | 172 ++++++++++++++++++ 4 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-09-23-absolute-execution-extent.md create mode 100644 crates/lance-graph-mask-risc/examples/extent_probe.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index d9d86d3fc..9f1daee4f 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -53,6 +53,7 @@ materialization, and that is the ideal Layer-0 operation, not a compromise. | D-WFL-AXIS | **Three independent axes, not one binary:** OPERATORS (fold · mask/ternlog · project · rotate · neighbour · reduce) × CARRIERS (canonical lane · range · descriptor · resident mask · cached mask) × MATERIALIZATION CHOICE (fused vs materialized bitmap). The BBB question is not *fold or mask?* but **is this membership relation transient algebra, or has it been PROMOTED to a mask carrier?** | Queued | every plan must record the promotion, not the operator choice. Falsified if a plan can promote a membership relation to a carrier without that appearing in it | | D-WFL-ENTROPY | **Representation entropy should follow ANSWER entropy.** *"Do these two million-row regions intersect?"* ≈ 1 bit; building 125 KB of mask to find it is the obscenity — and 125 KB is Seam B's MEASURED number at N=1M, not rhetoric. *"How many overlap?"* = 32–64 bits; also fused. *"Give me the overlap, six thoughts will manipulate it"* justifies the bitmap, which is then low-entropy **relative to its future workload** | Queued | the ratio `materialized bytes : answer bytes` reported per operation (§6's `R_info`), with the downstream workload named whenever it exceeds 1 | | D-WFL-W2b‴ | ⊘ supersedes W2b″'s "fold arm vs mask arm" — **both arms mask.** W2b-A: `Range × resident → FUSED masking → Count/Any`, no result mask. W2b-B: `→ masking → a MATERIALIZED bounded mask`. Same masking semantics, different result carrier | In PR — W2b-A shipped (fused `Range ∩ plane → Count/Any`, 0 derived words; `entries/2026-09-23-terminal-elects-materialization-range-plane.md`); W2b-B reuse burden OPEN | differential across arms and against the oracle. **W2b-B carries a burden W2b-A does not: it must NAME and MEASURE the downstream reuse justifying the carrier — a materialization with no demonstrated consumer FAILS the arm.** That is what deliberate promotion costs | +| D-WFL-EXTENT | Absolute execution extent: `execute_extent(program, planes, foreign, scratch, out, lo..hi)`; the extent is an OUTER restriction in `Planes`' absolute row coordinates (never a rebasing); `execute_into` = extent `0..n_rows` | In PR — shipped with split/merge, non-rebasing and structural gates; `entries/2026-09-23-absolute-execution-extent.md` | whole == merge of any partition in any order; extent starting at 65/129 reads absolute rows (worker-local reading shown to differ); a 1-row extent over 1M rows visits one word. OQ-5 untouched | **Shortest form:** fold the datasets, mask the folds, materialize only when the mask itself is worth keeping. diff --git a/.claude/board/entries/2026-09-23-absolute-execution-extent.md b/.claude/board/entries/2026-09-23-absolute-execution-extent.md new file mode 100644 index 000000000..589174ea4 --- /dev/null +++ b/.claude/board/entries/2026-09-23-absolute-execution-extent.md @@ -0,0 +1,114 @@ +# 2026-09-23 — Absolute execution extent for mask-risc + +**Status:** MEASURED (extent seam shipped in PR) · OPEN (scheduler, OQ-5, the `_sym` / group merge laws) +**D-ids:** D-WFL-EXTENT (new, In PR). Corrects the wording of +`entries/2026-09-23-cubecl-llvm-boundary-and-audit-regrade.md` lesson 1 (below). +OQ-5 is untouched and still open. + +## API shape +`execute_extent(program, planes, foreign, scratch, out, lo..hi)` in +`crates/lance-graph-mask-risc/src/exec.rs`. `execute_into` is now that call with +`0..n_rows`, so existing callers are unchanged and the whole extent accepts every terminal. +`extent_tiles(n_rows, tile_words, lo..hi)` is public: it is the plan the executor iterates, +which makes the structural claim checkable. + +## The absolute-coordinate law (CURRENT-CONTRACT, TEST-PINNED) +The extent is an OUTER restriction in the same row coordinates as `Planes`. A program +`Range [1000, 2000)` executed over the extent `[1500, 1700)` means +`[1000, 2000) ∩ [1500, 1700)`. Lane element `r` is row `r` whatever the extent, and tile word +`w` is word `w` of every resident mask. Nothing is rebased, copied or renumbered. + +**Implementation:** +- **Word bounds:** only `span_words(lo, hi)` is walked, the same law `touched_words` + now delegates to. +- **Tile start/stop:** a word the extent cuts (an unaligned `lo`, or an unaligned `hi` short + of `n_rows`) is its own one-word tile. +- **Edge masks:** on that tile the terminal's mask is ANDed with the in-extent bits in a + one-word register temporary. For `All`, the out-of-extent bits are ORed in instead, since + set bits are its identity. +- **No lane offsets:** lanes are read at `w * 64` exactly as before. +- **No copies:** no lane or mask copy was required. + +The #1268 fused fold composes by intersection +(`program_range ∩ extent ∩ resident_plane`) inside the same seam. There is no second +evaluator. + +## What execution now skips / what still materializes +- **Skipped:** every tile outside the extent. A 1-row extent over 1M rows visits one word. +- **Still materializes, deliberately:** + - `Keep` writes its population-addressed `Out::Mask`, but only the in-extent bits: edge + words are merged bit-exactly, and every other bit stays as the caller holds it. Disjoint + extents therefore compose into one absolute buffer in any order. + - The tiled (non-fused) path still writes tile-local scratch, but only for touched tiles. +- **Refused on a partial extent**, as `ExtentUnsupported`, because there is no shipped merge + law or disjoint sink here: `BlendI32`, `ScatterOrU32`, `ScatterCountU32`, + `CountKeyRunsU32`, `GroupSumI32`, `GroupSumViaI32`, `GroupReduce`. `_sym` stays a recorded + law only (`TD-SYM-SUM-MERGE-IS-NOT-ADDITION-1`). +- **Foreign planes and lanes** (`Gather`, `GroupKey::Via`) are addressed by key and are never + sliced. That is pinned by a test whose extent rows all name foreign keys below the extent's + own row numbers. + +## Split/merge (TEST-PINNED, `tests/extent.rs`) +- **Terminals:** Count (+), Any (∨), All (∧), MaskedSumI32 (+), MaskedMin/Max (min/max), + each over the fused and tiled shapes. +- **Partitions:** + - two-way at k ∈ {0, 1, 63, 64, 65, 127, 128, 129, N/2, N−1, N}; + - 40 random three-way partitions; + - N ∈ {1317, 4133}, neither a multiple of 64. +- **Orders:** every partition is merged in forward, reverse and rotated order, and each equals + whole-population execution. `Keep` partials written in forward and reverse order into one + buffer equal the whole `Keep`. + +## Non-rebasing falsifier +Distinct lane values sit at rows 63, 64, 65, 127, 128 and 129. +- `MaskedSumI32` over extents starting at 65, 129, 63 and 127 must equal the absolute-row sum. +- The test also asserts that the worker-local sum differs, so a rebasing executor cannot pass. +- A second test pins program `Range [1000,2000)` over extent `[1500,1700)` as exactly 200 + rows, and `Keep` writes exactly bits 1500..1700. + +## Disable runs (committed first, then each restored) +| disable | tests red | +|---|---| +| tiles walk from row 0 instead of the extent | 7 of 8, incl. the structural tile gate | +| edge word not restricted | 6 | +| lanes read worker-local (`r0` relative to the extent) | 5, incl. the absolute-rows falsifier | +| fused fold ignores the extent | 4 | +| `Keep` overwrites the whole edge word | 3 | + +## Benchmarks +Run with `cargo run --release -p lance-graph-mask-risc --example extent_probe`: N = 1,048,576 +rows, a 2/3-dense plane and unaligned extents. Every result is checked against a scalar +oracle over the same absolute rows. + +| shape | extent | median ns | population words read | lane elements | derived words written | +|---|---|---|---|---|---| +| fused Range∩plane→Count | 1 row | 62 | 1 | 0 | 0 | +| fused | 512 rows | 70 | 9 | 0 | 0 | +| fused | 25% | 1,172 | 4,097 | 0 | 0 | +| fused | whole | 4,102 | 16,354 | 0 | 0 | +| tiled Range→And→Count | 1 row | 161 | 1 | 0 | 2 | +| tiled | 1% | 2,820 | 165 | 0 | 330 | +| tiled | whole | 158,051 | 16,384 | 0 | 32,768 | +| lane EqU32 under plane→MaskedSumI32 | 1 row | 172 | 1 | 128 | 1 | +| lane | 25% | 379,509 | 4,097 | 524,416 | 4,097 | +| lane | whole | 1,828,878 | 16,384 | 2,097,152 | 16,384 | + +- Cost follows extent width, not `n_rows`: a 1-row extent is 62–172 ns on every shape. +- The word columns come from the plan the executor iterates (`extent_tiles` / `touched_words`), + not from instrumentation. + +## Correction of #1267 (append-only) +`2026-09-23-cubecl-llvm-boundary-and-audit-regrade.md` lesson 1 says the ranged entry point +needs "rebasing every mask and lane for ranges that do not start on a word boundary". +**The required property is correct absolute execution over a non-zero, non-word-aligned +extent, not physical rebasing.** It is met with word bounds and register edge masks, and +rebasing would have broken the absolute-coordinate law above. + +## What remains before scheduling +- OQ-5 (thread/rayon vendor) is untouched and not resolved. +- No scheduler, queue, thread or affinity exists, and no scheduler work has started. +- Merge laws are still missing for the group terminals and the `_sym` SUM, which are refused on + partial extents. +- The next expression→terminal folds (`plane ∩ plane → Count`, lane predicate → Count, + ternlog → Count) would make an extent-dispatched unit a small algebraic program rather than a + tile-writing bitmap machine. They are not started. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 77c7f905b..d41cf0bf1 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,13 +25,14 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -150 entries, 2026-08-06 .. 2026-09-23. +151 entries, 2026-08-06 .. 2026-09-23. | date | entry id | finding | file | |---|---|---|---| | 2026-09-23 | `terminal-elects-materialization-range-plane` | | [2026-09-23-terminal-elects-materialization-range-plane.md](2026-09-23-terminal-elects-materialization-range-plane.md) | | 2026-09-23 | `quack-having-sym-sum-presence-mask` | | [2026-09-23-quack-having-sym-sum-presence-mask.md](2026-09-23-quack-having-sym-sum-presence-mask.md) | | 2026-09-23 | `cubecl-llvm-boundary-and-audit-regrade` | | [2026-09-23-cubecl-llvm-boundary-and-audit-regrade.md](2026-09-23-cubecl-llvm-boundary-and-audit-regrade.md) | +| 2026-09-23 | `absolute-execution-extent` | | [2026-09-23-absolute-execution-extent.md](2026-09-23-absolute-execution-extent.md) | | 2026-09-22 | `quack-duckdb-parity-t0-keyed-reduction` | | [2026-09-22-quack-duckdb-parity-t0-keyed-reduction.md](2026-09-22-quack-duckdb-parity-t0-keyed-reduction.md) | | 2026-09-22 | `E-W0C-THE-ROW-BRIDGE-IS-A-DIALECT-NOT-AN-INTERPRETER-1` | a merged relational op carried as loco program data reaches the fused executor with no population crossing; the enum explosion is upstream of mask-risc | [2026-09-22-e-w0c-the-row-bridge-is-a-dialect-not-an-interpreter-1.md](2026-09-22-e-w0c-the-row-bridge-is-a-dialect-not-an-interpreter-1.md) | | 2026-09-22 | `E-CATS-FOLD-DOES-NOT-RETAIN-A-POPULATION-BITMAP-1` | CATS aggregate lowers to one tiled grouped terminal; bitmap realization is a requested boundary sink | [2026-09-22-e-cats-fold-does-not-retain-a-population-bitmap-1.md](2026-09-22-e-cats-fold-does-not-retain-a-population-bitmap-1.md) | diff --git a/crates/lance-graph-mask-risc/examples/extent_probe.rs b/crates/lance-graph-mask-risc/examples/extent_probe.rs new file mode 100644 index 000000000..a4e9aa50e --- /dev/null +++ b/crates/lance-graph-mask-risc/examples/extent_probe.rs @@ -0,0 +1,172 @@ +//! Dispatch locality of an absolute execution extent at N = 1M rows. +//! +//! One program family, three physical shapes, each run over extents of +//! 1 row, 64 rows, 512 rows, 1 %, 25 % and the whole population: +//! +//! - FUSED: `Range ∩ plane → Count` (the #1268 fold, intersected with the +//! extent — writes nothing); +//! - TILED: `Range → And(plane) → Count` (writes two scratch slots per tile); +//! - LANE: `EqU32(lane) under plane → MaskedSumI32` (reads two value lanes). +//! +//! Reported separately: median latency, population words visited, lane +//! elements visited, and derived words written. The word counts are DERIVED +//! from the plan the executor iterates (`extent_tiles`) and the fold's span +//! (`touched_words`), not guessed; every extent result is cross-checked +//! against a scalar oracle over the same absolute rows. +//! +//! `cargo run --release -p lance-graph-mask-risc --example extent_probe` + +use std::time::Instant; + +use lance_graph_mask_risc::exec::{execute_extent, Scratch}; +use lance_graph_mask_risc::{ + extent_tiles, touched_words, Foreign, LaneRef, MaskOp, Operand, Out, Planes, Pred, Program, + Terminal, Value, TILE_WORDS, +}; + +fn lcg(seed: &mut u64) -> u64 { + *seed = seed + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + *seed >> 11 +} + +fn median_ns(mut f: impl FnMut() -> Value, reps: usize) -> (f64, Value) { + let mut times = Vec::with_capacity(reps); + let mut last = Value::Count(0); + for _ in 0..reps { + let t = Instant::now(); + last = std::hint::black_box(f()); + times.push(t.elapsed().as_nanos() as f64); + } + times.sort_by(|a, b| a.total_cmp(b)); + (times[reps / 2], last) +} + +fn main() { + let n = 1usize << 20; + let mut seed = 0xE7E_u64; + let bits: Vec = (0..n).map(|_| !lcg(&mut seed).is_multiple_of(3)).collect(); + let mut pl = vec![0u64; n.div_ceil(64)]; + for (r, &b) in bits.iter().enumerate() { + if b { + pl[r / 64] |= 1 << (r % 64); + } + } + let keys: Vec = (0..n).map(|_| (lcg(&mut seed) % 8) as u32).collect(); + let vals: Vec = (0..n).map(|_| (lcg(&mut seed) % 1000) as i32).collect(); + let masks: [&[u64]; 1] = [&pl]; + let lanes = [LaneRef::U32(&keys), LaneRef::I32(&vals)]; + let planes = Planes { + n_rows: n, + masks: &masks, + lanes: &lanes, + }; + let (plo, phi) = (1000u32, n as u32 - 1000); + let fused = Program::new( + vec![MaskOp::Pred { + pred: Pred::Range { lo: plo, hi: phi }, + under: Some(Operand::Plane(0)), + dst: 0, + }], + Terminal::Count { + mask: Operand::Scratch(0), + }, + ); + let tiled = Program::new( + vec![ + MaskOp::Pred { + pred: Pred::Range { lo: plo, hi: phi }, + under: None, + dst: 0, + }, + MaskOp::And { + a: Operand::Scratch(0), + b: Operand::Plane(0), + dst: 1, + }, + ], + Terminal::Count { + mask: Operand::Scratch(1), + }, + ); + let lane = Program::new( + vec![MaskOp::Pred { + pred: Pred::EqU32 { lane: 0, v: 3 }, + under: Some(Operand::Plane(0)), + dst: 0, + }], + Terminal::MaskedSumI32 { + mask: Operand::Scratch(0), + lane: 1, + }, + ); + let mid = n / 2 + 17; // deliberately unaligned + let extents: [(&str, usize, usize); 6] = [ + ("1 row", mid, mid + 1), + ("64 rows", mid, mid + 64), + ("512 rows", mid, mid + 512), + ("1%", mid, mid + n / 100), + ("25%", n / 5 + 3, n / 5 + 3 + n / 4), + ("whole", 0, n), + ]; + println!( + "{:>6} {:>9} {:>10} {:>12} {:>12} {:>14}", + "shape", "extent", "median_ns", "words_read", "lane_elems", "derived_wr" + ); + for (shape, p, slots) in [ + ("fused", &fused, 0usize), + ("tiled", &tiled, 2), + ("lane", &lane, 1), + ] { + let mut s = Scratch::for_program(p, n).expect("scratch"); + let tw = s.words(); + for (name, lo, hi) in extents { + // Scalar oracle over the same ABSOLUTE rows. + let want = match shape { + "lane" => Value::SumI64( + (lo..hi) + .filter(|&r| bits[r] && keys[r] == 3) + .map(|r| i64::from(vals[r])) + .sum(), + ), + _ => Value::Count( + (lo..hi) + .filter(|&r| bits[r] && (plo as usize..phi as usize).contains(&r)) + .count(), + ), + }; + let reps = if hi - lo > 100_000 { 31 } else { 2001 }; + let (ns, got) = median_ns( + || { + execute_extent(p, &planes, &Foreign::NONE, &mut s, Out::None, lo..hi) + .expect("extent") + }, + reps, + ); + assert_eq!(got, want, "{shape} {name}"); + let tiles_words: usize = extent_tiles(n, tw, lo..hi).map(|(w, _)| w.len()).sum(); + let (words_read, lane_elems, derived) = match shape { + "fused" => { + let (a, b) = ((plo as usize).max(lo), (phi as usize).min(hi)); + let span = if a < b { + touched_words(a as u32, b as u32).len() + } else { + 0 + }; + (span, 0, 0) + } + "tiled" => (tiles_words, 0, slots * tiles_words), + _ => (tiles_words, 2 * tiles_words * 64, slots * tiles_words), + }; + println!( + "{shape:>6} {name:>9} {ns:>10.0} {words_read:>12} {lane_elems:>12} {derived:>14}" + ); + } + } + println!( + "(n = {n}, {} population words, scratch tile = {} words)", + n / 64, + TILE_WORDS + ); +} From fa408ad2ef0c1225caeed2960dcee3848ba3c715 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 14:36:37 +0000 Subject: [PATCH 5/5] mask-risc: extent_tiles crate-private; pin Keep extents as sequential-only extent_tiles/ExtentTiles are executor detail (tile width, edge representation), so they drop out of the public API; the structural tile-plan test moves in-crate (exec::extent_tile_tests) and the probe reports touched words from the semantic span. execute_extent's doc and the board entry now state that partial Keep sinks compose in any SEQUENTIAL order only: an unaligned split shares a physical u64 and the edge merge is read-modify-write, so concurrent writers need word-disjoint ownership, separate partial sinks plus merge, or explicit synchronization. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GXUahz73MZxtxWcfpHp9dG --- .../2026-09-23-absolute-execution-extent.md | 20 +++-- .../examples/extent_probe.rs | 13 ++-- crates/lance-graph-mask-risc/src/exec.rs | 76 +++++++++++++++++-- crates/lance-graph-mask-risc/src/lib.rs | 4 +- crates/lance-graph-mask-risc/tests/extent.rs | 60 ++------------- 5 files changed, 99 insertions(+), 74 deletions(-) diff --git a/.claude/board/entries/2026-09-23-absolute-execution-extent.md b/.claude/board/entries/2026-09-23-absolute-execution-extent.md index 589174ea4..bf41df0f9 100644 --- a/.claude/board/entries/2026-09-23-absolute-execution-extent.md +++ b/.claude/board/entries/2026-09-23-absolute-execution-extent.md @@ -9,8 +9,10 @@ OQ-5 is untouched and still open. `execute_extent(program, planes, foreign, scratch, out, lo..hi)` in `crates/lance-graph-mask-risc/src/exec.rs`. `execute_into` is now that call with `0..n_rows`, so existing callers are unchanged and the whole extent accepts every terminal. -`extent_tiles(n_rows, tile_words, lo..hi)` is public: it is the plan the executor iterates, -which makes the structural claim checkable. +`extent_tiles` (the tile plan the executor iterates) is **crate-private**: tile width and +edge representation are executor detail, not API. The structural claim is pinned by the +in-crate test `exec::extent_tile_tests`; the benchmark reports touched words from the +semantic span (`touched_words`), which that test proves the plan covers exactly. ## The absolute-coordinate law (CURRENT-CONTRACT, TEST-PINNED) The extent is an OUTER restriction in the same row coordinates as `Planes`. A program @@ -38,7 +40,13 @@ evaluator. - **Still materializes, deliberately:** - `Keep` writes its population-addressed `Out::Mask`, but only the in-extent bits: edge words are merged bit-exactly, and every other bit stays as the caller holds it. Disjoint - extents therefore compose into one absolute buffer in any order. + extents therefore compose into one absolute buffer in any **sequential** order. + - **Not a concurrency licence (CURRENT-CONTRACT).** An unaligned split such as + `[0, 65)` + `[65, N)` puts both extents in the same physical `u64`, and the edge merge + is a read-modify-write. Partial `Keep` sinks compose in any sequential order; concurrent + execution requires word-disjoint sink ownership (boundaries on multiples of 64), + separate partial sinks plus a merge, or another explicitly synchronized strategy. A + future scheduler must not infer two simultaneous writers on one `Out::Mask` from this. - The tiled (non-fused) path still writes tile-local scratch, but only for touched tiles. - **Refused on a partial extent**, as `ExtentUnsupported`, because there is no shipped merge law or disjoint sink here: `BlendI32`, `ScatterOrU32`, `ScatterCountU32`, @@ -69,7 +77,7 @@ Distinct lane values sit at rows 63, 64, 65, 127, 128 and 129. ## Disable runs (committed first, then each restored) | disable | tests red | |---|---| -| tiles walk from row 0 instead of the extent | 7 of 8, incl. the structural tile gate | +| tiles walk from row 0 instead of the extent | 7 of 8, incl. the structural tile gate (since moved in-crate as `exec::extent_tile_tests`) | | edge word not restricted | 6 | | lanes read worker-local (`r0` relative to the extent) | 5, incl. the absolute-rows falsifier | | fused fold ignores the extent | 4 | @@ -94,8 +102,8 @@ oracle over the same absolute rows. | lane | whole | 1,828,878 | 16,384 | 2,097,152 | 16,384 | - Cost follows extent width, not `n_rows`: a 1-row extent is 62–172 ns on every shape. -- The word columns come from the plan the executor iterates (`extent_tiles` / `touched_words`), - not from instrumentation. +- The word columns come from the extent's semantic span (`touched_words`), which the in-crate + tile-plan test pins the executor to cover exactly; not from instrumentation. ## Correction of #1267 (append-only) `2026-09-23-cubecl-llvm-boundary-and-audit-regrade.md` lesson 1 says the ranged entry point diff --git a/crates/lance-graph-mask-risc/examples/extent_probe.rs b/crates/lance-graph-mask-risc/examples/extent_probe.rs index a4e9aa50e..5d82fcf25 100644 --- a/crates/lance-graph-mask-risc/examples/extent_probe.rs +++ b/crates/lance-graph-mask-risc/examples/extent_probe.rs @@ -10,8 +10,8 @@ //! //! Reported separately: median latency, population words visited, lane //! elements visited, and derived words written. The word counts are DERIVED -//! from the plan the executor iterates (`extent_tiles`) and the fold's span -//! (`touched_words`), not guessed; every extent result is cross-checked +//! from the extent's semantic span (`touched_words`), which the executor's +//! tile plan is test-pinned to cover exactly — not guessed; every extent result is cross-checked //! against a scalar oracle over the same absolute rows. //! //! `cargo run --release -p lance-graph-mask-risc --example extent_probe` @@ -20,8 +20,8 @@ use std::time::Instant; use lance_graph_mask_risc::exec::{execute_extent, Scratch}; use lance_graph_mask_risc::{ - extent_tiles, touched_words, Foreign, LaneRef, MaskOp, Operand, Out, Planes, Pred, Program, - Terminal, Value, TILE_WORDS, + touched_words, Foreign, LaneRef, MaskOp, Operand, Out, Planes, Pred, Program, Terminal, Value, + TILE_WORDS, }; fn lcg(seed: &mut u64) -> u64 { @@ -120,7 +120,6 @@ fn main() { ("lane", &lane, 1), ] { let mut s = Scratch::for_program(p, n).expect("scratch"); - let tw = s.words(); for (name, lo, hi) in extents { // Scalar oracle over the same ABSOLUTE rows. let want = match shape { @@ -145,7 +144,9 @@ fn main() { reps, ); assert_eq!(got, want, "{shape} {name}"); - let tiles_words: usize = extent_tiles(n, tw, lo..hi).map(|(w, _)| w.len()).sum(); + // Words the extent touches: the semantic span, which the executor's + // tile plan covers exactly (pinned in-crate by `extent_tile_tests`). + let tiles_words = touched_words(lo as u32, hi as u32).len(); let (words_read, lane_elems, derived) = match shape { "fused" => { let (a, b) = ((plo as usize).max(lo), (phi as usize).min(hi)); diff --git a/crates/lance-graph-mask-risc/src/exec.rs b/crates/lance-graph-mask-risc/src/exec.rs index dde80d4af..6ade6d0f7 100644 --- a/crates/lance-graph-mask-risc/src/exec.rs +++ b/crates/lance-graph-mask-risc/src/exec.rs @@ -726,9 +726,11 @@ fn run_fused(f: FusedTerminal, planes: &Planes<'_>) -> Value { /// `[0, n_rows)` there is no edge, and the tiles are exactly the ones /// whole-population execution has always walked. /// -/// Public so the structural claim — work scales with the extent, not with -/// `n_rows` — is checkable against the plan the executor actually iterates. -pub fn extent_tiles(n_rows: usize, tile_words: usize, extent: Range) -> ExtentTiles { +/// Crate-private: tile width and edge representation are executor +/// implementation detail, not API. The structural claim — work scales with +/// the extent, not with `n_rows` — is pinned by the in-crate test +/// `extent_tile_tests` against this exact plan. +pub(crate) fn extent_tiles(n_rows: usize, tile_words: usize, extent: Range) -> ExtentTiles { let span = span_words(extent.start, extent.end); ExtentTiles { first: span.start, @@ -744,7 +746,7 @@ pub fn extent_tiles(n_rows: usize, tile_words: usize, extent: Range) -> E /// Iterator returned by [`extent_tiles`]. #[derive(Debug, Clone)] -pub struct ExtentTiles { +pub(crate) struct ExtentTiles { first: usize, cur: usize, end: usize, @@ -853,10 +855,18 @@ pub fn execute_into( /// `MaskedSumI32` (sum), `MaskedMinI32` / `MaskedMaxI32` (min / max) — plus /// `Keep`, which writes only the in-extent bits of its population-addressed /// [`Out::Mask`] and leaves every other bit as the caller holds it (so -/// disjoint extents compose into one buffer in any order). Anything else is +/// disjoint extents compose into one buffer in any SEQUENTIAL order). Anything else is /// [`ExecError::ExtentUnsupported`]; `lo > hi` or `hi > n_rows` is /// [`ExecError::ExtentOutOfRange`]. Both are refused before execution. /// +/// **Sequential, not concurrent.** An unaligned boundary puts two extents in +/// one physical `u64` of the `Keep` sink, and the edge merge is a +/// read-modify-write. Partial `Keep` sinks compose in any sequential order; +/// concurrent execution requires word-disjoint sink ownership (boundaries on +/// multiples of 64), separate partial sinks plus a merge, or another +/// explicitly synchronized strategy. Nothing here licenses two writers on one +/// `Out::Mask` at once. +/// /// Foreign planes and lanes ([`MaskOp::Gather`], `GroupKey::Via`) are /// addressed by KEY, not by this table's rows, so the extent never slices /// them. @@ -1346,6 +1356,62 @@ pub fn execute_extent( }) } +#[cfg(test)] +mod extent_tile_tests { + use super::*; + + /// The plan the executor iterates is proportional to the extent. A tiny + /// extent in a million rows is one one-word tile; the whole population is + /// exactly the `TILE_WORDS` chunking whole-population execution always used. + #[test] + fn the_tiles_visited_scale_with_the_extent_not_the_population() { + let n = 1 << 20; + let tile_count = |lo: usize, hi: usize| extent_tiles(n, TILE_WORDS, lo..hi).count(); + let words_visited = |lo: usize, hi: usize| { + extent_tiles(n, TILE_WORDS, lo..hi) + .map(|(w, _)| w.len()) + .sum::() + }; + assert_eq!(tile_count(500_001, 500_002), 1); + assert_eq!(words_visited(500_001, 500_002), 1); + // 64 rows across a word seam: two one-word edge tiles; aligned: one. + assert_eq!(tile_count(500_001, 500_065), 2); + assert_eq!(tile_count(500_032, 500_096), 1); // 500_032 = 64 * 7813 + assert_eq!(words_visited(512_000, 512_512), 8); + assert_eq!(tile_count(512_000, 512_512), 1); + assert_eq!(tile_count(0, 0), 0); + let whole: Vec<_> = extent_tiles(n, TILE_WORDS, 0..n).collect(); + assert_eq!(whole.len(), n / 64 / TILE_WORDS); + for (i, (w, edge)) in whole.iter().enumerate() { + assert_eq!(*w, i * TILE_WORDS..(i + 1) * TILE_WORDS); + assert!(edge.is_none()); + } + // Every extent: tiles are contiguous, disjoint, cover exactly the touched + // words, and only a word the extent cuts carries an edge. + let n = 1317; + for lo in [0usize, 1, 63, 64, 65, 500, 1300] { + for hi in [lo, lo + 1, lo + 63, lo + 64, lo + 700, n] { + let hi = hi.min(n); + if hi < lo { + continue; + } + let tiles: Vec<_> = extent_tiles(n, TILE_WORDS, lo..hi).collect(); + let covered: Vec = tiles.iter().flat_map(|(w, _)| w.clone()).collect(); + let want: Vec = touched_words(lo as u32, hi as u32).collect(); + assert_eq!(covered, want, "extent {lo}..{hi}"); + for (w, edge) in &tiles { + if edge.is_some() { + assert_eq!(w.len(), 1, "an edge tile is one word"); + let cut_lo = w.start == lo / 64 && lo % 64 != 0; + let cut_hi = w.start == (hi - 1) / 64 && hi % 64 != 0 && hi < n; + assert!(cut_lo || cut_hi, "edge on a word the extent does not cut"); + } + } + } + } + } +} + #[cfg(test)] mod borrowed_scratch_tests { use super::*; diff --git a/crates/lance-graph-mask-risc/src/lib.rs b/crates/lance-graph-mask-risc/src/lib.rs index 5e13343d5..280d24652 100644 --- a/crates/lance-graph-mask-risc/src/lib.rs +++ b/crates/lance-graph-mask-risc/src/lib.rs @@ -119,8 +119,8 @@ pub mod ternlog_dispatch; pub mod value; pub use exec::{ - execute, execute_extent, execute_into, extent_tiles, materialize_rows, scratch_words_for, - tile_words_for, ExtentTiles, Scratch, TILE_WORDS, + execute, execute_extent, execute_into, materialize_rows, scratch_words_for, tile_words_for, + Scratch, TILE_WORDS, }; pub use fuse::{fuse, fuse_program, ternlog_imm, BoolExpr, FuseError, Fused}; pub use ir::{ diff --git a/crates/lance-graph-mask-risc/tests/extent.rs b/crates/lance-graph-mask-risc/tests/extent.rs index bddb94aec..cd4cf049a 100644 --- a/crates/lance-graph-mask-risc/tests/extent.rs +++ b/crates/lance-graph-mask-risc/tests/extent.rs @@ -12,14 +12,15 @@ //! - split composition: whole == the merge of any partition, in any order; //! - the non-rebasing falsifier: an extent that does not start at row 0 must //! read the absolute rows, and the rebased reading is shown to differ; -//! - the structural gate: the tiles visited scale with the extent, not with -//! `n_rows`; +//! - the structural gate (tiles visited scale with the extent, not with +//! `n_rows`) is in-crate: `exec::extent_tile_tests`, since the tile plan +//! is executor detail, not API; //! - foreign planes are addressed by key and are never sliced by the extent. use lance_graph_mask_risc::exec::{execute_extent, execute_into, Scratch}; use lance_graph_mask_risc::{ - extent_tiles, touched_words, ExecError, Foreign, ForeignPlane, LaneRef, MaskOp, Operand, Out, - Planes, Pred, Program, Terminal, Value, TILE_WORDS, + ExecError, Foreign, ForeignPlane, LaneRef, MaskOp, Operand, Out, Planes, Pred, Program, + Terminal, Value, }; fn lcg(seed: &mut u64) -> u64 { @@ -475,57 +476,6 @@ fn whole_execution_equals_the_merge_of_any_partition_in_any_order() { } } -/// The plan the executor iterates is proportional to the extent. A tiny -/// extent in a million rows is one one-word tile; the whole population is -/// exactly the `TILE_WORDS` chunking whole-population execution always used. -#[test] -fn the_tiles_visited_scale_with_the_extent_not_the_population() { - let n = 1 << 20; - let tile_count = |lo: usize, hi: usize| extent_tiles(n, TILE_WORDS, lo..hi).count(); - let words_visited = |lo: usize, hi: usize| { - extent_tiles(n, TILE_WORDS, lo..hi) - .map(|(w, _)| w.len()) - .sum::() - }; - assert_eq!(tile_count(500_001, 500_002), 1); - assert_eq!(words_visited(500_001, 500_002), 1); - // 64 rows across a word seam: two one-word edge tiles; aligned: one. - assert_eq!(tile_count(500_001, 500_065), 2); - assert_eq!(tile_count(500_032, 500_096), 1); // 500_032 = 64 * 7813 - assert_eq!(words_visited(512_000, 512_512), 8); - assert_eq!(tile_count(512_000, 512_512), 1); - assert_eq!(tile_count(0, 0), 0); - let whole: Vec<_> = extent_tiles(n, TILE_WORDS, 0..n).collect(); - assert_eq!(whole.len(), n / 64 / TILE_WORDS); - for (i, (w, edge)) in whole.iter().enumerate() { - assert_eq!(*w, i * TILE_WORDS..(i + 1) * TILE_WORDS); - assert!(edge.is_none()); - } - // Every extent: tiles are contiguous, disjoint, cover exactly the touched - // words, and only a word the extent cuts carries an edge. - let n = 1317; - for lo in [0usize, 1, 63, 64, 65, 500, 1300] { - for hi in [lo, lo + 1, lo + 63, lo + 64, lo + 700, n] { - let hi = hi.min(n); - if hi < lo { - continue; - } - let tiles: Vec<_> = extent_tiles(n, TILE_WORDS, lo..hi).collect(); - let covered: Vec = tiles.iter().flat_map(|(w, _)| w.clone()).collect(); - let want: Vec = touched_words(lo as u32, hi as u32).collect(); - assert_eq!(covered, want, "extent {lo}..{hi}"); - for (w, edge) in &tiles { - if edge.is_some() { - assert_eq!(w.len(), 1, "an edge tile is one word"); - let cut_lo = w.start == lo / 64 && lo % 64 != 0; - let cut_hi = w.start == (hi - 1) / 64 && hi % 64 != 0 && hi < n; - assert!(cut_lo || cut_hi, "edge on a word the extent does not cut"); - } - } - } - } -} - #[test] fn bad_extents_and_unmergeable_terminals_are_refused_before_execution() { let n = 200;