diff --git a/interactive/server/README.md b/interactive/server/README.md index 67a69619a..aba99f27e 100644 --- a/interactive/server/README.md +++ b/interactive/server/README.md @@ -166,11 +166,6 @@ fronting proxy if a deployment ever needs one. ./target/release/ddir_server < interactive/server/demo/txn.txt python3 interactive/server/demo/two_sessions.py # races + size gate over TCP -`load --explain` and `query` are reserved but unimplemented: explanation -support belongs on the scope-tree explanation machinery, and until that lands -the server reports an error rather than giving those commands an improvised -meaning. - ## Benchmarking The [LDBC-derived workload](bench/ldbc/README.md) runs parameterized interactive diff --git a/interactive/server/src/cmd.rs b/interactive/server/src/cmd.rs index 01bc7a979..666e91ca5 100644 --- a/interactive/server/src/cmd.rs +++ b/interactive/server/src/cmd.rs @@ -5,7 +5,7 @@ //! - `ok [body...]` — terminal success line //! - `err [body...]` — terminal error line //! - `data ` — one streamed body line (peek/tail batches) -//! - `end` — terminator after a stream of `data` lines +//! - `end` — terminator of a `tail` stream, once it is stopped //! //! Multi-line bodies use two-phase framing: `load ... begin` accepts literal //! DDIR through `end-load`, while `feed begin` accepts row updates @@ -33,8 +33,7 @@ pub type ConnectionId = u64; #[derive(Debug)] pub enum Cmd { /// Install a dataflow. - /// `id_hint` — a client-chosen name; the server may keep it or assign - /// a fresh id (echo'd in the response). + /// `id_hint` — the client-chosen name the dataflow is installed under. /// `bindings` — `import-name -> binding`, where the binding is either a /// registered trace name or a builtin call (`random(...)`). /// `program` — DDIR text, inline (`load … begin` … `end-load`) or read @@ -48,10 +47,9 @@ pub enum Cmd { program: String, explain: Option<(usize, bool)>, }, - /// Drop the dataflow named by id or by `id_hint`. Fails if any - /// export of this dataflow is still imported by another live - /// dataflow or held by a reader. - Drop { target: DataflowRef }, + /// Drop the named dataflow. Fails if any export of this dataflow is + /// still imported by another live dataflow or held by a reader. + Drop { name: String }, /// List held names. List, /// One-shot snapshot of a named trace, optionally of one key. @@ -100,39 +98,12 @@ pub enum Cmd { prog: String, input: usize, }, - /// Push a row into the query input of a `--explain` dataflow - /// (reserved; unimplemented). Sign is `+1` for `add`, `-1` for `del`. - #[allow(dead_code)] - Query { - target: DataflowRef, - kind: QueryKind, - key: Vec, - val: Vec, - }, /// Advance ambient time by `n` (default 1). Tick { n: u64 }, /// End the session. Exit, } -#[derive(Debug, Clone, Copy)] -pub enum QueryKind { - Add, - Del, -} - -/// A reference to a registered dataflow, used by both `drop` and -/// `query`. Either a numeric dataflow id or a name (the load's -/// `id_hint`). Parsed by reading the token as a `u64` first, then -/// falling back to a string name. So `drop 5` and `drop my_reach` -/// both work, and `drop 5_alt` (which fails to parse as u64) falls -/// through to the name lookup. -#[derive(Debug, Clone)] -pub enum DataflowRef { - Id(u64), - Name(String), -} - /// A command ready to broadcast to the worker group. Protocol-only tail /// lifecycle wraps the typed server command vocabulary rather than leaking /// response channels into the dataflow. @@ -172,17 +143,7 @@ pub fn prepare(command: Cmd) -> Result { program, } } - Cmd::Drop { target } => ServerCommand::Drop { - name: match target { - DataflowRef::Name(name) => name, - DataflowRef::Id(id) => { - return Err(format!( - "numeric dataflow id {} is no longer exposed; use its name", - id - )) - } - }, - }, + Cmd::Drop { name } => ServerCommand::Drop { name }, Cmd::List => ServerCommand::List, Cmd::Peek { name, key } => ServerCommand::Peek { trace: name, key }, Cmd::Tail { name } => return Ok(PreparedCommand::Tail { name }), @@ -214,9 +175,6 @@ pub fn prepare(command: Cmd) -> Result { Cmd::Source { prog, input, source } => ServerCommand::Load { prog, input, source }, Cmd::Bind { trace, prog, input } => ServerCommand::Bind { trace, prog, input }, Cmd::Unbind { trace, prog, input } => ServerCommand::Unbind { trace, prog, input }, - Cmd::Query { .. } => { - return Err("query is reserved for --explain dataflows and is not implemented".into()) - } Cmd::Tick { n } => ServerCommand::Tick { n }, Cmd::Exit => ServerCommand::Exit, }; @@ -295,8 +253,8 @@ pub struct Request { } /// State carried between lines so the parser can splice a multi-line load or -/// feed body together. The parser hands back either a complete `Request` or -/// `None` (more lines required). +/// feed body together. The parser hands back a request id with its parsed +/// command (or parse error), or `None` when more lines are required. #[derive(Default)] pub struct LineParser { pending_load: Option, @@ -309,7 +267,7 @@ pub struct LineParser { /// Tokens that introduce a command. If a line begins with one of these /// instead of an explicit reqid, the parser synthesizes a reqid. const COMMAND_KEYWORDS: &[&str] = &[ - "load", "drop", "list", "peek", "tail", "stop", "tick", "query", "exit", "feed", "bind", + "load", "drop", "list", "peek", "tail", "stop", "tick", "exit", "feed", "bind", "unbind", ]; @@ -683,63 +641,6 @@ fn parse_cmd(cmd: &str, args: &[&str]) -> ParseOutcome { }, } } - "query" => { - // Syntax: `query add|del ; ` - // Where k/v-fields are comma-separated i64. Empty side allowed - // (write nothing before/after the `;`). - if args.len() < 3 { - return ParseOutcome::Err( - "query: expected ` add|del ; `".into(), - ); - } - let target = match args[0].parse::() { - Ok(n) => DataflowRef::Id(n), - Err(_) => DataflowRef::Name(args[0].to_string()), - }; - let kind = match args[1] { - "add" => QueryKind::Add, - "del" => QueryKind::Del, - other => { - return ParseOutcome::Err(format!( - "query: kind must be add|del, got {:?}", - other - )) - } - }; - // Find the `;` separator among the remaining tokens. - let rest = &args[2..]; - let sep = rest.iter().position(|t| *t == ";"); - let (k_toks, v_toks): (&[&str], &[&str]) = match sep { - Some(i) => (&rest[..i], &rest[i + 1..]), - None => (rest, &[]), - }; - fn parse_fields(toks: &[&str]) -> Result, String> { - let mut out = Vec::new(); - for t in toks { - for piece in t.split(',') { - if piece.is_empty() { - continue; - } - out.push(piece.parse().map_err(|_| format!("bad i64 {:?}", piece))?); - } - } - Ok(out) - } - let key = match parse_fields(k_toks) { - Ok(v) => v, - Err(e) => return ParseOutcome::Err(format!("query key: {}", e)), - }; - let val = match parse_fields(v_toks) { - Ok(v) => v, - Err(e) => return ParseOutcome::Err(format!("query val: {}", e)), - }; - ParseOutcome::Cmd(Cmd::Query { - target, - kind, - key, - val, - }) - } "feed" => { if let [prog, input, "begin"] = args { let input = match input.parse() { @@ -848,14 +749,8 @@ fn parse_cmd(cmd: &str, args: &[&str]) -> ParseOutcome { _ => ParseOutcome::Err(format!("{}: expected ` `", cmd)), }, "drop" => match args { - [tok] => { - let target = match tok.parse::() { - Ok(n) => DataflowRef::Id(n), - Err(_) => DataflowRef::Name((*tok).to_string()), - }; - ParseOutcome::Cmd(Cmd::Drop { target }) - } - _ => ParseOutcome::Err("drop: expected ``".into()), + [name] => ParseOutcome::Cmd(Cmd::Drop { name: (*name).to_string() }), + _ => ParseOutcome::Err("drop: expected ``".into()), }, "list" => match args { [] => ParseOutcome::Cmd(Cmd::List), @@ -939,12 +834,7 @@ mod tests { assert_eq!(got.len(), 4); assert!(matches!(got[0].1, Ok(Cmd::List))); assert!(matches!(got[1].1, Ok(Cmd::Tick { n: 5 }))); - assert!(matches!( - got[2].1, - Ok(Cmd::Drop { - target: DataflowRef::Id(3) - }) - )); + assert!(matches!(got[2].1, Ok(Cmd::Drop { ref name }) if name == "3")); assert!(matches!(got[3].1, Ok(Cmd::Peek { ref name, key: None }) if name == "foo")); } @@ -1038,27 +928,6 @@ mod tests { assert!(matches!(&got[2].1, Err(_))); } - #[test] - fn query_cmd() { - let mut p = LineParser::new(); - let got = feed_all(&mut p, &["rQ query 3 add 1,2 ; 99"]); - assert_eq!(got.len(), 1); - match &got[0].1 { - Ok(Cmd::Query { - target, - kind, - key, - val, - }) => { - assert!(matches!(target, DataflowRef::Id(3))); - assert!(matches!(kind, QueryKind::Add)); - assert_eq!(key, &vec![1, 2]); - assert_eq!(val, &vec![99]); - } - _ => panic!("expected Query, got {:?}", got[0].1), - } - } - #[test] fn auto_reqid_for_bare_command() { let mut p = LineParser::new(); @@ -1221,15 +1090,8 @@ mod tests { ], ); assert_eq!(got.len(), 3); - assert!(matches!( - got[0].1, - Ok(Cmd::Drop { - target: DataflowRef::Id(3) - }) - )); - assert!( - matches!(got[1].1, Ok(Cmd::Drop { target: DataflowRef::Name(ref n) }) if n == "my_reach") - ); + assert!(matches!(got[0].1, Ok(Cmd::Drop { ref name }) if name == "3")); + assert!(matches!(got[1].1, Ok(Cmd::Drop { ref name }) if name == "my_reach")); assert!(matches!(got[2].1, Err(_))); } } diff --git a/interactive/server/src/main.rs b/interactive/server/src/main.rs index b2654b462..7502d3a8d 100644 --- a/interactive/server/src/main.rs +++ b/interactive/server/src/main.rs @@ -12,8 +12,8 @@ //! independently. //! //! Threads: -//! - main: spawns the worker, the TCP listener, and the stdin session; -//! then waits for the worker to finish. +//! - main: spawns the worker, the TCP and WebSocket listeners, and the stdin +//! session; then waits for the worker to finish. //! - worker 0: admits the transport FIFO to the distributed control stream. //! - all workers: replay the same registry + dispatch operations. //! - per-session reader: parses lines into commands, tags each with this @@ -27,7 +27,7 @@ mod control_loop; use mimalloc::MiMalloc; -/// The allocator the retired example driver ran on; the arrangement-heavy paths lean on it. +/// The arrangement-heavy paths lean on mimalloc. #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; diff --git a/interactive/src/backend/corgi.rs b/interactive/src/backend/corgi.rs index 636d58105..4c1bb3e34 100644 --- a/interactive/src/backend/corgi.rs +++ b/interactive/src/backend/corgi.rs @@ -4,7 +4,7 @@ //! comparison, but its representation choices do not define corgi's physical semantics. //! //! All `Backend` methods are corgi-native: `linear` folds a `LinearOp` chain over each container -//! ([`apply_ops`], columnar fast paths with row-wise fallbacks); `arrange` ingests columns without +//! ([`apply_ops`]); `arrange` ingests columns without //! a row round-trip; `join`/`reduce` run through the int-proxy tactics ([`CorgiJoinBackend`], //! [`CorgiReduceBackend`]) over the columnar chunks. @@ -335,7 +335,7 @@ impl Backend for CorgiBackend { } fn leave_dynamic<'s>(c: Collection<'s, Time, CC>, level: usize) -> Collection<'s, Time, CC> { - // Mirror DD's `Collection::leave_dynamic` (dynamic/mod.rs:40), but over a `CorgiContainer`: + // Mirror DD's `Collection::leave_dynamic`, but over a `CorgiContainer`: // strip all but `level-1` PointStamp coordinates from the capability AND from each row's time // (stored columnar in `CorgiContainer.times`, not inline in the data tuples). The input // connection summary advertises the `retain` so timely's progress tracking stays correct. diff --git a/interactive/src/corgi/chunk.rs b/interactive/src/corgi/chunk.rs index 05eb7d347..a78a38ad3 100644 --- a/interactive/src/corgi/chunk.rs +++ b/interactive/src/corgi/chunk.rs @@ -480,8 +480,6 @@ where } -/// Concatenate chunks' columns into flat `(keys, vals, times, diffs)` with **no transcode** — for -/// reading an arrangement back column-natively (e.g. `Backend::as_collection` straight into a /// Build a `ChunkBatch` from corgi key/val COLUMNS directly (no transcode): sort + /// consolidate into one chunk, then `settle`. The column-native egress the reduce backend seals its /// output with (it resolves proxy ids to real columns by `gather` and hands them here). @@ -494,7 +492,7 @@ where settle_one(chunk) } -/// Grade one chunk into a `ChunkBatch` (shared tail of `rows_to_batch`/`columns_to_batch`). +/// Grade one chunk into a `ChunkBatch`. fn settle_one(chunk: CorgiChunk) -> ChunkBatch> where T: ColTime, @@ -512,7 +510,7 @@ where /// standard `ChunkBatcher`/`ChunkBuilder`, the arrange ingest stays column-native — no /// columns→rows→columns round-trip at the arrangement boundary. /// -/// Crucially it **accumulates to `TARGET`** before consolidating (like `ContainerChunker`), so it +/// Crucially it **accumulates to `INGEST`** before consolidating (like `ContainerChunker`), so it /// emits few large chunks rather than one tiny chunk per input container — otherwise the columnar /// per-chunk set-up (`gather`/`sort_perm`) dominates when input arrives as many small batches. pub struct CorgiChunker { @@ -659,7 +657,7 @@ where R: Semigroup + Clone + 'static, { type Container = CorgiChunk; - // `extract` ships ready chunks, leaving the sub-TARGET remainder to accumulate further. + // `extract` ships ready chunks, leaving the sub-`INGEST` remainder to accumulate further. fn extract(&mut self) -> Option<&mut Self::Container> { self.current = self.ready.pop_front(); self.current.as_mut() diff --git a/interactive/src/corgi/container.rs b/interactive/src/corgi/container.rs index 1bb4438b8..9a5c830c9 100644 --- a/interactive/src/corgi/container.rs +++ b/interactive/src/corgi/container.rs @@ -1,4 +1,4 @@ -//! Phase 2 — the corgi-native container: corgi columns for the (key,val) payload, times as a +//! The corgi-native container: corgi columns for the (key,val) payload, times as a //! lane column, diffs a plain Rust Vec (corgi never touches the lattice). This is what flows on //! dataflow edges in the corgi backend; operators transform block→block via `eval_graph` with NO //! per-op transcode. Conversion to/from DDIR rows happens only at I/O boundaries diff --git a/interactive/src/corgi/exchange.rs b/interactive/src/corgi/exchange.rs index 85d89786b..a058be32e 100644 --- a/interactive/src/corgi/exchange.rs +++ b/interactive/src/corgi/exchange.rs @@ -26,12 +26,10 @@ //! # The hash //! //! Routing uses [`corgi::hash`] — the same structural content hash that -//! [`present_key`](crate::corgi::chunk::present_key) prepends as a key's identifier lane. Any -//! deterministic function of the key would be correct here; choosing *this* one means the -//! distributor and the arrangement agree on what a key's identifier is, so the hash a receiver -//! recomputes is the one the sender routed by. (It is also seed-free and structural, so it agrees -//! across processes and across runs, and its low bits are mixed — a raw key column would route a -//! strided identifier space onto a fraction of the workers.) +//! [`present_key`](crate::corgi::chunk::present_key) prepends as a compound key's identifier +//! lane. Any deterministic function of the key would be correct here; this one is seed-free and +//! structural, so it agrees across processes and runs, and its low bits are mixed — a raw key +//! column would route a strided identifier space onto a fraction of the workers. use timely::communication::Push; use timely::dataflow::channels::Message; diff --git a/interactive/src/corgi/join.rs b/interactive/src/corgi/join.rs index 54f71caac..0f559472a 100644 --- a/interactive/src/corgi/join.rs +++ b/interactive/src/corgi/join.rs @@ -631,8 +631,7 @@ fn stage_collision( /// Two regimes: when one side is much smaller (the fresh delta against an accumulated /// trace), the small side DRIVES and the large side is presented only at the driver's keys /// (sorted probes — cost tracks the driver plus matches). When the sides are -/// comparable, probing costs `n log n` against a merge's `n`, so both sides are pulled and -/// merged symmetrically instead. +/// comparable, both sides are pulled and merged symmetrically instead. fn advance_leaf( chunks0: &[&CorgiChunk], chunks1: &[&CorgiChunk], @@ -767,7 +766,7 @@ fn leaf_probe<'a, T: ColTime>( } /// Comparable-sides regime: both sides pulled and merged symmetrically on the `u64` -/// buffers — a probe here would cost `n log n` against this merge's `n`. +/// buffers. fn leaf_merge<'a, T: ColTime>( mut views0: Vec>, mut views1: Vec>, diff --git a/interactive/src/corgi/logic.rs b/interactive/src/corgi/logic.rs index b99e326a2..ba8c78f85 100644 --- a/interactive/src/corgi/logic.rs +++ b/interactive/src/corgi/logic.rs @@ -10,10 +10,8 @@ //! The typer is corgi's. [`shape_of_term`] compiles a term into a scratch graph and asks //! `corgi::shape_of` (its evaluator on zero rows) for the result shape, so every shape rule — //! which lanes a `case` sees, what an `if` may blend, whether two operands compare — is the one -//! the kernels enforce, and a program that lowers is a program that runs. The compiler covers -//! Var/Bound/Int/Tuple(+Spread)/Proj/Binary/If/Fold, list and sum intro (`List`/`Inject`), sum -//! elimination (`Case`), and the Neg/Not/Len/IsTag unaries; `Err` is a type error, reported with -//! corgi's message. Ordered compares are signed-correct (`ToSigned`); `hash` is corgi's structural +//! the kernels enforce, and a program that lowers is a program that runs. `Err` is a type error, +//! reported with corgi's message. Ordered compares are signed-correct (`ToSigned`); `hash` is corgi's structural //! `Op::Hash`, the same function `ir::eval` folds row-wise. use crate::ir::{BinOp, SumTy, Term, UnOp, Value as DValue}; @@ -600,8 +598,7 @@ pub fn compile( // the existing kernel matrix, with no per-row work and no new corgi op — `Enlist` each // element (a length-1 lane per row), `Iota` a per-row `[0..k)` tag list, `Weave` // interleaves the lanes in field order into `List`, and `MapList(Unwrap)` - // strips the now-homogeneous sum (and reports a heterogeneous literal). A fused - // list-intro kernel is corgi's call if this composition ever profiles hot. + // strips the now-homogeneous sum (and reports a heterogeneous literal). Term::List(fields) => { if fields.is_empty() { let Some(Shape::List(element)) = expected else { diff --git a/interactive/src/explain/decouple.rs b/interactive/src/explain/decouple.rs index 217ffd103..ca43c2cc5 100644 --- a/interactive/src/explain/decouple.rs +++ b/interactive/src/explain/decouple.rs @@ -1,19 +1,12 @@ //! Data-model-agnostic surface for explain's reverse-tracing. //! -//! The reverse rules used to hard-code the flat `[i64]` row layout — building -//! projections as `FieldExpr` index ranges inline. This module factors that -//! out: the rules are written **once** over two traits, and the data model -//! supplies the representation. +//! The reverse rules are written once, over two traits: //! -//! * [`RowModel`] builds the projections/predicates each rule needs. +//! * [`RowModel`] builds the projections/predicates each rule needs; +//! [`crate::explain::Val`] is the model the crate uses. //! * [`Dataflow`] is the orchestration backend (concat/project/filter/join) — //! the scope builder `Sb` for real explain. //! -//! Changing the data model means reimplementing [`RowModel`]; the rules and the -//! orchestration are untouched. [`crate::explain::Val`] is the only model the -//! crate evaluates; the flat `[i64]` model this was factored out for, and the -//! `folded` algebra it used, are both retired. -//! //! ## The demand envelope //! //! Every method speaks in terms of one object — a **demand row**, written @@ -25,12 +18,11 @@ //! where `;` separates the *key* from the *value*. The value packs three //! logical parts, in order: //! -//! * **`V`** — the underlying data value (`v` columns in the flat model). The +//! * **`V`** — the underlying data value. The //! demanded output's value on the `dep` side, the candidate input's value on //! the pair-table side. //! * **`chain`** — the loop-iteration coordinates, innermost-first, length = -//! the node's scope depth. Time lives in data here; `folded` owns the -//! per-coordinate algebra (compare outer-aligned, strip). +//! the node's scope depth. Time lives in data here. //! * **`q`** — a single trailing query id, present on `dep` rows (the thing //! being explained), absent on pair-table rows. //! @@ -91,16 +83,11 @@ pub trait RowModel { // --- lossy lookup (Linear[Project]) --- - /// Fast path used when `proj` is invertible and the chains have equal length - /// (`in_len == out_len`): map `dep` directly to a contrib without the pair - /// table. `(K_out ; V_out, chain_out, q) -> (K_in ; V_in, chain_out, q)`, - /// reconstructing `(K_in, V_in)` from the output. `None` if not invertible. - fn lossy_try_invert(proj: &Self::Proj, k_in: usize, v_in: usize, k_out: usize, v_out: usize, out_len: usize) -> Option; - /// Fallback pair re-key: apply the user `proj.key` to a pair row to compute + /// Pair re-key: apply the user `proj.key` to a pair row to compute /// `K_out`, carrying the input data through. /// `(K_in ; V_in, chain_in) -> (K_out ; K_in, V_in, chain_in)`. fn lossy_pair(proj: &Self::Proj, k_in: usize, v_in: usize, in_len: usize) -> Self::Proj; - /// Reassemble after the fallback join on `K_out`. Input: + /// Reassemble after the join on `K_out`. Input: /// `[$0 = K_out, $1 = V_out ++ chain_out ++ q (dep), $2 = K_in ++ V_in ++ chain_in (pair)]`. /// Output: `(K_in ; V_in, chain_in, chain_out, q)`. fn lossy_reassemble(k_in: usize, v_in: usize, v_out: usize, in_len: usize, out_len: usize) -> Self::Proj; @@ -127,7 +114,7 @@ pub trait RowModel { #[allow(clippy::too_many_arguments)] fn join_split(left: bool, k: usize, v_l: usize, v_r: usize, l_len: usize, r_len: usize, out_len: usize) -> Self::Proj; - // --- the chain (`folded`) algebra, shared by SP / keyed / lossy --- + // --- the chain algebra, shared by SP / keyed / lossy --- /// Soundness filter on a `(K ; V, chain_in, chain_out, q)` row: keep rows /// with `chain_in ≤ chain_out`, compared outer-aligned (an input can only @@ -206,17 +193,12 @@ where M: RowModel, D: Dataflow { df.project(&filtered, M::strip(k, v_in, in_len, out_len, in_len)) } -/// Lossy (Project) lookup: invert fast-path (equal chain lengths) else pair table. +/// Lossy (Project) lookup, through the pair table. pub fn lossy_lookup(df: &mut D, dep: &D::Handle, side: &SideInfo, output_shape: (usize, usize), out_len: usize, proj: &M::Proj) -> D::Handle where M: RowModel, D: Dataflow { let (k_in, v_in) = side.shape; let in_len = side.user_len; - let (k_out, v_out) = output_shape; - if in_len == out_len { - if let Some(p) = M::lossy_try_invert(proj, k_in, v_in, k_out, v_out, out_len) { - return df.project(dep, p); - } - } + let (_, v_out) = output_shape; let pair = df.concat(vec![side.witness.clone(), side.forward.clone()]); let pair_keyed = df.project(&pair, M::lossy_pair(proj, k_in, v_in, in_len)); let joined = df.join(dep, &pair_keyed, M::lossy_reassemble(k_in, v_in, v_out, in_len, out_len)); @@ -243,14 +225,9 @@ where M: RowModel, D: Dataflow { (lc, rc) } -// The flat-row executable contract was removed with the `[i64]` model. The -// model-agnostic proof below (`nested_contract`) runs the same generic rules -// against a nested `Value`-shaped `RowModel` — the shape the real `explain::Val` -// model uses — so it remains the runnable spec for the reverse rules. - -/// In-memory [`Dataflow`] over `Vec<(Value, Value)>`, shared by the contract -/// modules below: projections and predicates run through the `Term` -/// interpreter (`ir::eval`), and `join` is a nested-loop equi-join on the key. +/// In-memory [`Dataflow`] over `Vec<(Value, Value)>`, for the contract below: +/// projections and predicates run through the `Term` interpreter (`ir::eval`), +/// and `join` is a nested-loop equi-join on the key. #[cfg(test)] mod mem { use super::Dataflow; @@ -291,258 +268,11 @@ mod mem { } } -#[cfg(test)] -mod nested_contract { - //! Proof that the trait is model-agnostic: a second `RowModel` over a - //! *nested* value (`Value::Tuple`/`Int`) — the shape an AST/JSON data model - //! would use — implemented against the SAME rules, run through the SAME - //! by-example specs as the flat model. Where the flat model lays the - //! envelope out positionally, this one nests it as `Tuple([V, chain, q])`; - //! e.g. the Min narrowing is one whole-`Value` equality, not a column loop. - //! If these pass, "swap the data model = reimplement `RowModel`" is earned. - - use super::*; - - /// Minimal nested value: an integer or a tuple. (`Variant`/`List` would - /// extend this; the reverse rules need only product nesting.) - #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] - enum Value { Int(i64), Tuple(Vec) } - use Value::{Int, Tuple}; - fn int(n: i64) -> Value { Int(n) } - fn tup(xs: Vec) -> Value { Tuple(xs) } - - /// A scalar term over a nested row. `Field`/`Tuple` are project/construct. - #[derive(Clone, Debug)] - enum Term { Var(usize), Field(Box, usize), Tup(Vec), Sub1(Box) } - fn var(s: usize) -> Term { Term::Var(s) } - fn fld(t: Term, i: usize) -> Term { Term::Field(Box::new(t), i) } - fn f(s: usize, i: usize) -> Term { fld(var(s), i) } - - #[derive(Clone, Debug)] - struct Proj { key: Term, val: Term } - #[derive(Clone, Debug)] - enum Pred { Le(Term, Term), Eq(Term, Term), Gt0(Term), And(Box, Box) } - - fn eval(t: &Term, env: &[Value]) -> Value { - match t { - Term::Var(s) => env[*s].clone(), - Term::Field(x, i) => match eval(x, env) { Tuple(xs) => xs[*i].clone(), v => panic!("field of {:?}", v) }, - Term::Tup(xs) => Tuple(xs.iter().map(|x| eval(x, env)).collect()), - Term::Sub1(x) => match eval(x, env) { Int(n) => Int(n - 1), v => panic!("sub1 of {:?}", v) }, - } - } - fn eval_pred(p: &Pred, env: &[Value]) -> bool { - match p { - Pred::Le(a, b) => eval(a, env) <= eval(b, env), - Pred::Eq(a, b) => eval(a, env) == eval(b, env), - Pred::Gt0(a) => matches!(eval(a, env), Int(n) if n > 0), - Pred::And(a, b) => eval_pred(a, env) && eval_pred(b, env), - } - } - - /// Redirect a user projection's value-access: `Var(r)` for `r` in `vars` - /// becomes `Var(r)[0]` (the `V` field of that slot's envelope). - fn subst(t: &Term, vars: &[usize]) -> Term { - match t { - Term::Var(r) if vars.contains(r) => f(*r, 0), - Term::Var(r) => var(*r), - Term::Field(x, i) => fld(subst(x, vars), *i), - Term::Tup(xs) => Term::Tup(xs.iter().map(|x| subst(x, vars)).collect()), - Term::Sub1(x) => Term::Sub1(Box::new(subst(x, vars))), - } - } - - type Row = Value; - type Coll = Vec<(Row, Row)>; - struct Mem; - impl Dataflow for Mem { - type Handle = Coll; - type Proj = Proj; - type Pred = Pred; - fn project(&mut self, c: &Coll, p: Proj) -> Coll { - c.iter().map(|(k, v)| { let e = [k.clone(), v.clone()]; (eval(&p.key, &e), eval(&p.val, &e)) }).collect() - } - fn filter(&mut self, c: &Coll, p: Pred) -> Coll { - c.iter().filter(|(k, v)| eval_pred(&p, &[k.clone(), v.clone()])).cloned().collect() - } - fn join(&mut self, l: &Coll, r: &Coll, p: Proj) -> Coll { - let mut out = Vec::new(); - for (lk, lv) in l { for (rk, rv) in r { - if lk == rk { let e = [lk.clone(), lv.clone(), rv.clone()]; out.push((eval(&p.key, &e), eval(&p.val, &e))); } - }} - out - } - fn concat(&mut self, cs: Vec) -> Coll { cs.into_iter().flatten().collect() } - } - - /// The nested model. `K`/`V` are single (opaque) values, so the column - /// counts `k`/`v`/… are ignored; only chain lengths matter for the algebra. - struct Nested; - fn chain_le(chain_in: Term, chain_out: Term, in_len: usize, out_len: usize) -> Option { - let n = in_len.min(out_len); - let (is, os) = (in_len - n, out_len - n); - (0..n).map(|i| Pred::Le(fld(chain_in.clone(), is + i), fld(chain_out.clone(), os + i))) - .reduce(|a, b| Pred::And(Box::new(a), Box::new(b))) - } - impl RowModel for Nested { - type Proj = Proj; - type Pred = Pred; - fn pack(_k: usize, _v: usize, _cl: usize, has_q: bool) -> Proj { - let key = Term::Tup(vec![var(0), f(1, 0)]); // [K, V] - let val = if has_q { Term::Tup(vec![f(1, 1), f(1, 2)]) } else { Term::Tup(vec![f(1, 1)]) }; - Proj { key, val } - } - fn distinct_unpack(_k: usize, _v: usize) -> Proj { Proj { key: f(0, 0), val: f(0, 1) } } - fn project_kv(_k: usize, _v: usize) -> Proj { Proj { key: var(0), val: f(1, 0) } } - fn sp_reassemble(_k: usize, _v: usize, _il: usize, _ol: usize) -> Proj { - // [pack[K,V], dep=(chain_out,q), pair=(chain_in)] - Proj { key: f(0, 0), val: Term::Tup(vec![f(0, 1), f(2, 0), f(1, 0), f(1, 1)]) } - } - fn ky_join(_k: usize, _vi: usize, _vo: usize, _il: usize, _ol: usize, min: bool) -> Proj { - // [K, dep=(V_out,chain_out,q), pair=(V_in,chain_in)] - let mut val = vec![f(2, 0)]; // V_in - if min { val.push(f(1, 0)); } // V_out - val.extend([f(2, 1), f(1, 1), f(1, 2)]); // chain_in, chain_out, q - Proj { key: var(0), val: Term::Tup(val) } - } - fn ky_data_eq(_vi: usize) -> Option { Some(Pred::Eq(f(1, 0), f(1, 1))) } // whole-Value - fn ky_drop_vout(_k: usize, _vi: usize, _vo: usize, _il: usize, _ol: usize) -> Proj { - Proj { key: var(0), val: Term::Tup(vec![f(1, 0), f(1, 2), f(1, 3), f(1, 4)]) } - } - fn lossy_try_invert(_p: &Proj, _ki: usize, _vi: usize, _ko: usize, _vo: usize, _ol: usize) -> Option { - None // a nested model may skip the invert optimization; the fallback is always sound. - } - fn lossy_pair(proj: &Proj, _ki: usize, _vi: usize, _il: usize) -> Proj { - Proj { key: subst(&proj.key, &[1]), val: Term::Tup(vec![var(0), f(1, 0), f(1, 1)]) } - } - fn lossy_reassemble(_ki: usize, _vi: usize, _vo: usize, _il: usize, _ol: usize) -> Proj { - // [K_out, dep=(V_out,chain_out,q), pair=(K_in,V_in,chain_in)] - Proj { key: f(2, 0), val: Term::Tup(vec![f(2, 1), f(2, 2), f(1, 1), f(1, 2)]) } - } - fn join_forward(proj: &Proj, _k: usize, _vl: usize, _vr: usize, _ll: usize, _rl: usize) -> Proj { - // [K, left=(V_L,chain_L), right=(V_R,chain_R)] - let key = subst(&proj.key, &[1, 2]); - let v_out = subst(&proj.val, &[1, 2]); - Proj { key, val: Term::Tup(vec![var(0), f(1, 0), f(2, 0), f(1, 1), f(2, 1), v_out]) } - } - fn join_combined(_k: usize, _vl: usize, _vr: usize, _vo: usize, _ll: usize, _rl: usize, _ol: usize) -> Proj { - // [K_out, dep=(V_out,chain_out,q), pair=(K,V_L,V_R,chain_L,chain_R,V_out)] - Proj { key: f(2, 0), val: Term::Tup(vec![ - f(2, 1), f(2, 2), f(2, 3), f(2, 4), f(1, 1), f(1, 2), f(1, 0), f(2, 5), - ]) } - } - fn join_filter(_vl: usize, _vr: usize, _vo: usize, ll: usize, rl: usize, ol: usize) -> Option { - // wide val: V_L,V_R,chain_L,chain_R,chain_out,q,V_out_dep,V_out_pair - let conds = [ - chain_le(f(1, 2), f(1, 4), ll, ol), - chain_le(f(1, 3), f(1, 4), rl, ol), - Some(Pred::Eq(f(1, 6), f(1, 7))), - ]; - conds.into_iter().flatten().reduce(|a, b| Pred::And(Box::new(a), Box::new(b))) - } - fn join_split(left: bool, _k: usize, _vl: usize, _vr: usize, _ll: usize, _rl: usize, _ol: usize) -> Proj { - let val = if left { Term::Tup(vec![f(1, 0), f(1, 2), f(1, 5)]) } - else { Term::Tup(vec![f(1, 1), f(1, 3), f(1, 5)]) }; - Proj { key: var(0), val } - } - fn time_le(_v: usize, il: usize, ol: usize) -> Option { - chain_le(f(1, 1), f(1, 2), il, ol) - } - fn strip(_k: usize, _v: usize, il: usize, _ol: usize, keep: usize) -> Proj { - let drop = il - keep; - let chain = Term::Tup((0..keep).map(|i| fld(f(1, 1), drop + i)).collect()); - Proj { key: var(0), val: Term::Tup(vec![f(1, 0), chain, f(1, 3)]) } - } - fn bind_filter(_v: usize) -> Option { Some(Pred::Gt0(fld(f(1, 1), 0))) } - fn bind_decrement(_k: usize, _v: usize, user_len: usize) -> Proj { - let mut coords = vec![Term::Sub1(Box::new(fld(f(1, 1), 0)))]; - coords.extend((1..user_len).map(|i| fld(f(1, 1), i))); - Proj { key: var(0), val: Term::Tup(vec![f(1, 0), Term::Tup(coords), f(1, 2)]) } - } - } - - fn side(witness: Coll, user_len: usize) -> SideInfo { - SideInfo { witness, forward: vec![], shape: (1, 1), user_len } - } - fn chain(cs: &[i64]) -> Value { tup(cs.iter().map(|n| int(*n)).collect()) } - - // The same six specs as the flat contract, in nested envelopes. - - #[test] - fn sp_depth0_keeps_only_value_matching_pair() { - // pair vals are (V, chain); dep val is (V, chain, q). - let pair = vec![ - (int(5), tup(vec![int(7), chain(&[])])), - (int(5), tup(vec![int(8), chain(&[])])), - ]; - let dep = vec![(int(5), tup(vec![int(7), chain(&[]), int(9)]))]; - let out = shape_preserving_lookup::(&mut Mem, &dep, &side(pair, 0), 0); - assert_eq!(out, vec![(int(5), tup(vec![int(7), chain(&[]), int(9)]))]); - } - - #[test] - fn sp_depth1_time_filters_late_inputs() { - let dep = vec![(int(5), tup(vec![int(7), chain(&[3]), int(9)]))]; - let keep = vec![(int(5), tup(vec![int(7), chain(&[2])]))]; // iter 2 ≤ 3 - let drop = vec![(int(5), tup(vec![int(7), chain(&[4])]))]; // iter 4 > 3 - let kept = shape_preserving_lookup::(&mut Mem, &dep, &side(keep, 1), 1); - assert_eq!(kept, vec![(int(5), tup(vec![int(7), chain(&[2]), int(9)]))]); - let dropped = shape_preserving_lookup::(&mut Mem, &dep, &side(drop, 1), 1); - assert!(dropped.is_empty()); - } - - #[test] - fn keyed_min_narrows_to_the_demanded_value() { - let pair = vec![(int(5), tup(vec![int(7), chain(&[])])), (int(5), tup(vec![int(6), chain(&[])]))]; - let dep = vec![(int(5), tup(vec![int(7), chain(&[]), int(9)]))]; - let out = keyed_lookup::(&mut Mem, &dep, &side(pair, 0), (1, 1), 0, true); - assert_eq!(out, vec![(int(5), tup(vec![int(7), chain(&[]), int(9)]))]); - } - - #[test] - fn keyed_nonmin_demands_all_same_key_inputs() { - let pair = vec![(int(5), tup(vec![int(7), chain(&[])])), (int(5), tup(vec![int(6), chain(&[])]))]; - let dep = vec![(int(5), tup(vec![int(1), chain(&[]), int(9)]))]; - let mut out = keyed_lookup::(&mut Mem, &dep, &side(pair, 0), (1, 1), 0, false); - out.sort(); - assert_eq!(out, vec![ - (int(5), tup(vec![int(6), chain(&[]), int(9)])), - (int(5), tup(vec![int(7), chain(&[]), int(9)])), - ]); - } - - #[test] - fn lossy_via_fallback_recovers_input() { - // proj: K_out = V_in (Var 1), V_out = K_in (Var 0). No invert -> fallback. - let proj = Proj { key: var(1), val: var(0) }; - let pair = vec![(int(3), tup(vec![int(8), chain(&[])]))]; // (K_in=3 ; V_in=8) - let dep = vec![(int(8), tup(vec![int(3), chain(&[]), int(9)]))]; // (K_out=8 ; V_out=3, q=9) - let out = lossy_lookup::(&mut Mem, &dep, &side(pair, 0), (1, 1), 0, &proj); - assert_eq!(out, vec![(int(3), tup(vec![int(8), chain(&[]), int(9)]))]); - } - - #[test] - fn join_demands_both_inputs() { - let proj = Proj { key: var(0), val: var(1) }; // K_out = K, V_out = V_L - let left = side(vec![(int(5), tup(vec![int(7), chain(&[])]))], 0); - let right = side(vec![(int(5), tup(vec![int(9), chain(&[])]))], 0); - let dep = vec![(int(5), tup(vec![int(7), chain(&[]), int(1)]))]; - let (lc, rc) = join_lookup::(&mut Mem, &dep, &left, &right, (1, 1), 0, &proj); - assert_eq!(lc, vec![(int(5), tup(vec![int(7), chain(&[]), int(1)]))]); - assert_eq!(rc, vec![(int(5), tup(vec![int(9), chain(&[]), int(1)]))]); - } -} - #[cfg(test)] mod value_contract { - //! Executable contract for the reverse rules over the real `Value` model. - //! - //! The same by-example specs as the (removed) flat `[i64]` contract, but on - //! `Value` rows in `Val`'s flat envelope `[V | chain | q]`, run through an - //! in-memory `Value` dataflow against `crate::explain::Val` — the unit-level - //! spec for the model the crate actually evaluates. (`nested_contract` above - //! proves the *rules* are model-agnostic with a different, nested layout; - //! this pins the model the backend runs.) + //! Executable contract for the reverse rules: by-example specs on `Value` + //! rows in `Val`'s envelope `[V | chain | q]`, run through the in-memory + //! dataflow against `crate::explain::Val`. use super::*; use crate::explain::Val; @@ -604,8 +334,7 @@ mod value_contract { #[test] fn lossy_via_fallback_recovers_input() { - // proj: K_out = $1 (V_in), V_out = $0 (K_in). Val always takes the - // pair-table fallback (lossy_try_invert returns None). + // proj: K_out = $1 (V_in), V_out = $0 (K_in). let p = proj(spread(1), spread(0)); let pair = vec![(key(3), val(&[8]))]; // (K_in=3 ; V_in=8) let dep = vec![(key(8), val(&[3, 9]))]; // (K_out=8 ; V_out=3, q=9) @@ -625,50 +354,3 @@ mod value_contract { assert_eq!(rc, vec![(key(5), val(&[9, 1]))]); } } - -#[cfg(test)] -mod backstop { - //! The *universal backstop* reverses `flatmap` — the op the live rewrite - //! still `panic!`s on — using only the existing `Dataflow` primitives. The - //! forward clone runs the op and keys each output by itself, carrying the - //! input (the `(output -> input)` pair table); the reverse is one join on - //! the output plus a `REFORM` projection. No op-supplied inverse: the `None` - //! endpoint, so `RESIDUAL` is the whole input (here, the list). This pins - //! "the gap is closable" before the real rule + wiring are built. - - use super::mem::{Coll, Mem}; - use super::*; - use crate::ir::Value; - use crate::ir::{Projection, Term}; - - fn int(n: i64) -> Value { Value::Int(n) } - fn list(xs: &[i64]) -> Value { Value::List(xs.iter().map(|&n| int(n)).collect()) } - fn tup(xs: Vec) -> Value { Value::Tuple(xs) } - fn f(s: usize, i: usize) -> Term { Term::Proj(Box::new(Term::Var(s)), i) } - fn tterm(xs: Vec) -> Term { Term::Tuple(xs) } - - fn flatmap_forward(k: &Value, lst: &Value) -> Coll { - let Value::List(xs) = lst else { panic!("flatmap on non-list") }; - xs.iter().enumerate().map(|(p, e)| (k.clone(), tup(vec![int(p as i64), e.clone()]))).collect() - } - - #[test] - fn backstop_reverses_flatmap() { - let witness: Coll = vec![ - (int(1), list(&[10, 20, 30])), - (int(1), list(&[40, 50])), - (int(2), list(&[30])), - ]; - let pairs: Coll = witness.iter().flat_map(|(k, lst)| { - flatmap_forward(k, lst).into_iter().map(move |(ok, ov)| { - let Value::Tuple(o) = &ov else { unreachable!() }; - (tup(vec![ok.clone(), o[0].clone(), o[1].clone()]), tup(vec![k.clone(), lst.clone()])) - }) - }).collect(); - let demand: Coll = vec![(tup(vec![int(1), int(2), int(30)]), tup(vec![int(9)]))]; - let reform = Projection { key: f(2, 0), val: tterm(vec![f(2, 1), f(1, 0)]) }; - let mut df = Mem; - let got = df.join(&demand, &pairs, reform); - assert_eq!(got, vec![(int(1), tup(vec![list(&[10, 20, 30]), int(9)]))]); - } -} diff --git a/interactive/src/explain/mod.rs b/interactive/src/explain/mod.rs index d31054d05..ccc2afe87 100644 --- a/interactive/src/explain/mod.rs +++ b/interactive/src/explain/mod.rs @@ -14,13 +14,10 @@ //! user-iter coordinates folded into the value, innermost first — at the //! embedding level. //! -//! This is the tree form of the flat rewrite's `host` map. There it required -//! positional scope tracking, a pending pile per scope, depth-offset -//! arithmetic for each `leave`, and a fix-up pass for `Leave` aliasing; here -//! it is "a scope exports its lifted internals", and the cascade through -//! enclosing scopes is the recursion. The embedding depth is not a parameter: -//! the renderer derives depth structurally, so a clone needn't know where it -//! will sit. +//! A scope exports its lifted internals, and the cascade through enclosing +//! scopes is the recursion. The embedding depth is not a parameter: the +//! renderer derives depth structurally, so a clone needn't know where it will +//! sit. use crate::ir::LinearOp; use crate::scope_ir::{Bind, Export, Import, Item, Node, Program, Ref, Scope, Source, Var}; @@ -179,21 +176,12 @@ fn clone_rec(orig: &Scope, out: &mut Scope, import_map: &[Ref], path: &[usize]) // ===== The explanation transform ===== // -// `explain(p)` produces a Program whose execution yields per-source -// demand-set explanations for queries against `p`'s first export. Output -// shape: root { sources, query input, witness clone } and an iterative -// `explain` scope { demand-set vars, forward clone on demanded rows, -// reverse-tracing ops, demand exports }. -// -// The reverse dataflow is *flat inside the explain scope* by design: demand -// rows carry the user-iteration chain folded into the value (the `folded` -// layout), so no nesting is needed. The per-op reverse rules port from the -// flat rewrite nearly unchanged; what the tree changes is the boundary -// bookkeeping. Flat `Leave` had a special backward rule injecting the inner -// user-chain coordinate; here a reference is *resolved* through explicit -// import/export edges to the value site it names, and the ordinary -// shape-preserving lookup against that site's host form injects or strips -// coordinates as the depths dictate. No op needs to know about boundaries. +// The reverse dataflow is flat inside the explain scope: demand rows carry the +// user-iteration chain in the value, so no nesting is needed. A reference is +// *resolved* through explicit import/export edges to the value site it names, +// and the ordinary shape-preserving lookup against that site's host form +// injects or strips coordinates as the depths dictate. No op needs to know +// about boundaries. use std::collections::BTreeMap; use crate::ir::{BinOp, Projection, Reducer, Term}; @@ -430,8 +418,6 @@ impl Sb { /// One upstream edge into a backward rule: the target's host-side `(data, /// user)` collections from both clones, its shape, and its user-chain length. -/// (The flat version carried two lengths that diverged at `Leave`; with -/// per-site hosts there is one.) struct Side { witness: Ref, forward: Ref, @@ -470,8 +456,7 @@ impl Sb { decouple::keyed_lookup::(self, &dep_y, &side.info(), output_shape, out_user_len, min) } - /// Lossy lookup (Linear[Project]): pure-map shortcut when invertible and - /// same-scope, pair-table fallback otherwise. + /// Lossy lookup (Linear[Project]), through the pair table. fn emit_lookup_lossy(&mut self, dep_y: Ref, side: &Side, output_shape: (usize, usize), out_user_len: usize, proj: &Projection) -> Ref { decouple::lossy_lookup::(self, &dep_y, &side.info(), output_shape, out_user_len, proj) } @@ -528,8 +513,7 @@ impl Sb { /// The `Value` data model for explain: a demand row is `(K ; Tuple([V…, chain…, /// q]))` — a flat value tuple of `V`'s fields, then the loop-iteration chain /// (innermost-first), then the trailing query id, matching the host lift's -/// `append_iter`. Every builder works in field-index ranges; the flat `[i64]` -/// model implemented the same trait over `[i64]` column ranges. +/// `append_iter`. Every builder works in field-index ranges. pub(crate) struct Val; impl Dataflow for Sb { @@ -620,9 +604,7 @@ impl RowModel for Val { Projection { key: tup(fidx(0, 0, k)), val: tup(val) } } - fn lossy_try_invert(_p: &Projection, _ki: usize, _vi: usize, _ko: usize, _vo: usize, _ol: usize) -> Option { - None // always take the pair-table fallback; `lossy_pair` bounds Spread so it is sound. - } + // Bounds Spread, so the pair table serves every projection. fn lossy_pair(proj: &Projection, k_in: usize, v_in: usize, in_len: usize) -> Projection { let mut val = fidx(0, 0, k_in); val.extend(fidx(1, 0, v_in + in_len)); @@ -1000,8 +982,8 @@ impl<'a> Reverse<'a> { } LinearOp::Negate | LinearOp::EnterAt(_) => { // Negate: pure pass-through. EnterAt: sound but - // over-broad pass-through (see the flat rule's note); - // the routing adapter handles any depth difference. + // over-broad pass-through; the routing adapter handles + // any depth difference. self.push(ex, path, input, dep_this, out_user_len); } LinearOp::LiftIter => panic!("explain: LiftIter in user program"), diff --git a/interactive/src/ir.rs b/interactive/src/ir.rs index 26072ed6b..61c67c068 100644 --- a/interactive/src/ir.rs +++ b/interactive/src/ir.rs @@ -86,8 +86,7 @@ pub enum Term { /// List intro. A `Spread` child splices in place. List(Vec), /// Splice marker; only meaningful as a direct child of `Tuple`/`List`. - /// Lets a whole input row (`$n`) contribute all its fields, preserving - /// the flat-row concatenation the original `[i64]` model relied on. + /// Lets a whole input row (`$n`) contribute all its fields. Spread(Box), /// Product/list elimination: index into a `Tuple` or `List`. Proj(Box, usize), @@ -109,7 +108,7 @@ pub enum Term { Unary(UnOp, Box), Binary(BinOp, Box, Box), /// `hash(bound, keys…)`: a deterministic pseudo-random `Int` in `[0, bound)` - /// (the raw non-negative hash if `bound <= 0`), mixed from the key `Int`s. + /// (the raw non-negative hash if `bound <= 0`), mixed from the keys. /// The building block for generators derived from `iota`/`clock`. Hash(Vec), } @@ -197,7 +196,8 @@ pub enum LinearOp { // DDIR's `hash` IS corgi's structural hash, evaluated a row at a time here and a column at a // time in the corgi backend. The two must agree bit for bit — they are the same program value, // and the backends are checked against each other — so this is a transcription of -// `corgi::hash`'s fold, not an independent design. `roundtrip_hash_matches_corgi` pins it. +// `corgi::hash`'s fold, not an independent design. The `hash_matches_corgi_*` tests in +// `corgi::logic` pin it. // // The values are DDIR's; the shapes they transcode to are corgi's, and the fold follows those: // `Int` is a `Prim` leaf, the empty `Tuple` is `Unit` (NOT a fieldless `Prod`), a `Tuple` is a diff --git a/interactive/src/lower.rs b/interactive/src/lower.rs index 4469befa1..3941af0f2 100644 --- a/interactive/src/lower.rs +++ b/interactive/src/lower.rs @@ -82,8 +82,8 @@ fn collect_body_free_names<'a>(body: &'a [Stmt], out: &mut BTreeSet<&'a str>) { // Produces the tree IR (see `scope_ir`): each `{ .. }` becomes an // owned child scope, cross-scope flow becomes explicit import/export edges, // and feedback vars are first-class. `input`/`import` external sources are -// accepted at the root scope only. Shapes are not stored on the tree; a -// shape pass derives them when a consumer needs them. +// accepted at the root scope only. Shapes are not stored on the tree (beyond +// an import's optional contract); they are derived from the data where needed. use crate::scope_ir as st; @@ -159,8 +159,8 @@ impl ScopeLower { Expr::LiftIter(e) => { let r = self.lower_expr(e); self.push(st::Node::Linear { input: r, ops: vec![LinearOp::LiftIter] }) }, Expr::Arrange(e) => { let r = self.lower_expr(e); self.push(st::Node::Arrange(r)) }, // Join/Reduce consume arrangements; arrange their inputs explicitly - // (as the flat lowering does) so identical arrangements are visible - // to `optimize`'s within-scope dedup and shared at render. + // so identical arrangements are visible to `optimize`'s within-scope + // dedup and shared at render. Expr::Join(l, r, p) => { let lr = self.lower_expr(l); let la = self.push(st::Node::Arrange(lr)); let rr = self.lower_expr(r); let ra = self.push(st::Node::Arrange(rr)); diff --git a/interactive/src/scope_ir.rs b/interactive/src/scope_ir.rs index fb0ffd257..f7d9b2067 100644 --- a/interactive/src/scope_ir.rs +++ b/interactive/src/scope_ir.rs @@ -6,12 +6,9 @@ //! # Why a tree //! //! A scope boundary is a real semantic barrier: an operator cannot be hoisted -//! across it, and an arrangement cannot be shared across it freely. The flat -//! IR encodes boundaries *positionally* (`Scope`/`EndScope` markers in a node -//! list), so every consumer that cares reconstructs them by analysis — and -//! that reconstruction is where the explanation rewrite's level errors lived. -//! Here the boundary is structural: a `Scope` owns its items, and nothing -//! crosses except through an explicit import or export. +//! across it, and an arrangement cannot be shared across it freely. So the +//! boundary is structural: a `Scope` owns its items, and nothing crosses except +//! through an explicit import or export. //! //! The guiding principle throughout: make explicit anything a consumer would //! otherwise have to analyze the IR to recover. Feedback variables are a @@ -89,9 +86,8 @@ pub struct Export { } /// A feedback variable, identified by its index in `Scope::vars`. Carries its -/// source name (readability, cross-scope name visibility). Its shape is *not* -/// stored — the shape pass derives every node/var shape when a consumer needs -/// it, so storing it here would cache a derivable (and currently unknown) value. +/// source name (readability, cross-scope name visibility). Its shape is not +/// stored; shapes are derived from the data where needed. #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct Var { pub name: String, @@ -125,8 +121,8 @@ pub enum Item { Sub(Scope), } -/// A scope owns its IR. It iterates iff it has `vars` (no `kind` field yet; see -/// the design doc). Children are owned, inline in `items`. +/// A scope owns its IR. It iterates iff it has `vars`. Children are owned, +/// inline in `items`. #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] pub struct Scope { /// The source name of the `{ .. }` block ("root" for the program root). @@ -179,10 +175,7 @@ impl Scope { /// - deduplicate structurally identical operators. /// /// All three are *within-scope*: a scope boundary is a semantic barrier, - /// so nothing merges or moves across one. (The flat IR's dedup was - /// position-blind and could merge across boundaries — sound only because - /// the dynamic model has no structural nesting; here the structure makes - /// the restriction automatic.) + /// so nothing merges or moves across one. pub fn optimize(&mut self) { for item in self.items.iter_mut() { if let Item::Sub(child) = item { child.optimize(); } diff --git a/interactive/src/server.rs b/interactive/src/server.rs index 0bd6473de..d835762e8 100644 --- a/interactive/src/server.rs +++ b/interactive/src/server.rs @@ -20,8 +20,7 @@ //! # The two binding points //! //! The named-trace IR (`import "x"` / `export "y"`) flows through parse → lower -//! → `scope_ir`; every batch backend simply `panic!`s on a non-`Input` source -//! because it has no registry. The server resolves both ends: +//! → `scope_ir`. The server resolves both ends: //! //! - **`Source::Trace(name)`** — `import` the registered [`ServerTrace`] into the //! new dataflow and feed it as a root collection. @@ -522,7 +521,7 @@ impl Server { .collect(); // Render the program body in its own iterative scope, then bring - // every export back out to the host time (mirrors `vec::evaluate`). + // every export back out to the host time. let leaved: Vec> = outer .iterative::, _, _>(|inner| { let entered: Vec<_> =