diff --git a/Cargo.lock b/Cargo.lock index 45a2bb39f..73539bd61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5766,6 +5766,7 @@ dependencies = [ name = "omnigraph-engine" version = "0.14.1" dependencies = [ + "ahash", "aho-corasick", "arc-swap", "arrow-array", @@ -5781,6 +5782,7 @@ dependencies = [ "csv-core", "datafusion", "futures", + "hashbrown 0.17.1", "lance", "lance-core", "lance-datafusion", diff --git a/changelog.d/traversal-per-row-work.changed.md b/changelog.d/traversal-per-row-work.changed.md new file mode 100644 index 000000000..22dd5624b --- /dev/null +++ b/changelog.d/traversal-per-row-work.changed.md @@ -0,0 +1 @@ +- Traversals allocate less per row: one-pass expand sizing, bulk-hashed joins, one visited table per walk, engine-side key sieving, block predicates run once. Results are unchanged. diff --git a/crates/omnigraph/Cargo.toml b/crates/omnigraph/Cargo.toml index da2f65566..8d35801e4 100644 --- a/crates/omnigraph/Cargo.toml +++ b/crates/omnigraph/Cargo.toml @@ -41,6 +41,9 @@ lance = { workspace = true } lance-core = { workspace = true } lance-select = { workspace = true } lance-datafusion = { workspace = true } +ahash = { workspace = true } +# The hash join's build side: the `HashTable` DataFusion's own join uses. +hashbrown = "0.17" datafusion = { workspace = true } lance-file = { workspace = true } lance-index = { workspace = true } diff --git a/crates/omnigraph/src/engine/graph.rs b/crates/omnigraph/src/engine/graph.rs index 8f32975ef..75276b39a 100644 --- a/crates/omnigraph/src/engine/graph.rs +++ b/crates/omnigraph/src/engine/graph.rs @@ -474,13 +474,73 @@ pub(super) fn endpoint_probes( probes } -/// At most `batch_size` emitted ID pairs, charged to `_memory`. +/// At most `batch_size` emitted pairs, charged to `_memory`: the source row +/// ordinals and the destination ids as one `Utf8` column built from the +/// walk's dictionary. pub(super) struct ExpandedPairs { pub(super) source_rows: Vec, - pub(super) destination_ids: Vec, + pub(super) destination: ArrayRef, _memory: WorkMemory, } +/// A per-source set member as one key: the source row in the high half, the +/// dense node in the low half, so every source's visited and emitted sets +/// share one table instead of one allocation per source row. +fn source_node_key(source: usize, node: u32) -> u64 { + ((source as u64) << 32) | u64::from(node) +} + +/// The destination column of one chunk: `dense` resolved through `ids`, the +/// dictionary of the walk's destination space, charged to `memory` first. +fn destination_column(dense: &[u32], ids: &[String], memory: &WorkMemory) -> Result { + // One pass checks every id against the dictionary and sums the bytes; + // the column is then built straight from the dictionary. + let mut bytes = 0usize; + for &d in dense { + let id = ids.get(d as usize).ok_or_else(|| { + OmniError::manifest_internal(format!( + "dense destination id {d} is outside its dictionary of {} ids", + ids.len() + )) + })?; + bytes = bytes.saturating_add(id.len()); + } + memory.string(bytes).map_err(|error| memory.error(error))?; + memory + .grow((dense.len() + 1).saturating_mul(std::mem::size_of::())) + .map_err(|error| memory.error(error))?; + Ok(Arc::new(StringArray::from_iter_values( + dense.iter().map(|&d| ids[d as usize].as_str()), + ))) +} + +/// The pending chunk as pairs ready to emit: its destinations resolved +/// through `ids`, the dictionary of the source that found them, the buffers +/// and the emission charge swapped for fresh ones. +fn take_pending_pairs( + emitted_src: &mut Vec, + emitted_dst: &mut Vec, + emission_memory: &mut WorkMemory, + ids: &[String], + memory: &WorkMemory, + chunk_rows: usize, +) -> Result { + let next_emission = memory + .child("expand emissions") + .map_err(|error| memory.error(error))?; + next_emission + .entries::<(u32, u32)>(chunk_rows) + .map_err(|error| memory.error(error))?; + memory.metric("expand_pairs", emitted_src.len()); + let destination = destination_column(emitted_dst, ids, emission_memory)?; + emitted_dst.clear(); + Ok(ExpandedPairs { + source_rows: std::mem::replace(emitted_src, Vec::with_capacity(chunk_rows)), + destination, + _memory: std::mem::replace(emission_memory, next_emission), + }) +} + pub(super) struct PreparedEdge { pub(super) dataset: Dataset, pub(super) probes: Vec, @@ -551,7 +611,7 @@ where .iter() .map(|edge| (edge.dataset.clone(), edge.probes.clone())) .collect(); - (Some(datasets), HopPolicy::Off) + (Some((datasets, None)), HopPolicy::Off) } ExpandExecution::Named(named) => { let start = decide_expand_start( @@ -572,13 +632,17 @@ where ExpandStart::Indexed { edge_ds, hop_policy, + pushdown, } => { Switch::IndexedScan.record(switch); ( - Some(vec![( - *edge_ds, - endpoint_probes(named.member.direction, env.catalog.system_columns), - )]), + Some(( + vec![( + *edge_ds, + endpoint_probes(named.member.direction, env.catalog.system_columns), + )], + Some(pushdown), + )), hop_policy, ) } @@ -604,6 +668,10 @@ pub(super) enum ExpandStart { Indexed { edge_ds: Box, hop_policy: HopPolicy, + /// Whether the key column's BTREE coverage, as the plan's pinned + /// index facts or the runtime probe judged it, makes a pushed + /// `key IN (keys)` list index lookups (`scan_edges_stream`). + pushdown: bool, }, } @@ -755,6 +823,13 @@ pub(super) async fn decide_expand_start( if let Some(coverage) = &coverage { warn_on_degraded_coverage(coverage, key_col, edge_type); } + let pushdown = match &coverage { + Some(Ok(crate::table_store::IndexCoverage::Indexed)) => true, + Some(_) => false, + None => corrected + .as_ref() + .is_none_or(|inputs| inputs.coverage == omnigraph_planner::IndexCoverage::Indexed), + }; let hop_policy = match corrected { Some(inputs) => HopPolicy::Full(inputs), None if matches!(named.origin, ModeOrigin::Pinned) => HopPolicy::Off, @@ -767,6 +842,7 @@ pub(super) async fn decide_expand_start( Ok(ExpandStart::Indexed { edge_ds: Box::new(edge_ds), hop_policy, + pushdown, }) } @@ -910,8 +986,15 @@ where let scan_memory = memory .child("bound edge scan") .map_err(|error| memory.error(error))?; - let mut stream = - scan_edges_stream(&dataset, orientation, &attach_columns, &keys, &scan_memory).await?; + let mut stream = scan_edges_stream( + &dataset, + orientation, + &attach_columns, + &keys, + None, + &scan_memory, + ) + .await?; while let Some(batch) = stream.next().await { let batch = batch.map_err(|error| memory.error(error))?; let input_memory = scan_memory @@ -1145,7 +1228,7 @@ pub(super) async fn execute_expand_bfs( graph_index: &GraphIndexHandle, catalog: &Catalog, step: &ExpandStep, - start_indexed: Option)>>, + start_indexed: Option, hop_policy: HopPolicy, side: &Gauge, memory: &WorkMemory, @@ -1189,8 +1272,9 @@ where let max = step.max_hops; let mut active = match start_indexed { - Some(datasets) => ActiveExpandSource::Indexed(Box::new(IndexedExpandSource { + Some((datasets, pushdown)) => ActiveExpandSource::Indexed(Box::new(IndexedExpandSource { datasets, + pushdown, interner: crate::graph_index::TypeIndex::new(), neighbor_map: HashMap::new(), })), @@ -1212,11 +1296,13 @@ where let n = src_ids.len(); memory - .entries::<(Vec, HashSet, HashSet, u32)>(n) + .entries::<(Vec, u64, u32)>(n) .map_err(|error| memory.error(error))?; let mut frontiers: Vec> = Vec::with_capacity(n); - let mut visited: Vec> = Vec::with_capacity(n); - let mut seen_dst: Vec> = Vec::with_capacity(n); + // One table per set over every source row (`source_node_key`): the nodes + // a source reached and the destinations it emitted. + let mut visited: HashSet = HashSet::default(); + let mut seen_dst: HashSet = HashSet::default(); for i in 0..n { let seed = match &mut active { ActiveExpandSource::Indexed(src) => { @@ -1226,15 +1312,12 @@ where src_idx.to_dense(src_ids.value(i)) } }; - let mut v = HashSet::new(); if same_type { if let Some(s) = seed { - v.insert(s); + visited.insert(source_node_key(i, s)); } } frontiers.push(seed.map(|s| vec![s]).unwrap_or_default()); - visited.push(v); - seen_dst.push(HashSet::new()); } memory.checkpoint().map_err(|error| memory.error(error))?; @@ -1245,9 +1328,21 @@ where let mut frontier_memory = memory .child("expand frontier") .map_err(|error| memory.error(error))?; + emission_memory + .entries::<(u32, u32)>(chunk_rows) + .map_err(|error| memory.error(error))?; let mut emitted_src: Vec = Vec::with_capacity(chunk_rows); - let mut emitted_dst: Vec = Vec::with_capacity(chunk_rows); + let mut emitted_dst: Vec = Vec::with_capacity(chunk_rows); let mut prev_union_len: usize = 0; + // What one examined neighbour may add: a visited entry, a frontier + // entry, an emitted-set entry and a pair. A node admits that bound for + // all its neighbours before its slice is walked and settles to what the + // walk added once it is done: two pool calls per node, none per edge, + // and the held charge is the live state. + let per_neighbour_bytes = WorkMemory::entry_bytes::() + .saturating_mul(2) + .saturating_add(WorkMemory::entry_bytes::()) + .saturating_add(WorkMemory::entry_bytes::<(u32, u32)>()); for hop in 1..=max { memory.check().map_err(|error| memory.error(error))?; @@ -1298,6 +1393,23 @@ where let gi = graph_index.get().await?.ok_or_else(|| { OmniError::manifest("graph index required for CSR traversal".to_string()) })?; + // The pending chunk's destinations are dense ids of the + // interner: they leave now, as strings, before the + // dictionaries change. A `Utf8` column has no id space. + if !emitted_src.is_empty() { + let ActiveExpandSource::Indexed(src) = &active else { + unreachable!("switch only fires while the Indexed source is active"); + }; + let pairs = take_pending_pairs( + &mut emitted_src, + &mut emitted_dst, + &mut emission_memory, + src.interner.ids(), + memory, + chunk_rows, + )?; + emit(pairs).await?; + } let csr_source = ActiveExpandSource::Csr(resolve_csr( gi, edge_def, @@ -1316,26 +1428,33 @@ where unreachable!("active source was just replaced with Csr"); }; hop_memory - .entries::( + .entries::( frontiers.iter().map(Vec::len).sum::() - + visited.iter().map(HashSet::len).sum::() - + seen_dst.iter().map(HashSet::len).sum::(), + + visited.len() + + seen_dst.len(), ) .map_err(|error| memory.error(error))?; - let translate_set = - |set: &HashSet, idx: &crate::graph_index::TypeIndex| -> HashSet { - set.iter() - .filter_map(|&d| interner.to_id(d).and_then(|id| idx.to_dense(id))) - .collect() - }; - for i in 0..n { - frontiers[i] = frontiers[i] + let translate_set = |set: &HashSet, + idx: &crate::graph_index::TypeIndex| + -> HashSet { + set.iter() + .filter_map(|&key| { + let (source, dense) = ((key >> 32) as usize, key as u32); + interner + .to_id(dense) + .and_then(|id| idx.to_dense(id)) + .map(|dense| source_node_key(source, dense)) + }) + .collect() + }; + for frontier in frontiers.iter_mut() { + *frontier = frontier .iter() .filter_map(|&d| interner.to_id(d).and_then(|id| src_idx.to_dense(id))) .collect(); - visited[i] = translate_set(&visited[i], src_idx); - seen_dst[i] = translate_set(&seen_dst[i], dst_idx); } + visited = translate_set(&visited, src_idx); + seen_dst = translate_set(&seen_dst, dst_idx); tracing::debug!( target: "omnigraph::traverse", edge = %member.edge_type, @@ -1373,6 +1492,7 @@ where dataset, probes, &union_keys, + src.pushdown, &mut src.interner, &mut src.neighbor_map, &hop_memory, @@ -1382,8 +1502,8 @@ where } } - for i in 0..n { - let cur = std::mem::take(&mut frontiers[i]); + for (i, frontier) in frontiers.iter_mut().enumerate() { + let cur = std::mem::take(frontier); let mut next: Vec = Vec::new(); for &node in &cur { let (fwd, rev): (&[u32], &[u32]) = match &active { @@ -1399,83 +1519,75 @@ where adj_rev.map(|a| a.neighbors(node)).unwrap_or(&[]), ), }; + let ids: &[String] = match &active { + ActiveExpandSource::Indexed(src) => src.interner.ids(), + ActiveExpandSource::Csr(CsrSource { dst_idx, .. }) => dst_idx.ids(), + }; + // Admit what this node's neighbours can add before the walk. + let before = memory.charged(); + memory + .grow((fwd.len() + rev.len()).saturating_mul(per_neighbour_bytes)) + .map_err(|error| memory.error(error))?; + let mut added = 0usize; for &neighbor in fwd.iter().chain(rev) { if budgeted { memory .charge_traversal(1) .map_err(|error| memory.error(error))?; - } else { - memory.check().map_err(|error| memory.error(error))?; } let is_self = same_type && hop == 1 && neighbor == node; if !is_self && same_type { - if visited[i].contains(&neighbor) { + if !visited.insert(source_node_key(i, neighbor)) { continue; } - memory - .entries::(1) - .map_err(|error| memory.error(error))?; - visited[i].insert(neighbor); + added = added.saturating_add(WorkMemory::entry_bytes::()); } if !is_self { - next_memory - .entries::(1) - .map_err(|error| memory.error(error))?; next.push(neighbor); + added = added.saturating_add(WorkMemory::entry_bytes::()); } - if hop >= min_hops && !seen_dst[i].contains(&neighbor) { - memory - .entries::(1) - .map_err(|error| memory.error(error))?; - seen_dst[i].insert(neighbor); - let dst_id = match &active { - ActiveExpandSource::Indexed(src) => Some( - src.interner - .to_id(neighbor) - .expect("interned dst id must resolve"), - ), - ActiveExpandSource::Csr(CsrSource { dst_idx, .. }) => { - dst_idx.to_id(neighbor) - } - }; - if let Some(dst_id) = dst_id { - emission_memory - .entries::<(u32, String)>(1) - .map_err(|error| memory.error(error))?; - emission_memory - .string(dst_id.len()) - .map_err(|error| memory.error(error))?; - emitted_src.push(i as u32); - emitted_dst.push(dst_id.to_string()); - if emitted_src.len() >= chunk_rows { - let next = memory - .child("expand emissions") - .map_err(|error| memory.error(error))?; - memory.metric("expand_pairs", emitted_src.len()); - emit(ExpandedPairs { - source_rows: std::mem::replace( - &mut emitted_src, - Vec::with_capacity(chunk_rows), - ), - destination_ids: std::mem::replace( - &mut emitted_dst, - Vec::with_capacity(chunk_rows), - ), - _memory: std::mem::replace(&mut emission_memory, next), - }) - .await?; - } + if hop >= min_hops + && (neighbor as usize) < ids.len() + && seen_dst.insert(source_node_key(i, neighbor)) + { + added = added.saturating_add( + WorkMemory::entry_bytes::() + .saturating_add(WorkMemory::entry_bytes::<(u32, u32)>()), + ); + emitted_src.push(i as u32); + emitted_dst.push(neighbor); + if emitted_src.len() >= chunk_rows { + let pairs = take_pending_pairs( + &mut emitted_src, + &mut emitted_dst, + &mut emission_memory, + ids, + memory, + chunk_rows, + )?; + emit(pairs).await?; } } } + memory.shrink_to(before.saturating_add(added)); + memory.check().map_err(|error| memory.error(error))?; } - frontiers[i] = next; + *frontier = next; } drop(std::mem::replace(&mut frontier_memory, next_memory)); if let ActiveExpandSource::Indexed(src) = &mut active { src.neighbor_map = HashMap::new(); } } + let destination = if emitted_src.is_empty() { + None + } else { + let ids: &[String] = match &active { + ActiveExpandSource::Indexed(src) => src.interner.ids(), + ActiveExpandSource::Csr(CsrSource { dst_idx, .. }) => dst_idx.ids(), + }; + Some(destination_column(&emitted_dst, ids, &emission_memory)?) + }; drop(active); drop(frontiers); drop(visited); @@ -1483,13 +1595,13 @@ where drop(frontier_memory); memory.release_work(); - if emitted_src.is_empty() { + let Some(destination) = destination else { return Ok(()); - } + }; memory.metric("expand_pairs", emitted_src.len()); emit(ExpandedPairs { source_rows: emitted_src, - destination_ids: emitted_dst, + destination, _memory: emission_memory, }) .await @@ -1539,9 +1651,9 @@ pub(super) fn bulk_anti_join_mask( return Ok(None); }; memory - .entries::(outer_ids.len()) + .entries::(outer_ids.len()) .map_err(|error| memory.error(error))?; - let mut keep_mask = Vec::with_capacity(outer_ids.len()); + let mut counts = Vec::with_capacity(outer_ids.len()); let mut targets = Vec::new(); for i in 0..outer_ids.len() { memory.check().map_err(|error| memory.error(error))?; @@ -1559,9 +1671,12 @@ pub(super) fn bulk_anti_join_mask( } None => 0, }; - keep_mask.push(row_count.holds(matches)?); + counts.push(matches); } - Ok(Some(BooleanArray::from(keep_mask))) + memory + .entries::<(i64, bool)>(counts.len()) + .map_err(|error| memory.error(error))?; + Ok(Some(row_count.holds_all(&counts)?)) } /// Where the shared BFS core reads each hop's neighbors from. The two sources @@ -1579,8 +1694,15 @@ pub(super) enum ActiveExpandSource<'g> { Csr(CsrSource<'g>), } +/// An indexed walk's edge datasets with their probes, and the pushdown +/// verdict the start decision reached (`None`: each scan probes). +pub(super) type IndexedStart = (Vec<(Dataset, Vec)>, Option); + pub(super) struct IndexedExpandSource { pub(super) datasets: Vec<(Dataset, Vec)>, + /// The known coverage verdict for the pushed key list, when the start + /// decision judged it; `None` lets each scan probe. + pub(super) pushdown: Option, pub(super) interner: crate::graph_index::TypeIndex, /// This hop's dense key -> dense neighbors (scan order; duplicates /// preserved, like CSR multi-edges). Rebuilt per hop. @@ -1598,10 +1720,12 @@ pub(super) enum HopPolicy { /// One `key IN (keys)` scan per probe, every row's endpoints interned into /// `interner` and the opposite appended to `neighbor_map[key]` in scan order /// (probe 0's rows, then probe 1's; parallel edges kept, like the CSR). +#[allow(clippy::too_many_arguments)] pub(super) async fn scan_neighbor_map( edge_ds: &Dataset, probes: &[EndpointColumns], keys: &[String], + pushdown: Option, interner: &mut crate::graph_index::TypeIndex, neighbor_map: &mut HashMap>, hop_memory: &WorkMemory, @@ -1612,7 +1736,7 @@ pub(super) async fn scan_neighbor_map( key: key_col, opposite: opp_col, } = orientation; - let batches = scan_edges(edge_ds, orientation, &[], keys, hop_memory).await?; + let batches = scan_edges(edge_ds, orientation, &[], keys, pushdown, hop_memory).await?; for batch in &batches { let keys = utf8_column(batch, key_col)?; let opposites = utf8_column(batch, opp_col)?; @@ -1660,12 +1784,13 @@ async fn scan_edges( orientation: EndpointColumns, extras: &[&str], keys: &[String], + pushdown: Option, memory: &WorkMemory, ) -> Result> { if keys.is_empty() { return Ok(Vec::new()); } - let stream = scan_edges_stream(ds, orientation, extras, keys, memory).await?; + let stream = scan_edges_stream(ds, orientation, extras, keys, pushdown, memory).await?; memory .collect(stream) .await @@ -1692,11 +1817,19 @@ fn physical_scan_rows( }) } +/// The longest key list an unindexed edge scan still pushes into Lance; see +/// the pushdown rule in `scan_edges_stream`. The indexed expand path's +/// frontier ceiling (`OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER`, 1,024 by +/// default) is this size: a frontier the plan sends to per-hop scans stays +/// pushed, a bound edge over a larger source set is sieved. +const PUSHDOWN_KEYS_MAX: usize = 1024; + async fn scan_edges_stream( ds: &Dataset, orientation: EndpointColumns, extras: &[&str], keys: &[String], + pushdown: Option, memory: &WorkMemory, ) -> Result { if memory.traversal_limited() { @@ -1726,14 +1859,34 @@ async fn scan_edges_stream( .copied() .filter(|column| *column != key_col && *column != opposite_col), ); - let filter = id_in_list_expr(keys, key_col); + // A `key IN (keys)` filter reaches Lance where a BTREE on the key column + // turns it into index lookups, and where the list is small: Lance 13 + // clones a pushed list for every fragment it reads (`filtered_read.rs` + // clones `full_filter` per fragment; the physical expression is built + // once per distinct expression), but a pushed filter also lets it read + // a fragment's other columns only for the rows the key column admits, + // one read fewer per fragment no key is in. Up to `PUSHDOWN_KEYS_MAX` + // keys the per-fragment clone is cheaper than that read; past it the + // engine sieves the batches itself through one set of the keys. + let pushdown = match pushdown { + Some(true) => true, + _ if keys.len() <= PUSHDOWN_KEYS_MAX => true, + Some(false) => false, + None => matches!( + omnigraph_core::dataset_index::key_column_index_coverage(ds, key_col).await, + Ok(omnigraph_core::dataset_index::IndexCoverage::Indexed) + ), + }; + let filter = pushdown.then(|| id_in_list_expr(keys, key_col)); let plan = crate::table_store::TableStore::scan_plan_with( ds, Some(&projection), None, false, |scanner| { - scanner.filter_expr(filter); + if let Some(filter) = filter { + scanner.filter_expr(filter); + } scanner.batch_size(memory.batch_rows()); scanner.batch_size_bytes(memory.batch_bytes() as u64); Ok(()) @@ -1741,7 +1894,31 @@ async fn scan_edges_stream( ) .await?; let (_, stream) = memory.stream(plan).map_err(|error| memory.error(error))?; - Ok(stream) + if pushdown { + return Ok(stream); + } + let wanted: hashbrown::HashSet = keys.iter().cloned().collect(); + let schema = stream.schema(); + let key_col = key_col.to_string(); + let sieved = stream.map(move |batch| { + let batch = batch?; + let keys = batch + .column_by_name(&key_col) + .and_then(|column| column.as_any().downcast_ref::()) + .ok_or_else(|| { + datafusion::common::DataFusionError::Internal(format!( + "edge batch has no Utf8 '{key_col}'" + )) + })?; + let mask = arrow_array::BooleanArray::from_iter( + keys.iter() + .map(|key| Some(key.is_some_and(|key| wanted.contains(key)))), + ); + Ok(arrow_select::filter::filter_record_batch(&batch, &mask)?) + }); + Ok(Box::pin( + datafusion::physical_plan::stream::RecordBatchStreamAdapter::new(schema, sieved), + )) } #[cfg(test)] diff --git a/crates/omnigraph/src/engine/operators/expand_stream.rs b/crates/omnigraph/src/engine/operators/expand_stream.rs index 8a2c33347..2545b37cb 100644 --- a/crates/omnigraph/src/engine/operators/expand_stream.rs +++ b/crates/omnigraph/src/engine/operators/expand_stream.rs @@ -4,7 +4,7 @@ use std::fmt; use std::sync::Arc; -use arrow_array::{ArrayRef, RecordBatch, UInt32Array, builder::StringBuilder}; +use arrow_array::{ArrayRef, RecordBatch, UInt32Array}; use arrow_schema::SchemaRef; use datafusion::common::{DataFusionError, Result}; use datafusion::execution::TaskContext; @@ -480,22 +480,8 @@ async fn emit_pairs( let end = (offset + rows).min(pairs.source_rows.len()); work.entries::(end - offset)?; let source = UInt32Array::from(pairs.source_rows[offset..end].to_vec()); - let ids = &pairs.destination_ids[offset..end]; - let bytes = ids.iter().map(String::len).sum(); - work.string(bytes)?; - let mut destination = StringBuilder::with_capacity(ids.len(), bytes); - for id in ids { - work.check()?; - destination.append_value(id); - } - let output = align_sources( - wide, - &source, - Arc::new(destination.finish()), - &[], - schema, - &work, - )?; + let destination = pairs.destination.slice(offset, end - offset); + let output = align_sources(wide, &source, destination, &[], schema, &work)?; sender.send_bounded(output, work).await?; } Ok(()) diff --git a/crates/omnigraph/src/engine/operators/hash_join.rs b/crates/omnigraph/src/engine/operators/hash_join.rs index f0f610128..a2abcaadc 100644 --- a/crates/omnigraph/src/engine/operators/hash_join.rs +++ b/crates/omnigraph/src/engine/operators/hash_join.rs @@ -8,12 +8,12 @@ //! the node declares that fallback. An empty build stops the probe after its //! first batch. -use std::collections::HashMap; use std::fmt; use std::sync::Arc; -use arrow_array::{Array, RecordBatch, StringArray, UInt32Array}; +use arrow_array::{Array, ArrayRef, RecordBatch, StringArray, UInt32Array}; use arrow_schema::SchemaRef; +use datafusion::common::hash_utils::{RandomState, create_hashes}; use datafusion::common::{DataFusionError, Result as DfResult}; use datafusion::execution::TaskContext; use datafusion::physical_plan::metrics::{ @@ -23,6 +23,7 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream, }; use futures::future::BoxFuture; +use hashbrown::HashTable; use omnigraph_compiler::catalog::Catalog; use omnigraph_compiler::ir::{IRExpr, ParamMap}; use omnigraph_planner::AccessPath; @@ -124,14 +125,63 @@ impl DisplayAs for HashJoinExec { } } -/// The drained build side: its rows and the row of every id, held under -/// `memory` for as long as the probe runs. +/// The seed of the join's hash function: fixed, so a build hashes the same +/// way on every run. +const HASH_SEED: u64 = 0x6f6d_6e69_6a6f_696e; + +/// The drained build side: its rows and, per distinct non-null id, the first +/// row holding it, as `(hash, row)` entries in a table that borrows nothing +/// (an entry compares through the id column of `batch`), held under +/// `memory` for as long as the probe runs. The ids are hashed in bulk with +/// DataFusion's `create_hashes`; no id is copied out of the Arrow column. struct Built { batch: RecordBatch, - row_by_id: HashMap, + ids: StringArray, + rows: HashTable<(u64, u32)>, + state: RandomState, _memory: WorkMemory, } +impl Built { + /// The first build row holding `id`, hashed as `hash`. + fn row_of(&self, hash: u64, id: &str) -> Option { + find_row(&self.rows, &self.ids, hash, id) + } +} + +/// The build table over `ids` hashed as `hashes`: one `(hash, row)` entry +/// per distinct non-null id, the first row of a duplicate kept; two ids +/// with one hash keep separate entries, told apart by their strings. +fn index_rows(ids: &StringArray, hashes: &[u64]) -> HashTable<(u64, u32)> { + let mut rows: HashTable<(u64, u32)> = HashTable::with_capacity(ids.len()); + for (row, &hash) in hashes.iter().enumerate() { + if ids.is_null(row) { + continue; + } + if find_row(&rows, ids, hash, ids.value(row)).is_none() { + rows.insert_unique(hash, (hash, row as u32), |&(entry_hash, _)| entry_hash); + } + } + rows +} + +/// The build row of `id`, hashed as `hash`, in `rows` over `ids`. +fn find_row(rows: &HashTable<(u64, u32)>, ids: &StringArray, hash: u64, id: &str) -> Option { + rows.find(hash, |&(entry_hash, row)| { + entry_hash == hash && ids.value(row as usize) == id + }) + .map(|&(_, row)| row) +} + +/// `hashes` filled with the hash of every value of `ids` (nulls included, +/// which the callers skip) under `state`. +fn hash_ids(ids: &ArrayRef, state: &RandomState, hashes: &mut Vec) -> DfResult<()> { + hashes.clear(); + hashes.resize(ids.len(), 0); + create_hashes([ids], state, hashes)?; + Ok(()) +} + async fn build_side( build: &Arc, id_column: &str, @@ -148,22 +198,30 @@ async fn build_side( "destination scan must return Utf8 '{id_column}'" ))) })?; - work.entries::<(String, u32)>(ids.len())?; - work.string(ids.iter().flatten().map(str::len).sum())?; - let mut row_by_id = HashMap::with_capacity(ids.len()); - for (row, id) in ids.iter().enumerate() { - if let Some(id) = id { - let row = u32::try_from(row).map_err(|_| { - external(OmniError::manifest_internal( - "destination scan exceeds row index range", - )) - })?; - row_by_id.entry(id.to_string()).or_insert(row); - } - } + u32::try_from(ids.len()).map_err(|_| { + external(OmniError::manifest_internal( + "destination scan exceeds row index range", + )) + })?; + // The table's entries plus the hash buffer, one `u64` per row; the + // buffer's share is released once the table is built. + let before = work.charged(); + let table_bytes = WorkMemory::entry_bytes::<(u64, u32)>().saturating_mul(ids.len()); + work.entries::<(u64, u32)>(ids.len())?; + work.entries::(ids.len())?; + let state = RandomState::with_seed(HASH_SEED); + let id_array: ArrayRef = Arc::new(ids.clone()); + let mut hashes = Vec::new(); + hash_ids(&id_array, &state, &mut hashes)?; + let rows = index_rows(ids, &hashes); + drop(hashes); + work.shrink_to(before.saturating_add(table_bytes)); + let ids = ids.clone(); Ok(Built { batch, - row_by_id, + ids, + rows, + state, _memory: work, }) } @@ -315,16 +373,23 @@ impl Probe { })?; let work = Arc::new(memory.child(OUTPUT)?); work.entries::(ids.len().saturating_mul(2))?; + work.entries::(ids.len())?; + u32::try_from(ids.len()).map_err(|_| { + external(OmniError::manifest_internal( + "traversal batch exceeds row index range", + )) + })?; + let mut hashes = Vec::new(); + hash_ids(batch.column(id_index), &built.state, &mut hashes)?; let mut probe_rows = Vec::with_capacity(ids.len()); let mut build_rows = Vec::with_capacity(ids.len()); - for (row, id) in ids.iter().enumerate() { - if let Some(build_row) = id.and_then(|id| built.row_by_id.get(id)) { - probe_rows.push(u32::try_from(row).map_err(|_| { - external(OmniError::manifest_internal( - "traversal batch exceeds row index range", - )) - })?); - build_rows.push(*build_row); + for (row, &hash) in hashes.iter().enumerate() { + if ids.is_null(row) { + continue; + } + if let Some(build_row) = built.row_of(hash, ids.value(row)) { + probe_rows.push(row as u32); + build_rows.push(build_row); } } if probe_rows.is_empty() { @@ -375,3 +440,24 @@ impl Streamed for Probe { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// The build table keeps the first row of a duplicate id, tells two ids + /// with one hash apart by their strings, and never indexes a null. + #[test] + fn build_table_keeps_the_first_row_and_resolves_a_collision() { + let ids = StringArray::from(vec![Some("a"), Some("b"), Some("a"), None, Some("c")]); + // `a` and `b` collide; the null shares `c`'s hash and must stay out. + let hashes = [7, 7, 7, 9, 9]; + let rows = index_rows(&ids, &hashes); + assert_eq!(rows.len(), 3); + assert_eq!(find_row(&rows, &ids, 7, "a"), Some(0)); + assert_eq!(find_row(&rows, &ids, 7, "b"), Some(1)); + assert_eq!(find_row(&rows, &ids, 9, "c"), Some(4)); + assert_eq!(find_row(&rows, &ids, 7, "c"), None); + assert_eq!(find_row(&rows, &ids, 9, "a"), None); + } +} diff --git a/crates/omnigraph/src/engine/operators/memory.rs b/crates/omnigraph/src/engine/operators/memory.rs index f3a0fcc64..e315dc31c 100644 --- a/crates/omnigraph/src/engine/operators/memory.rs +++ b/crates/omnigraph/src/engine/operators/memory.rs @@ -732,6 +732,11 @@ impl WorkMemory { /// Lower this work's charge to `bytes` once its peak has passed; a charge /// already at or under `bytes`, or freed meanwhile, stays as it is. + /// This work's current scratch charge, the number `shrink_to` lowers. + pub(in crate::engine) fn charged(&self) -> usize { + self.scratch.size() + } + pub(in crate::engine) fn shrink_to(&self, bytes: usize) { let excess = self.scratch.size().saturating_sub(bytes); if self.scratch.try_shrink(excess).is_ok() { @@ -867,6 +872,19 @@ impl WorkMemory { }) } + /// `slice_bytes(batch, row, 1)` for every row of `batch` in one pass per + /// column: the same numbers, read off each column's layout (a constant per + /// row for fixed-width columns, the offset deltas for variable-width ones, + /// children summed for structs) instead of one `ArrayData` slice per row + /// per column. + pub(in crate::engine) fn row_bytes(&self, batch: &RecordBatch) -> DfResult> { + let mut rows = vec![0usize; batch.num_rows()]; + for column in batch.columns() { + add_row_bytes(&column.to_data(), &mut rows)?; + } + Ok(rows) + } + pub(in crate::engine) fn take( &self, batch: &RecordBatch, @@ -1169,6 +1187,66 @@ fn take_scratch_size(data: &ArrayData, rows: usize) -> usize { /// Bytes copied from an Arrow range, including nested child ranges. Arrow's /// ArrayData slice keeps list child buffers whole, so recursing through its /// get_slice_memory_size would count the entire vector column per picked row. +/// Add each row's `slice_memory_size(data, row, 1)` to `rows`, one pass over +/// the column; a layout with no shortcut falls back to the per-row slice. +fn add_row_bytes(data: &ArrayData, rows: &mut [usize]) -> DfResult<()> { + if rows.is_empty() { + return Ok(()); + } + let validity = usize::from(data.nulls().is_some()); + match data.data_type() { + DataType::Utf8 | DataType::Binary => { + let offsets = &data.buffers()[0].typed_data::()[data.offset()..]; + for (row, bytes) in rows.iter_mut().enumerate() { + let value = (offsets[row + 1] - offsets[row]) as usize; + *bytes = bytes.saturating_add(4 + value + validity); + } + } + DataType::LargeUtf8 | DataType::LargeBinary => { + let offsets = &data.buffers()[0].typed_data::()[data.offset()..]; + for (row, bytes) in rows.iter_mut().enumerate() { + let value = (offsets[row + 1] - offsets[row]) as usize; + *bytes = bytes.saturating_add(8 + value + validity); + } + } + DataType::Struct(_) => { + for child in data.child_data() { + add_row_bytes(&child.slice(data.offset(), rows.len()), rows)?; + } + for bytes in rows.iter_mut() { + *bytes = bytes.saturating_add(validity); + } + } + DataType::List(_) + | DataType::LargeList(_) + | DataType::Map(_, _) + | DataType::FixedSizeList(_, _) => { + for (row, bytes) in rows.iter_mut().enumerate() { + *bytes = bytes.saturating_add(slice_memory_size(data, row, 1)?); + } + } + _ => { + if data.data_type().is_primitive() + || matches!( + data.data_type(), + DataType::Boolean | DataType::FixedSizeBinary(_) | DataType::Null + ) + { + // Fixed-width layouts cost the same for every row. + let each = slice_memory_size(data, 0, 1)?; + for bytes in rows.iter_mut() { + *bytes = bytes.saturating_add(each); + } + } else { + for (row, bytes) in rows.iter_mut().enumerate() { + *bytes = bytes.saturating_add(slice_memory_size(data, row, 1)?); + } + } + } + } + Ok(()) +} + fn slice_memory_size(data: &ArrayData, offset: usize, len: usize) -> DfResult { let data = data.slice(offset, len); let validity = if data.nulls().is_some() { @@ -1736,3 +1814,74 @@ mod tests { assert_eq!(pool.reserved(), 0); } } + +#[cfg(test)] +mod row_bytes_tests { + use super::*; + use arrow_array::builder::{FixedSizeListBuilder, Float32Builder, ListBuilder, StringBuilder}; + use arrow_array::{ArrayRef, BooleanArray, Int64Array, StringArray, StructArray}; + use arrow_schema::{Field, Fields}; + use datafusion::execution::context::SessionConfig; + use datafusion::execution::memory_pool::GreedyMemoryPool; + + /// One pass per column gives every row the bytes a one-row slice gives. + #[test] + fn row_bytes_equals_the_per_row_slice_for_every_layout() { + let limit = 1 << 20; + let pool: Arc = Arc::new(GreedyMemoryPool::new(limit)); + let resources = Arc::new(QueryResources::new(pool, limit as u64)); + let ctx = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_extension(resources)), + ); + let memory = WorkMemory::new(ctx, "row bytes").unwrap(); + let ints: ArrayRef = Arc::new(Int64Array::from(vec![Some(1), None, Some(3), Some(4)])); + let flags: ArrayRef = Arc::new(BooleanArray::from(vec![true, false, true, true])); + let names: ArrayRef = Arc::new(StringArray::from(vec![ + Some("a"), + Some("longer value"), + None, + Some(""), + ])); + let mut vectors = FixedSizeListBuilder::new(Float32Builder::new(), 3); + for row in 0..4 { + vectors.values().append_slice(&[row as f32; 3]); + vectors.append(row != 1); + } + let vectors: ArrayRef = Arc::new(vectors.finish()); + let mut lists = ListBuilder::new(StringBuilder::new()); + lists.values().append_value("x"); + lists.values().append_value("yy"); + lists.append(true); + lists.append(false); + lists.append(true); + lists.values().append_value("zzz"); + lists.append(true); + let lists: ArrayRef = Arc::new(lists.finish()); + let nested: ArrayRef = Arc::new(StructArray::new( + Fields::from(vec![ + Field::new("n", DataType::Int64, true), + Field::new("s", DataType::Utf8, true), + ]), + vec![Arc::clone(&ints), Arc::clone(&names)], + None, + )); + let batch = RecordBatch::try_from_iter([ + ("ints", ints), + ("flags", flags), + ("names", names), + ("vectors", vectors), + ("lists", lists), + ("nested", nested), + ]) + .unwrap(); + let sliced = batch.slice(1, 3); + for batch in [batch, sliced] { + let expected: Vec = (0..batch.num_rows()) + .map(|row| memory.slice_bytes(&batch, row, 1).unwrap()) + .collect(); + assert_eq!(memory.row_bytes(&batch).unwrap(), expected); + assert!(expected.iter().all(|bytes| *bytes > 0)); + } + } +} diff --git a/crates/omnigraph/src/engine/operators/pair_buffer.rs b/crates/omnigraph/src/engine/operators/pair_buffer.rs index 97c7f8130..3f63db887 100644 --- a/crates/omnigraph/src/engine/operators/pair_buffer.rs +++ b/crates/omnigraph/src/engine/operators/pair_buffer.rs @@ -53,10 +53,7 @@ pub(super) struct PairBuffer { } fn row_bytes(memory: &WorkMemory, batch: &RecordBatch, rows: &mut Vec) -> DfResult<()> { - rows.clear(); - for row in 0..batch.num_rows() { - rows.push(memory.slice_bytes(batch, row, 1)?); - } + *rows = memory.row_bytes(batch)?; Ok(()) } diff --git a/crates/omnigraph/src/engine/operators/single_hop.rs b/crates/omnigraph/src/engine/operators/single_hop.rs index 7258cfcca..1017d41cc 100644 --- a/crates/omnigraph/src/engine/operators/single_hop.rs +++ b/crates/omnigraph/src/engine/operators/single_hop.rs @@ -54,6 +54,7 @@ enum Mode { Indexed { edge_ds: Box, hop_policy: HopPolicy, + pushdown: bool, }, } @@ -118,11 +119,13 @@ impl SingleHop { ExpandStart::Indexed { edge_ds, hop_policy, + pushdown, } => { Switch::IndexedScan.record(&self.switch); Mode::Indexed { edge_ds, hop_policy, + pushdown, } } }; @@ -202,9 +205,7 @@ impl SingleHop { })?; let hop_memory = memory.child("expand hop")?; hop_memory.entries::(batch.num_rows())?; - let row_bytes = (0..batch.num_rows()) - .map(|row| memory.slice_bytes(&batch, row, 1)) - .collect::>>()?; + let row_bytes = memory.row_bytes(&batch)?; let source = match &started.mode { Mode::Csr => { let gi = env @@ -222,7 +223,12 @@ impl SingleHop { .map_err(external)?, } } - Mode::Indexed { edge_ds, .. } => Source::Indexed { edge_ds }, + Mode::Indexed { + edge_ds, pushdown, .. + } => Source::Indexed { + edge_ds, + pushdown: *pushdown, + }, }; let (src_dense, ids, neighbours) = source .prepare( @@ -274,8 +280,13 @@ impl Streamed for SingleHop { /// One batch's neighbour source: CSR slices borrowed from the index, or the /// edge dataset a batch-local scan reads. enum Source<'g> { - Csr { csr: CsrSource<'g> }, - Indexed { edge_ds: &'g lance::Dataset }, + Csr { + csr: CsrSource<'g>, + }, + Indexed { + edge_ds: &'g lance::Dataset, + pushdown: bool, + }, } /// One input batch's neighbour lists: CSR slices, or the batch's own scan. @@ -327,7 +338,7 @@ impl<'g> Source<'g> { }, )) } - Self::Indexed { edge_ds } => { + Self::Indexed { edge_ds, pushdown } => { let mut interner = TypeIndex::new(); let src_dense = (0..src_ids.len()) .map(|row| intern(&mut interner, src_ids.value(row), hop_memory).map(Some)) @@ -341,6 +352,7 @@ impl<'g> Source<'g> { edge_ds, probes, &keys, + Some(*pushdown), &mut interner, &mut map, hop_memory, diff --git a/crates/omnigraph/src/engine/operators/subquery_aggregate.rs b/crates/omnigraph/src/engine/operators/subquery_aggregate.rs index 731279823..2525097a1 100644 --- a/crates/omnigraph/src/engine/operators/subquery_aggregate.rs +++ b/crates/omnigraph/src/engine/operators/subquery_aggregate.rs @@ -6,7 +6,8 @@ use std::cmp::Ordering; use std::sync::Arc; use arrow_array::{ - Array, ArrayRef, BooleanArray, Decimal128Array, Float64Array, RecordBatch, UInt32Array, + Array, ArrayRef, BooleanArray, Decimal128Array, Float64Array, Int64Array, RecordBatch, + UInt32Array, }; use arrow_schema::DataType; use datafusion::common::ScalarValue; @@ -158,14 +159,38 @@ impl BoundComparison { }) } - fn holds(&self, value: ScalarValue) -> Result { - let value = value.to_array_of_size(1).map_err(OmniError::datafusion)?; - let value = cast_left(&self.left, &value)?; - let mask = compare_arrays(&value, self.op, &self.right)?; - Ok(mask.is_valid(0) && mask.value(0)) + /// Whether the predicate holds for each of `values`, the block + /// aggregate's results in the leaf's declared type: one cast chain and + /// one comparison over the whole column, a null verdict reading `false`. + fn holds_all(&self, values: &ArrayRef) -> Result { + let value = cast_left(&self.left, values)?; + let right = arrow_select::take::take( + self.right.as_ref(), + &UInt32Array::from(vec![0u32; value.len()]), + None, + ) + .map_err(OmniError::arrow_internal)?; + let mask = compare_arrays(&value, self.op, &right)?; + Ok(if mask.null_count() == 0 { + mask + } else { + arrow_select::filter::prep_null_mask_filter(&mask) + }) } } +/// `counts` as the `I64` column a block's row count is compared in. +fn count_values(counts: &[u64]) -> Result { + let counts = counts + .iter() + .map(|&count| { + i64::try_from(count) + .map_err(|_| OmniError::manifest("block count exceeds the I64 range")) + }) + .collect::>>()?; + Ok(Arc::new(Int64Array::from(counts))) +} + fn cast_left(expr: &BlockAggregateExpr, value: &ArrayRef) -> Result { let mut current = expr; let mut casts = Vec::new(); @@ -220,8 +245,14 @@ impl RowCountPredicate { self.existence_only } + /// The verdict of every outer row from its neighbour count. + pub(crate) fn holds_all(&self, counts: &[u64]) -> Result { + self.comparison.holds_all(&count_values(counts)?) + } + + #[cfg(test)] pub(crate) fn holds(&self, count: u64) -> Result { - self.comparison.holds(count_value(count)?) + Ok(self.holds_all(&[count])?.value(0)) } } @@ -395,11 +426,28 @@ impl SubqueryAggregate { Ok(value) } + /// Every outer row's verdict: the row counts as one `I64` column when + /// the block counts rows, else each row's finalized aggregate gathered + /// into one column, then one comparison. pub(crate) fn keep_mask(&self) -> Result { - (0..self.counts.len()) - .map(|row| self.comparison.holds(self.aggregate(row)?)) - .collect::>>() - .map(BooleanArray::from) + let leaf = self.comparison.left.leaf(); + let values = match (leaf, &self.values) { + (BlockAggregateExpr::CountRows { .. }, None) + | ( + BlockAggregateExpr::Aggregate { + func: AggFunc::Count, + .. + }, + None, + ) => count_values(&self.counts)?, + _ => { + let values = (0..self.counts.len()) + .map(|row| self.aggregate(row)) + .collect::>>()?; + ScalarValue::iter_to_array(values).map_err(OmniError::datafusion)? + } + }; + self.comparison.holds_all(&values) } } diff --git a/crates/omnigraph/tests/integration/traversal_adaptive.rs b/crates/omnigraph/tests/integration/traversal_adaptive.rs index dc3f70ce3..bbfcf5766 100644 --- a/crates/omnigraph/tests/integration/traversal_adaptive.rs +++ b/crates/omnigraph/tests/integration/traversal_adaptive.rs @@ -131,6 +131,44 @@ async fn adaptive_switch_returns_identical_rows() { // The switch under an UNDIRECTED traversal (Direction::Both — two endpoint // probes per hop, the probe-factor case, CSR+CSC merge after the swap). Same // clique, so the hop-2 policy check sees the same explosive growth. +/// The same switch from a seed whose dense id differs between the walk's +/// interner and the graph index's dictionary: a hop-1 chunk still buffered +/// when hop 2 swaps the source must leave as the ids it was found under. +#[tokio::test] +async fn adaptive_switch_from_a_later_seed_returns_identical_rows() { + let dir = tempfile::tempdir().unwrap(); + let db = clique_db(&dir).await; + let p = params(&[("$name", "p5")]); + + let csr_db = with_traversal(&db, Traversal::Csr); + let indexed_db = with_traversal(&db, Traversal::Indexed); + let csr = first_column_sorted(&query_main(&csr_db, REACH_3, "reach", &p).await.unwrap()); + let indexed = + first_column_sorted(&query_main(&indexed_db, REACH_3, "reach", &p).await.unwrap()); + let switches = Arc::new(AtomicU64::new(0)); + let probes = QueryIoProbes { + traversal_mid_switches: Arc::clone(&switches), + ..Default::default() + }; + let auto = first_column_sorted( + &with_query_io_probes(probes, query_main(&db, REACH_3, "reach", &p)) + .await + .unwrap(), + ); + + assert_eq!(csr.len(), 49, "p5 reaches every other clique member"); + assert!(csr.iter().any(|name| name == "p0") && !csr.iter().any(|name| name == "p5")); + assert_eq!(indexed, csr, "forced modes agree"); + assert_eq!( + auto, csr, + "auto (dispatch + mid-traversal re-decision) must match the forced paths" + ); + assert!( + switches.load(Ordering::Relaxed) >= 1, + "the mid-traversal Indexed→CSR switch must actually fire on this shape" + ); +} + #[tokio::test] async fn adaptive_switch_undirected_returns_identical_rows() { const REACH_3_UNDIRECTED: &str = r#" diff --git a/docs/dev/execution.md b/docs/dev/execution.md index 852d5bfd3..b952dc087 100644 --- a/docs/dev/execution.md +++ b/docs/dev/execution.md @@ -75,7 +75,15 @@ not force a graph-wide scan and a lazy branch may reuse an identical inherited table view. An explicitly bound edge must scan its edge table because topology alone does -not contain edge properties. It preserves incoming row/rank order and carries a +not contain edge properties. Every edge probe (`graph.rs::scan_edges_stream`: +the bound-edge producer, an indexed hop) pushes its `key IN (keys)` list into +Lance when the key column has usable BTREE coverage, which turns it into +index lookups, or when the list holds at most 1,024 keys; a longer list over +an unindexed column is not pushed, because Lance 13 clones the pushed list +for every fragment it reads (it builds the physical expression once per +distinct expression), so that scan reads unfiltered and the engine keeps the +rows whose key is in one set of the keys (one more column read per fragment +no key is in, which only a long list pays for). It preserves incoming row/rank order and carries a deterministic edge tie-break. A correlated block (`not { ... }`, `exists { ... }`, `count { ... } > 2`, `sum($d.size) { ... } > 100`) is one anti-join over its typed inner pipeline, keeping each outer row by a