From 4ec4d0a4a557fb0040522af6c0a4351dc38df9f1 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 08:54:19 +0000 Subject: [PATCH 01/13] perf(engine): size expand and pair-buffer rows in one pass per column (H1) A single-hop expand and the pair buffer estimated each input row's bytes with one ArrayData slice per column per row. WorkMemory::row_bytes reads the same numbers off each column's layout in one pass; a unit test pins equality with the per-row slice for every layout. Co-Authored-By: Claude Fable 5.1 --- .../omnigraph/src/engine/operators/memory.rs | 144 ++++++++++++++++++ .../src/engine/operators/pair_buffer.rs | 5 +- .../src/engine/operators/single_hop.rs | 4 +- 3 files changed, 146 insertions(+), 7 deletions(-) diff --git a/crates/omnigraph/src/engine/operators/memory.rs b/crates/omnigraph/src/engine/operators/memory.rs index f3a0fcc64..2ba073b48 100644 --- a/crates/omnigraph/src/engine/operators/memory.rs +++ b/crates/omnigraph/src/engine/operators/memory.rs @@ -867,6 +867,19 @@ impl WorkMemory { }) } + /// `slice_bytes(batch, row, 1)` for every row of `batch` in one pass per + /// column: the same numbers, read off each column's layout (a constant per + /// row for fixed-width columns, the offset deltas for variable-width ones, + /// children summed for structs) instead of one `ArrayData` slice per row + /// per column. + pub(in crate::engine) fn row_bytes(&self, batch: &RecordBatch) -> DfResult> { + let mut rows = vec![0usize; batch.num_rows()]; + for column in batch.columns() { + add_row_bytes(&column.to_data(), &mut rows)?; + } + Ok(rows) + } + pub(in crate::engine) fn take( &self, batch: &RecordBatch, @@ -1169,6 +1182,66 @@ fn take_scratch_size(data: &ArrayData, rows: usize) -> usize { /// Bytes copied from an Arrow range, including nested child ranges. Arrow's /// ArrayData slice keeps list child buffers whole, so recursing through its /// get_slice_memory_size would count the entire vector column per picked row. +/// Add each row's `slice_memory_size(data, row, 1)` to `rows`, one pass over +/// the column; a layout with no shortcut falls back to the per-row slice. +fn add_row_bytes(data: &ArrayData, rows: &mut [usize]) -> DfResult<()> { + if rows.is_empty() { + return Ok(()); + } + let validity = usize::from(data.nulls().is_some()); + match data.data_type() { + DataType::Utf8 | DataType::Binary => { + let offsets = &data.buffers()[0].typed_data::()[data.offset()..]; + for (row, bytes) in rows.iter_mut().enumerate() { + let value = (offsets[row + 1] - offsets[row]) as usize; + *bytes = bytes.saturating_add(4 + value + validity); + } + } + DataType::LargeUtf8 | DataType::LargeBinary => { + let offsets = &data.buffers()[0].typed_data::()[data.offset()..]; + for (row, bytes) in rows.iter_mut().enumerate() { + let value = (offsets[row + 1] - offsets[row]) as usize; + *bytes = bytes.saturating_add(8 + value + validity); + } + } + DataType::Struct(_) => { + for child in data.child_data() { + add_row_bytes(&child.slice(data.offset(), rows.len()), rows)?; + } + for bytes in rows.iter_mut() { + *bytes = bytes.saturating_add(validity); + } + } + DataType::List(_) + | DataType::LargeList(_) + | DataType::Map(_, _) + | DataType::FixedSizeList(_, _) => { + for (row, bytes) in rows.iter_mut().enumerate() { + *bytes = bytes.saturating_add(slice_memory_size(data, row, 1)?); + } + } + _ => { + if data.data_type().is_primitive() + || matches!( + data.data_type(), + DataType::Boolean | DataType::FixedSizeBinary(_) | DataType::Null + ) + { + // Fixed-width layouts cost the same for every row. + let each = slice_memory_size(data, 0, 1)?; + for bytes in rows.iter_mut() { + *bytes = bytes.saturating_add(each); + } + } else { + for (row, bytes) in rows.iter_mut().enumerate() { + *bytes = bytes.saturating_add(slice_memory_size(data, row, 1)?); + } + } + } + } + Ok(()) +} + fn slice_memory_size(data: &ArrayData, offset: usize, len: usize) -> DfResult { let data = data.slice(offset, len); let validity = if data.nulls().is_some() { @@ -1736,3 +1809,74 @@ mod tests { assert_eq!(pool.reserved(), 0); } } + +#[cfg(test)] +mod row_bytes_tests { + use super::*; + use arrow_array::builder::{FixedSizeListBuilder, Float32Builder, ListBuilder, StringBuilder}; + use arrow_array::{ArrayRef, BooleanArray, Int64Array, StringArray, StructArray}; + use arrow_schema::{Field, Fields}; + use datafusion::execution::context::SessionConfig; + use datafusion::execution::memory_pool::GreedyMemoryPool; + + /// One pass per column gives every row the bytes a one-row slice gives. + #[test] + fn row_bytes_equals_the_per_row_slice_for_every_layout() { + let limit = 1 << 20; + let pool: Arc = Arc::new(GreedyMemoryPool::new(limit)); + let resources = Arc::new(QueryResources::new(pool, limit as u64)); + let ctx = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_extension(resources)), + ); + let memory = WorkMemory::new(ctx, "row bytes").unwrap(); + let ints: ArrayRef = Arc::new(Int64Array::from(vec![Some(1), None, Some(3), Some(4)])); + let flags: ArrayRef = Arc::new(BooleanArray::from(vec![true, false, true, true])); + let names: ArrayRef = Arc::new(StringArray::from(vec![ + Some("a"), + Some("longer value"), + None, + Some(""), + ])); + let mut vectors = FixedSizeListBuilder::new(Float32Builder::new(), 3); + for row in 0..4 { + vectors.values().append_slice(&[row as f32; 3]); + vectors.append(row != 1); + } + let vectors: ArrayRef = Arc::new(vectors.finish()); + let mut lists = ListBuilder::new(StringBuilder::new()); + lists.values().append_value("x"); + lists.values().append_value("yy"); + lists.append(true); + lists.append(false); + lists.append(true); + lists.values().append_value("zzz"); + lists.append(true); + let lists: ArrayRef = Arc::new(lists.finish()); + let nested: ArrayRef = Arc::new(StructArray::new( + Fields::from(vec![ + Field::new("n", DataType::Int64, true), + Field::new("s", DataType::Utf8, true), + ]), + vec![Arc::clone(&ints), Arc::clone(&names)], + None, + )); + let batch = RecordBatch::try_from_iter([ + ("ints", ints), + ("flags", flags), + ("names", names), + ("vectors", vectors), + ("lists", lists), + ("nested", nested), + ]) + .unwrap(); + let sliced = batch.slice(1, 3); + for batch in [batch, sliced] { + let expected: Vec = (0..batch.num_rows()) + .map(|row| memory.slice_bytes(&batch, row, 1).unwrap()) + .collect(); + assert_eq!(memory.row_bytes(&batch).unwrap(), expected); + assert!(expected.iter().all(|bytes| *bytes > 0)); + } + } +} diff --git a/crates/omnigraph/src/engine/operators/pair_buffer.rs b/crates/omnigraph/src/engine/operators/pair_buffer.rs index 97c7f8130..3f63db887 100644 --- a/crates/omnigraph/src/engine/operators/pair_buffer.rs +++ b/crates/omnigraph/src/engine/operators/pair_buffer.rs @@ -53,10 +53,7 @@ pub(super) struct PairBuffer { } fn row_bytes(memory: &WorkMemory, batch: &RecordBatch, rows: &mut Vec) -> DfResult<()> { - rows.clear(); - for row in 0..batch.num_rows() { - rows.push(memory.slice_bytes(batch, row, 1)?); - } + *rows = memory.row_bytes(batch)?; Ok(()) } diff --git a/crates/omnigraph/src/engine/operators/single_hop.rs b/crates/omnigraph/src/engine/operators/single_hop.rs index 7258cfcca..e3378b108 100644 --- a/crates/omnigraph/src/engine/operators/single_hop.rs +++ b/crates/omnigraph/src/engine/operators/single_hop.rs @@ -202,9 +202,7 @@ impl SingleHop { })?; let hop_memory = memory.child("expand hop")?; hop_memory.entries::(batch.num_rows())?; - let row_bytes = (0..batch.num_rows()) - .map(|row| memory.slice_bytes(&batch, row, 1)) - .collect::>>()?; + let row_bytes = memory.row_bytes(&batch)?; let source = match &started.mode { Mode::Csr => { let gi = env From 7a62c1c098419f3449617fc75ad820c08f8b0a0f Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 09:06:38 +0000 Subject: [PATCH 02/13] perf(engine): evaluate a correlated block's predicate over all outer rows at once (H4) The anti-join compared each outer row's count or aggregate as a one-row array (a cast chain and a comparison per row). The counts, or the finalized aggregates, now form one column and the bound predicate runs once over it; a null verdict reads false as before. Co-Authored-By: Claude Fable 5.1 --- crates/omnigraph/src/engine/graph.rs | 11 +-- .../engine/operators/subquery_aggregate.rs | 69 ++++++++++++++++--- 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/crates/omnigraph/src/engine/graph.rs b/crates/omnigraph/src/engine/graph.rs index 8f32975ef..355227d61 100644 --- a/crates/omnigraph/src/engine/graph.rs +++ b/crates/omnigraph/src/engine/graph.rs @@ -1539,9 +1539,9 @@ pub(super) fn bulk_anti_join_mask( return Ok(None); }; memory - .entries::(outer_ids.len()) + .entries::(outer_ids.len()) .map_err(|error| memory.error(error))?; - let mut keep_mask = Vec::with_capacity(outer_ids.len()); + let mut counts = Vec::with_capacity(outer_ids.len()); let mut targets = Vec::new(); for i in 0..outer_ids.len() { memory.check().map_err(|error| memory.error(error))?; @@ -1559,9 +1559,12 @@ pub(super) fn bulk_anti_join_mask( } None => 0, }; - keep_mask.push(row_count.holds(matches)?); + counts.push(matches); } - Ok(Some(BooleanArray::from(keep_mask))) + memory + .entries::<(i64, bool)>(counts.len()) + .map_err(|error| memory.error(error))?; + Ok(Some(row_count.holds_all(&counts)?)) } /// Where the shared BFS core reads each hop's neighbors from. The two sources diff --git a/crates/omnigraph/src/engine/operators/subquery_aggregate.rs b/crates/omnigraph/src/engine/operators/subquery_aggregate.rs index 731279823..9fc3ccae9 100644 --- a/crates/omnigraph/src/engine/operators/subquery_aggregate.rs +++ b/crates/omnigraph/src/engine/operators/subquery_aggregate.rs @@ -6,7 +6,8 @@ use std::cmp::Ordering; use std::sync::Arc; use arrow_array::{ - Array, ArrayRef, BooleanArray, Decimal128Array, Float64Array, RecordBatch, UInt32Array, + Array, ArrayRef, BooleanArray, Decimal128Array, Float64Array, Int64Array, RecordBatch, + UInt32Array, }; use arrow_schema::DataType; use datafusion::common::ScalarValue; @@ -158,14 +159,37 @@ impl BoundComparison { }) } - fn holds(&self, value: ScalarValue) -> Result { - let value = value.to_array_of_size(1).map_err(OmniError::datafusion)?; - let value = cast_left(&self.left, &value)?; - let mask = compare_arrays(&value, self.op, &self.right)?; - Ok(mask.is_valid(0) && mask.value(0)) + /// Whether the predicate holds for each of `values`, the block + /// aggregate's results in the leaf's declared type: one cast chain and + /// one comparison over the whole column, a null verdict reading `false`. + fn holds_all(&self, values: &ArrayRef) -> Result { + let value = cast_left(&self.left, values)?; + let right = arrow_select::take::take( + self.right.as_ref(), + &UInt32Array::from(vec![0u32; value.len()]), + None, + ) + .map_err(OmniError::arrow_internal)?; + let mask = compare_arrays(&value, self.op, &right)?; + Ok(if mask.null_count() == 0 { + mask + } else { + arrow_select::filter::prep_null_mask_filter(&mask) + }) } } +/// `counts` as the `I64` column a block's row count is compared in. +fn count_values(counts: &[u64]) -> Result { + let counts = counts + .iter() + .map(|&count| { + i64::try_from(count).map_err(|_| OmniError::manifest("block count exceeds the I64 range")) + }) + .collect::>>()?; + Ok(Arc::new(Int64Array::from(counts))) +} + fn cast_left(expr: &BlockAggregateExpr, value: &ArrayRef) -> Result { let mut current = expr; let mut casts = Vec::new(); @@ -220,8 +244,14 @@ impl RowCountPredicate { self.existence_only } + /// The verdict of every outer row from its neighbour count. + pub(crate) fn holds_all(&self, counts: &[u64]) -> Result { + self.comparison.holds_all(&count_values(counts)?) + } + + #[cfg(test)] pub(crate) fn holds(&self, count: u64) -> Result { - self.comparison.holds(count_value(count)?) + Ok(self.holds_all(&[count])?.value(0)) } } @@ -395,11 +425,28 @@ impl SubqueryAggregate { Ok(value) } + /// Every outer row's verdict: the row counts as one `I64` column when + /// the block counts rows, else each row's finalized aggregate gathered + /// into one column, then one comparison. pub(crate) fn keep_mask(&self) -> Result { - (0..self.counts.len()) - .map(|row| self.comparison.holds(self.aggregate(row)?)) - .collect::>>() - .map(BooleanArray::from) + let leaf = self.comparison.left.leaf(); + let values = match (leaf, &self.values) { + (BlockAggregateExpr::CountRows { .. }, None) + | ( + BlockAggregateExpr::Aggregate { + func: AggFunc::Count, + .. + }, + None, + ) => count_values(&self.counts)?, + _ => { + let values = (0..self.counts.len()) + .map(|row| self.aggregate(row)) + .collect::>>()?; + ScalarValue::iter_to_array(values).map_err(OmniError::datafusion)? + } + }; + self.comparison.holds_all(&values) } } From 114d80f470f7f14d53a656a0ba5b34a40828078b Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 09:09:09 +0000 Subject: [PATCH 03/13] perf(engine): hash the join's ids in bulk and keep the build side in a borrowing table (H2) The destination hash join copied every build id into a HashMap and hashed each probe id through it. The build and probe ids are now hashed in bulk with DataFusion's create_hashes, and the build side is a hashbrown::HashTable<(hash, row)> that compares through the Arrow column, as DataFusion's own join does; the first row of a duplicate id still wins. Co-Authored-By: Claude Fable 5.1 --- Cargo.lock | 2 + crates/omnigraph/Cargo.toml | 3 + .../src/engine/operators/hash_join.rs | 104 +++++++++++++----- 3 files changed, 84 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67cf4ac0e..a84d1b97c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5766,6 +5766,7 @@ dependencies = [ name = "omnigraph-engine" version = "0.14.1" dependencies = [ + "ahash", "aho-corasick", "arc-swap", "arrow-array", @@ -5781,6 +5782,7 @@ dependencies = [ "csv-core", "datafusion", "futures", + "hashbrown 0.17.1", "lance", "lance-core", "lance-datafusion", diff --git a/crates/omnigraph/Cargo.toml b/crates/omnigraph/Cargo.toml index da2f65566..8d35801e4 100644 --- a/crates/omnigraph/Cargo.toml +++ b/crates/omnigraph/Cargo.toml @@ -41,6 +41,9 @@ lance = { workspace = true } lance-core = { workspace = true } lance-select = { workspace = true } lance-datafusion = { workspace = true } +ahash = { workspace = true } +# The hash join's build side: the `HashTable` DataFusion's own join uses. +hashbrown = "0.17" datafusion = { workspace = true } lance-file = { workspace = true } lance-index = { workspace = true } diff --git a/crates/omnigraph/src/engine/operators/hash_join.rs b/crates/omnigraph/src/engine/operators/hash_join.rs index f0f610128..6d6cd55bc 100644 --- a/crates/omnigraph/src/engine/operators/hash_join.rs +++ b/crates/omnigraph/src/engine/operators/hash_join.rs @@ -8,13 +8,14 @@ //! the node declares that fallback. An empty build stops the probe after its //! first batch. -use std::collections::HashMap; use std::fmt; use std::sync::Arc; -use arrow_array::{Array, RecordBatch, StringArray, UInt32Array}; +use arrow_array::{Array, ArrayRef, RecordBatch, StringArray, UInt32Array}; use arrow_schema::SchemaRef; +use datafusion::common::hash_utils::{RandomState, create_hashes}; use datafusion::common::{DataFusionError, Result as DfResult}; +use hashbrown::HashTable; use datafusion::execution::TaskContext; use datafusion::physical_plan::metrics::{ Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, MetricsSet, @@ -124,14 +125,43 @@ impl DisplayAs for HashJoinExec { } } -/// The drained build side: its rows and the row of every id, held under -/// `memory` for as long as the probe runs. +/// The seed of the join's hash function: fixed, so a build hashes the same +/// way on every run. +const HASH_SEED: u64 = 0x6f6d_6e69_6a6f_696e; + +/// The drained build side: its rows and, per distinct non-null id, the first +/// row holding it, as `(hash, row)` entries in a table that borrows nothing +/// (an entry compares through the id column of `batch`), held under +/// `memory` for as long as the probe runs. The ids are hashed in bulk with +/// DataFusion's `create_hashes`; no id is copied out of the Arrow column. struct Built { batch: RecordBatch, - row_by_id: HashMap, + ids: StringArray, + rows: HashTable<(u64, u32)>, + state: RandomState, _memory: WorkMemory, } +impl Built { + /// The first build row holding `id`, hashed as `hash`. + fn row_of(&self, hash: u64, id: &str) -> Option { + self.rows + .find(hash, |&(entry_hash, row)| { + entry_hash == hash && self.ids.value(row as usize) == id + }) + .map(|&(_, row)| row) + } +} + +/// `hashes` filled with the hash of every value of `ids` (nulls included, +/// which the callers skip) under `state`. +fn hash_ids(ids: &ArrayRef, state: &RandomState, hashes: &mut Vec) -> DfResult<()> { + hashes.clear(); + hashes.resize(ids.len(), 0); + create_hashes([ids], state, hashes)?; + Ok(()) +} + async fn build_side( build: &Arc, id_column: &str, @@ -148,22 +178,39 @@ async fn build_side( "destination scan must return Utf8 '{id_column}'" ))) })?; - work.entries::<(String, u32)>(ids.len())?; - work.string(ids.iter().flatten().map(str::len).sum())?; - let mut row_by_id = HashMap::with_capacity(ids.len()); - for (row, id) in ids.iter().enumerate() { - if let Some(id) = id { - let row = u32::try_from(row).map_err(|_| { - external(OmniError::manifest_internal( - "destination scan exceeds row index range", - )) - })?; - row_by_id.entry(id.to_string()).or_insert(row); + u32::try_from(ids.len()).map_err(|_| { + external(OmniError::manifest_internal( + "destination scan exceeds row index range", + )) + })?; + // The table's entries plus the hash buffer, one `u64` per row. + work.entries::<(u64, u32)>(ids.len())?; + work.entries::(ids.len())?; + let state = RandomState::with_seed(HASH_SEED); + let id_array: ArrayRef = Arc::new(ids.clone()); + let mut hashes = Vec::new(); + hash_ids(&id_array, &state, &mut hashes)?; + let mut rows: HashTable<(u64, u32)> = HashTable::with_capacity(ids.len()); + for (row, &hash) in hashes.iter().enumerate() { + if ids.is_null(row) { + continue; + } + let id = ids.value(row); + let occupied = rows + .find(hash, |&(entry_hash, entry_row)| { + entry_hash == hash && ids.value(entry_row as usize) == id + }) + .is_some(); + if !occupied { + rows.insert_unique(hash, (hash, row as u32), |&(entry_hash, _)| entry_hash); } } + let ids = ids.clone(); Ok(Built { batch, - row_by_id, + ids, + rows, + state, _memory: work, }) } @@ -315,16 +362,23 @@ impl Probe { })?; let work = Arc::new(memory.child(OUTPUT)?); work.entries::(ids.len().saturating_mul(2))?; + work.entries::(ids.len())?; + u32::try_from(ids.len()).map_err(|_| { + external(OmniError::manifest_internal( + "traversal batch exceeds row index range", + )) + })?; + let mut hashes = Vec::new(); + hash_ids(batch.column(id_index), &built.state, &mut hashes)?; let mut probe_rows = Vec::with_capacity(ids.len()); let mut build_rows = Vec::with_capacity(ids.len()); - for (row, id) in ids.iter().enumerate() { - if let Some(build_row) = id.and_then(|id| built.row_by_id.get(id)) { - probe_rows.push(u32::try_from(row).map_err(|_| { - external(OmniError::manifest_internal( - "traversal batch exceeds row index range", - )) - })?); - build_rows.push(*build_row); + for (row, &hash) in hashes.iter().enumerate() { + if ids.is_null(row) { + continue; + } + if let Some(build_row) = built.row_of(hash, ids.value(row)) { + probe_rows.push(row as u32); + build_rows.push(build_row); } } if probe_rows.is_empty() { From 3b471ef59cc823e76abb6e2b8f9bc649c39ba093 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 09:16:05 +0000 Subject: [PATCH 04/13] perf(engine): sieve an unindexed edge scan's keys in the engine instead of pushing the IN list (H5) Lance 13 clones the pushed filter and plans it again for every fragment, so a `key IN (keys)` over an edge table without a BTREE on the key column cost a copy and a hash set of every key per fragment. Where the key column has no usable index the edge scan now reads unfiltered and the engine keeps the rows whose key is in one set; an indexed key column still pushes the list, which is what makes it index lookups. Co-Authored-By: Claude Fable 5.1 --- crates/omnigraph/src/engine/graph.rs | 39 +++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/crates/omnigraph/src/engine/graph.rs b/crates/omnigraph/src/engine/graph.rs index 355227d61..9252c50b3 100644 --- a/crates/omnigraph/src/engine/graph.rs +++ b/crates/omnigraph/src/engine/graph.rs @@ -1729,14 +1729,25 @@ async fn scan_edges_stream( .copied() .filter(|column| *column != key_col && *column != opposite_col), ); - let filter = id_in_list_expr(keys, key_col); + // A `key IN (keys)` filter reaches Lance only where a BTREE on the key + // column turns it into index lookups. Elsewhere Lance scans every fragment + // anyway and re-plans the pushed list for each one (a clone and a hash + // set of every key per fragment), so the engine sieves the batches + // itself through one set of the keys. + let pushdown = matches!( + omnigraph_core::dataset_index::key_column_index_coverage(ds, key_col).await, + Ok(omnigraph_core::dataset_index::IndexCoverage::Indexed) + ); + let filter = pushdown.then(|| id_in_list_expr(keys, key_col)); let plan = crate::table_store::TableStore::scan_plan_with( ds, Some(&projection), None, false, |scanner| { - scanner.filter_expr(filter); + if let Some(filter) = filter { + scanner.filter_expr(filter); + } scanner.batch_size(memory.batch_rows()); scanner.batch_size_bytes(memory.batch_bytes() as u64); Ok(()) @@ -1744,7 +1755,29 @@ async fn scan_edges_stream( ) .await?; let (_, stream) = memory.stream(plan).map_err(|error| memory.error(error))?; - Ok(stream) + if pushdown { + return Ok(stream); + } + let wanted: hashbrown::HashSet = keys.iter().cloned().collect(); + let schema = stream.schema(); + let key_col = key_col.to_string(); + let sieved = stream.map(move |batch| { + let batch = batch?; + let keys = batch + .column_by_name(&key_col) + .and_then(|column| column.as_any().downcast_ref::()) + .ok_or_else(|| { + datafusion::common::DataFusionError::Internal(format!("edge batch has no Utf8 '{key_col}'")) + })?; + let mask = arrow_array::BooleanArray::from_iter( + keys.iter() + .map(|key| Some(key.is_some_and(|key| wanted.contains(key)))), + ); + Ok(arrow_select::filter::filter_record_batch(&batch, &mask)?) + }); + Ok(Box::pin( + datafusion::physical_plan::stream::RecordBatchStreamAdapter::new(schema, sieved), + )) } #[cfg(test)] From 4408f5ab9ffdaa21e22e88b108b69eeaf1478acf Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 09:22:17 +0000 Subject: [PATCH 05/13] perf(engine): walk a multi-hop BFS without per-source sets, per-edge pool charges or per-pair strings (H3) The BFS kept one visited set and one emitted set per source row, charged the pool up to three times per neighbour examined and copied every emitted destination id into a String and then into a builder. Both sets are now one table each over (source, node) keys, a node's neighbours are admitted once before its adjacency slice is walked, and emitted destinations stay dense until the chunk's Utf8 column is built from the walk's dictionary. Emission order, set semantics, hop bounds, the mid-walk switch to CSR and the budgeted path's per-neighbour work charge are unchanged. Co-Authored-By: Claude Fable 5.1 --- crates/omnigraph/src/engine/graph.rs | 191 ++++++++++-------- .../src/engine/operators/expand_stream.rs | 20 +- 2 files changed, 115 insertions(+), 96 deletions(-) diff --git a/crates/omnigraph/src/engine/graph.rs b/crates/omnigraph/src/engine/graph.rs index 9252c50b3..9fc87449d 100644 --- a/crates/omnigraph/src/engine/graph.rs +++ b/crates/omnigraph/src/engine/graph.rs @@ -474,13 +474,40 @@ pub(super) fn endpoint_probes( probes } -/// At most `batch_size` emitted ID pairs, charged to `_memory`. +/// At most `batch_size` emitted pairs, charged to `_memory`: the source row +/// ordinals and the destination ids as one `Utf8` column built from the +/// walk's dictionary. pub(super) struct ExpandedPairs { pub(super) source_rows: Vec, - pub(super) destination_ids: Vec, + pub(super) destination: ArrayRef, _memory: WorkMemory, } +/// A per-source set member as one key: the source row in the high half, the +/// dense node in the low half, so every source's visited and emitted sets +/// share one table instead of one allocation per source row. +fn source_node_key(source: usize, node: u32) -> u64 { + ((source as u64) << 32) | u64::from(node) +} + +/// The destination column of one chunk: `dense` resolved through `ids`, the +/// dictionary of the walk's destination space, charged to `memory` first. +fn destination_column(dense: &[u32], ids: &[String], memory: &WorkMemory) -> Result { + let bytes = dense + .iter() + .map(|&d| ids[d as usize].len()) + .fold(0usize, usize::saturating_add); + memory + .string(bytes) + .map_err(|error| memory.error(error))?; + memory + .grow((dense.len() + 1).saturating_mul(std::mem::size_of::())) + .map_err(|error| memory.error(error))?; + Ok(Arc::new(StringArray::from_iter_values( + dense.iter().map(|&d| ids[d as usize].as_str()), + ))) +} + pub(super) struct PreparedEdge { pub(super) dataset: Dataset, pub(super) probes: Vec, @@ -1212,11 +1239,13 @@ where let n = src_ids.len(); memory - .entries::<(Vec, HashSet, HashSet, u32)>(n) + .entries::<(Vec, u64, u32)>(n) .map_err(|error| memory.error(error))?; let mut frontiers: Vec> = Vec::with_capacity(n); - let mut visited: Vec> = Vec::with_capacity(n); - let mut seen_dst: Vec> = Vec::with_capacity(n); + // One table per set over every source row (`source_node_key`): the nodes + // a source reached and the destinations it emitted. + let mut visited: HashSet = HashSet::default(); + let mut seen_dst: HashSet = HashSet::default(); for i in 0..n { let seed = match &mut active { ActiveExpandSource::Indexed(src) => { @@ -1226,15 +1255,12 @@ where src_idx.to_dense(src_ids.value(i)) } }; - let mut v = HashSet::new(); if same_type { if let Some(s) = seed { - v.insert(s); + visited.insert(source_node_key(i, s)); } } frontiers.push(seed.map(|s| vec![s]).unwrap_or_default()); - visited.push(v); - seen_dst.push(HashSet::new()); } memory.checkpoint().map_err(|error| memory.error(error))?; @@ -1245,9 +1271,18 @@ where let mut frontier_memory = memory .child("expand frontier") .map_err(|error| memory.error(error))?; + emission_memory + .entries::<(u32, u32)>(chunk_rows) + .map_err(|error| memory.error(error))?; let mut emitted_src: Vec = Vec::with_capacity(chunk_rows); - let mut emitted_dst: Vec = Vec::with_capacity(chunk_rows); + let mut emitted_dst: Vec = Vec::with_capacity(chunk_rows); let mut prev_union_len: usize = 0; + // What one examined neighbour may add: a visited entry, a frontier + // entry, an emitted-set entry and a pair; charged per node, not per edge. + let per_neighbour_bytes = WorkMemory::entry_bytes::() + .saturating_mul(2) + .saturating_add(WorkMemory::entry_bytes::()) + .saturating_add(WorkMemory::entry_bytes::<(u32, u32)>()); for hop in 1..=max { memory.check().map_err(|error| memory.error(error))?; @@ -1316,26 +1351,33 @@ where unreachable!("active source was just replaced with Csr"); }; hop_memory - .entries::( + .entries::( frontiers.iter().map(Vec::len).sum::() - + visited.iter().map(HashSet::len).sum::() - + seen_dst.iter().map(HashSet::len).sum::(), + + visited.len() + + seen_dst.len(), ) .map_err(|error| memory.error(error))?; - let translate_set = - |set: &HashSet, idx: &crate::graph_index::TypeIndex| -> HashSet { - set.iter() - .filter_map(|&d| interner.to_id(d).and_then(|id| idx.to_dense(id))) - .collect() - }; - for i in 0..n { - frontiers[i] = frontiers[i] + let translate_set = |set: &HashSet, + idx: &crate::graph_index::TypeIndex| + -> HashSet { + set.iter() + .filter_map(|&key| { + let (source, dense) = ((key >> 32) as usize, key as u32); + interner + .to_id(dense) + .and_then(|id| idx.to_dense(id)) + .map(|dense| source_node_key(source, dense)) + }) + .collect() + }; + for frontier in frontiers.iter_mut() { + *frontier = frontier .iter() .filter_map(|&d| interner.to_id(d).and_then(|id| src_idx.to_dense(id))) .collect(); - visited[i] = translate_set(&visited[i], src_idx); - seen_dst[i] = translate_set(&seen_dst[i], dst_idx); } + visited = translate_set(&visited, src_idx); + seen_dst = translate_set(&seen_dst, dst_idx); tracing::debug!( target: "omnigraph::traverse", edge = %member.edge_type, @@ -1399,75 +1441,57 @@ where adj_rev.map(|a| a.neighbors(node)).unwrap_or(&[]), ), }; + let ids: &[String] = match &active { + ActiveExpandSource::Indexed(src) => src.interner.ids(), + ActiveExpandSource::Csr(CsrSource { dst_idx, .. }) => dst_idx.ids(), + }; + // Admit what this node's neighbours can add before the walk. + memory + .grow((fwd.len() + rev.len()).saturating_mul(per_neighbour_bytes)) + .map_err(|error| memory.error(error))?; for &neighbor in fwd.iter().chain(rev) { if budgeted { memory .charge_traversal(1) .map_err(|error| memory.error(error))?; - } else { - memory.check().map_err(|error| memory.error(error))?; } let is_self = same_type && hop == 1 && neighbor == node; - if !is_self && same_type { - if visited[i].contains(&neighbor) { - continue; - } - memory - .entries::(1) - .map_err(|error| memory.error(error))?; - visited[i].insert(neighbor); + if !is_self && same_type && !visited.insert(source_node_key(i, neighbor)) { + continue; } if !is_self { - next_memory - .entries::(1) - .map_err(|error| memory.error(error))?; next.push(neighbor); } - if hop >= min_hops && !seen_dst[i].contains(&neighbor) { - memory - .entries::(1) - .map_err(|error| memory.error(error))?; - seen_dst[i].insert(neighbor); - let dst_id = match &active { - ActiveExpandSource::Indexed(src) => Some( - src.interner - .to_id(neighbor) - .expect("interned dst id must resolve"), - ), - ActiveExpandSource::Csr(CsrSource { dst_idx, .. }) => { - dst_idx.to_id(neighbor) - } - }; - if let Some(dst_id) = dst_id { - emission_memory - .entries::<(u32, String)>(1) + if hop >= min_hops + && (neighbor as usize) < ids.len() + && seen_dst.insert(source_node_key(i, neighbor)) + { + emitted_src.push(i as u32); + emitted_dst.push(neighbor); + if emitted_src.len() >= chunk_rows { + let next_emission = memory + .child("expand emissions") .map_err(|error| memory.error(error))?; - emission_memory - .string(dst_id.len()) + next_emission + .entries::<(u32, u32)>(chunk_rows) .map_err(|error| memory.error(error))?; - emitted_src.push(i as u32); - emitted_dst.push(dst_id.to_string()); - if emitted_src.len() >= chunk_rows { - let next = memory - .child("expand emissions") - .map_err(|error| memory.error(error))?; - memory.metric("expand_pairs", emitted_src.len()); - emit(ExpandedPairs { - source_rows: std::mem::replace( - &mut emitted_src, - Vec::with_capacity(chunk_rows), - ), - destination_ids: std::mem::replace( - &mut emitted_dst, - Vec::with_capacity(chunk_rows), - ), - _memory: std::mem::replace(&mut emission_memory, next), - }) - .await?; - } + memory.metric("expand_pairs", emitted_src.len()); + let destination = + destination_column(&emitted_dst, ids, &emission_memory)?; + emitted_dst.clear(); + emit(ExpandedPairs { + source_rows: std::mem::replace( + &mut emitted_src, + Vec::with_capacity(chunk_rows), + ), + destination, + _memory: std::mem::replace(&mut emission_memory, next_emission), + }) + .await?; } } } + memory.check().map_err(|error| memory.error(error))?; } frontiers[i] = next; } @@ -1476,6 +1500,15 @@ where src.neighbor_map = HashMap::new(); } } + let destination = if emitted_src.is_empty() { + None + } else { + let ids: &[String] = match &active { + ActiveExpandSource::Indexed(src) => src.interner.ids(), + ActiveExpandSource::Csr(CsrSource { dst_idx, .. }) => dst_idx.ids(), + }; + Some(destination_column(&emitted_dst, ids, &emission_memory)?) + }; drop(active); drop(frontiers); drop(visited); @@ -1483,13 +1516,13 @@ where drop(frontier_memory); memory.release_work(); - if emitted_src.is_empty() { + let Some(destination) = destination else { return Ok(()); - } + }; memory.metric("expand_pairs", emitted_src.len()); emit(ExpandedPairs { source_rows: emitted_src, - destination_ids: emitted_dst, + destination, _memory: emission_memory, }) .await diff --git a/crates/omnigraph/src/engine/operators/expand_stream.rs b/crates/omnigraph/src/engine/operators/expand_stream.rs index 8a2c33347..2545b37cb 100644 --- a/crates/omnigraph/src/engine/operators/expand_stream.rs +++ b/crates/omnigraph/src/engine/operators/expand_stream.rs @@ -4,7 +4,7 @@ use std::fmt; use std::sync::Arc; -use arrow_array::{ArrayRef, RecordBatch, UInt32Array, builder::StringBuilder}; +use arrow_array::{ArrayRef, RecordBatch, UInt32Array}; use arrow_schema::SchemaRef; use datafusion::common::{DataFusionError, Result}; use datafusion::execution::TaskContext; @@ -480,22 +480,8 @@ async fn emit_pairs( let end = (offset + rows).min(pairs.source_rows.len()); work.entries::(end - offset)?; let source = UInt32Array::from(pairs.source_rows[offset..end].to_vec()); - let ids = &pairs.destination_ids[offset..end]; - let bytes = ids.iter().map(String::len).sum(); - work.string(bytes)?; - let mut destination = StringBuilder::with_capacity(ids.len(), bytes); - for id in ids { - work.check()?; - destination.append_value(id); - } - let output = align_sources( - wide, - &source, - Arc::new(destination.finish()), - &[], - schema, - &work, - )?; + let destination = pairs.destination.slice(offset, end - offset); + let output = align_sources(wide, &source, destination, &[], schema, &work)?; sender.send_bounded(output, work).await?; } Ok(()) From 1e87699e97c00ccfba536967e5beef9e136ae4cb Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 10:08:14 +0000 Subject: [PATCH 06/13] style(engine): iterate the BFS frontiers without indexing by the loop counter Co-Authored-By: Claude Fable 5.1 --- crates/omnigraph/src/engine/graph.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/omnigraph/src/engine/graph.rs b/crates/omnigraph/src/engine/graph.rs index 9fc87449d..9e0eb7b09 100644 --- a/crates/omnigraph/src/engine/graph.rs +++ b/crates/omnigraph/src/engine/graph.rs @@ -1424,8 +1424,8 @@ where } } - for i in 0..n { - let cur = std::mem::take(&mut frontiers[i]); + for (i, frontier) in frontiers.iter_mut().enumerate() { + let cur = std::mem::take(frontier); let mut next: Vec = Vec::new(); for &node in &cur { let (fwd, rev): (&[u32], &[u32]) = match &active { @@ -1493,7 +1493,7 @@ where } memory.check().map_err(|error| memory.error(error))?; } - frontiers[i] = next; + *frontier = next; } drop(std::mem::replace(&mut frontier_memory, next_memory)); if let ActiveExpandSource::Indexed(src) = &mut active { From fc0e33c804d35a335ebcff41282fbc0fe306c7bc Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 10:34:26 +0000 Subject: [PATCH 07/13] perf(engine): carry the expand's index-coverage verdict into its edge scans H5 probed the key column's index coverage in every edge scan, one metadata read per scan, which the DST storage-cost golden caught as one more get on every hop read. The expand's start decision already holds that verdict (the plan's pinned index facts or its runtime probe), so it travels with the indexed start and the BFS source; only a scan nothing has judged (a bound edge, a budgeted selection) probes for itself. Also records the quiet re-measurement and the gate findings in the report. Co-Authored-By: Claude Fable 5.1 --- crates/omnigraph/src/engine/graph.rs | 67 +++++++++++++++---- .../src/engine/operators/single_hop.rs | 18 ++++- 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/crates/omnigraph/src/engine/graph.rs b/crates/omnigraph/src/engine/graph.rs index 9e0eb7b09..50d5c19d8 100644 --- a/crates/omnigraph/src/engine/graph.rs +++ b/crates/omnigraph/src/engine/graph.rs @@ -578,7 +578,7 @@ where .iter() .map(|edge| (edge.dataset.clone(), edge.probes.clone())) .collect(); - (Some(datasets), HopPolicy::Off) + (Some((datasets, None)), HopPolicy::Off) } ExpandExecution::Named(named) => { let start = decide_expand_start( @@ -599,13 +599,20 @@ where ExpandStart::Indexed { edge_ds, hop_policy, + pushdown, } => { Switch::IndexedScan.record(switch); ( - Some(vec![( - *edge_ds, - endpoint_probes(named.member.direction, env.catalog.system_columns), - )]), + Some(( + vec![( + *edge_ds, + endpoint_probes( + named.member.direction, + env.catalog.system_columns, + ), + )], + Some(pushdown), + )), hop_policy, ) } @@ -631,6 +638,10 @@ pub(super) enum ExpandStart { Indexed { edge_ds: Box, hop_policy: HopPolicy, + /// Whether the key column's BTREE coverage, as the plan's pinned + /// index facts or the runtime probe judged it, makes a pushed + /// `key IN (keys)` list index lookups (`scan_edges_stream`). + pushdown: bool, }, } @@ -782,6 +793,13 @@ pub(super) async fn decide_expand_start( if let Some(coverage) = &coverage { warn_on_degraded_coverage(coverage, key_col, edge_type); } + let pushdown = match &coverage { + Some(Ok(crate::table_store::IndexCoverage::Indexed)) => true, + Some(_) => false, + None => corrected.as_ref().is_none_or(|inputs| { + inputs.coverage == omnigraph_planner::IndexCoverage::Indexed + }), + }; let hop_policy = match corrected { Some(inputs) => HopPolicy::Full(inputs), None if matches!(named.origin, ModeOrigin::Pinned) => HopPolicy::Off, @@ -794,6 +812,7 @@ pub(super) async fn decide_expand_start( Ok(ExpandStart::Indexed { edge_ds: Box::new(edge_ds), hop_policy, + pushdown, }) } @@ -938,7 +957,15 @@ where .child("bound edge scan") .map_err(|error| memory.error(error))?; let mut stream = - scan_edges_stream(&dataset, orientation, &attach_columns, &keys, &scan_memory).await?; + scan_edges_stream( + &dataset, + orientation, + &attach_columns, + &keys, + None, + &scan_memory, + ) + .await?; while let Some(batch) = stream.next().await { let batch = batch.map_err(|error| memory.error(error))?; let input_memory = scan_memory @@ -1172,7 +1199,7 @@ pub(super) async fn execute_expand_bfs( graph_index: &GraphIndexHandle, catalog: &Catalog, step: &ExpandStep, - start_indexed: Option)>>, + start_indexed: Option<(Vec<(Dataset, Vec)>, Option)>, hop_policy: HopPolicy, side: &Gauge, memory: &WorkMemory, @@ -1216,8 +1243,9 @@ where let max = step.max_hops; let mut active = match start_indexed { - Some(datasets) => ActiveExpandSource::Indexed(Box::new(IndexedExpandSource { + Some((datasets, pushdown)) => ActiveExpandSource::Indexed(Box::new(IndexedExpandSource { datasets, + pushdown, interner: crate::graph_index::TypeIndex::new(), neighbor_map: HashMap::new(), })), @@ -1415,6 +1443,7 @@ where dataset, probes, &union_keys, + src.pushdown, &mut src.interner, &mut src.neighbor_map, &hop_memory, @@ -1617,6 +1646,9 @@ pub(super) enum ActiveExpandSource<'g> { pub(super) struct IndexedExpandSource { pub(super) datasets: Vec<(Dataset, Vec)>, + /// The known coverage verdict for the pushed key list, when the start + /// decision judged it; `None` lets each scan probe. + pub(super) pushdown: Option, pub(super) interner: crate::graph_index::TypeIndex, /// This hop's dense key -> dense neighbors (scan order; duplicates /// preserved, like CSR multi-edges). Rebuilt per hop. @@ -1634,10 +1666,12 @@ pub(super) enum HopPolicy { /// One `key IN (keys)` scan per probe, every row's endpoints interned into /// `interner` and the opposite appended to `neighbor_map[key]` in scan order /// (probe 0's rows, then probe 1's; parallel edges kept, like the CSR). +#[allow(clippy::too_many_arguments)] pub(super) async fn scan_neighbor_map( edge_ds: &Dataset, probes: &[EndpointColumns], keys: &[String], + pushdown: Option, interner: &mut crate::graph_index::TypeIndex, neighbor_map: &mut HashMap>, hop_memory: &WorkMemory, @@ -1648,7 +1682,7 @@ pub(super) async fn scan_neighbor_map( key: key_col, opposite: opp_col, } = orientation; - let batches = scan_edges(edge_ds, orientation, &[], keys, hop_memory).await?; + let batches = scan_edges(edge_ds, orientation, &[], keys, pushdown, hop_memory).await?; for batch in &batches { let keys = utf8_column(batch, key_col)?; let opposites = utf8_column(batch, opp_col)?; @@ -1696,12 +1730,13 @@ async fn scan_edges( orientation: EndpointColumns, extras: &[&str], keys: &[String], + pushdown: Option, memory: &WorkMemory, ) -> Result> { if keys.is_empty() { return Ok(Vec::new()); } - let stream = scan_edges_stream(ds, orientation, extras, keys, memory).await?; + let stream = scan_edges_stream(ds, orientation, extras, keys, pushdown, memory).await?; memory .collect(stream) .await @@ -1733,6 +1768,7 @@ async fn scan_edges_stream( orientation: EndpointColumns, extras: &[&str], keys: &[String], + pushdown: Option, memory: &WorkMemory, ) -> Result { if memory.traversal_limited() { @@ -1767,10 +1803,13 @@ async fn scan_edges_stream( // anyway and re-plans the pushed list for each one (a clone and a hash // set of every key per fragment), so the engine sieves the batches // itself through one set of the keys. - let pushdown = matches!( - omnigraph_core::dataset_index::key_column_index_coverage(ds, key_col).await, - Ok(omnigraph_core::dataset_index::IndexCoverage::Indexed) - ); + let pushdown = match pushdown { + Some(known) => known, + None => matches!( + omnigraph_core::dataset_index::key_column_index_coverage(ds, key_col).await, + Ok(omnigraph_core::dataset_index::IndexCoverage::Indexed) + ), + }; let filter = pushdown.then(|| id_in_list_expr(keys, key_col)); let plan = crate::table_store::TableStore::scan_plan_with( ds, diff --git a/crates/omnigraph/src/engine/operators/single_hop.rs b/crates/omnigraph/src/engine/operators/single_hop.rs index e3378b108..151aec348 100644 --- a/crates/omnigraph/src/engine/operators/single_hop.rs +++ b/crates/omnigraph/src/engine/operators/single_hop.rs @@ -54,6 +54,7 @@ enum Mode { Indexed { edge_ds: Box, hop_policy: HopPolicy, + pushdown: bool, }, } @@ -118,11 +119,13 @@ impl SingleHop { ExpandStart::Indexed { edge_ds, hop_policy, + pushdown, } => { Switch::IndexedScan.record(&self.switch); Mode::Indexed { edge_ds, hop_policy, + pushdown, } } }; @@ -220,7 +223,12 @@ impl SingleHop { .map_err(external)?, } } - Mode::Indexed { edge_ds, .. } => Source::Indexed { edge_ds }, + Mode::Indexed { + edge_ds, pushdown, .. + } => Source::Indexed { + edge_ds, + pushdown: *pushdown, + }, }; let (src_dense, ids, neighbours) = source .prepare( @@ -273,7 +281,10 @@ impl Streamed for SingleHop { /// edge dataset a batch-local scan reads. enum Source<'g> { Csr { csr: CsrSource<'g> }, - Indexed { edge_ds: &'g lance::Dataset }, + Indexed { + edge_ds: &'g lance::Dataset, + pushdown: bool, + }, } /// One input batch's neighbour lists: CSR slices, or the batch's own scan. @@ -325,7 +336,7 @@ impl<'g> Source<'g> { }, )) } - Self::Indexed { edge_ds } => { + Self::Indexed { edge_ds, pushdown } => { let mut interner = TypeIndex::new(); let src_dense = (0..src_ids.len()) .map(|row| intern(&mut interner, src_ids.value(row), hop_memory).map(Some)) @@ -339,6 +350,7 @@ impl<'g> Source<'g> { edge_ds, probes, &keys, + Some(*pushdown), &mut interner, &mut map, hop_memory, From f9a6893c94cfe3495bd73d3d28e41e52f54e7e4f Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 11:06:49 +0000 Subject: [PATCH 08/13] perf(engine): push a short key list into an unindexed edge scan, sieve a long one The DST storage-cost golden showed the engine-side sieve costs one read per fragment no key is in, since a pushed filter lets Lance skip a fragment's other columns there. A list of at most 1,024 keys (the indexed path's frontier ceiling) is pushed again, where Lance's per-fragment replan is cheap; a longer list over an unindexed column is sieved. The golden is unchanged. Co-Authored-By: Claude Fable 5.1 --- crates/omnigraph/src/engine/graph.rs | 60 +++++++++++-------- .../src/engine/operators/single_hop.rs | 4 +- docs/dev/execution.md | 9 ++- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/crates/omnigraph/src/engine/graph.rs b/crates/omnigraph/src/engine/graph.rs index 50d5c19d8..fab0e679f 100644 --- a/crates/omnigraph/src/engine/graph.rs +++ b/crates/omnigraph/src/engine/graph.rs @@ -606,10 +606,7 @@ where Some(( vec![( *edge_ds, - endpoint_probes( - named.member.direction, - env.catalog.system_columns, - ), + endpoint_probes(named.member.direction, env.catalog.system_columns), )], Some(pushdown), )), @@ -796,9 +793,9 @@ pub(super) async fn decide_expand_start( let pushdown = match &coverage { Some(Ok(crate::table_store::IndexCoverage::Indexed)) => true, Some(_) => false, - None => corrected.as_ref().is_none_or(|inputs| { - inputs.coverage == omnigraph_planner::IndexCoverage::Indexed - }), + None => corrected + .as_ref() + .is_none_or(|inputs| inputs.coverage == omnigraph_planner::IndexCoverage::Indexed), }; let hop_policy = match corrected { Some(inputs) => HopPolicy::Full(inputs), @@ -956,16 +953,15 @@ where let scan_memory = memory .child("bound edge scan") .map_err(|error| memory.error(error))?; - let mut stream = - scan_edges_stream( - &dataset, - orientation, - &attach_columns, - &keys, - None, - &scan_memory, - ) - .await?; + let mut stream = scan_edges_stream( + &dataset, + orientation, + &attach_columns, + &keys, + None, + &scan_memory, + ) + .await?; while let Some(batch) = stream.next().await { let batch = batch.map_err(|error| memory.error(error))?; let input_memory = scan_memory @@ -1199,7 +1195,7 @@ pub(super) async fn execute_expand_bfs( graph_index: &GraphIndexHandle, catalog: &Catalog, step: &ExpandStep, - start_indexed: Option<(Vec<(Dataset, Vec)>, Option)>, + start_indexed: Option, hop_policy: HopPolicy, side: &Gauge, memory: &WorkMemory, @@ -1644,6 +1640,10 @@ pub(super) enum ActiveExpandSource<'g> { Csr(CsrSource<'g>), } +/// An indexed walk's edge datasets with their probes, and the pushdown +/// verdict the start decision reached (`None`: each scan probes). +pub(super) type IndexedStart = (Vec<(Dataset, Vec)>, Option); + pub(super) struct IndexedExpandSource { pub(super) datasets: Vec<(Dataset, Vec)>, /// The known coverage verdict for the pushed key list, when the start @@ -1763,6 +1763,13 @@ fn physical_scan_rows( }) } +/// The longest key list an unindexed edge scan still pushes into Lance; see +/// the pushdown rule in [`scan_edges_stream`]. The indexed expand path's +/// frontier ceiling (`OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER`, 1,024 by +/// default) is this size: a frontier the plan sends to per-hop scans stays +/// pushed, a bound edge over a larger source set is sieved. +const PUSHDOWN_KEYS_MAX: usize = 1024; + async fn scan_edges_stream( ds: &Dataset, orientation: EndpointColumns, @@ -1798,13 +1805,18 @@ async fn scan_edges_stream( .copied() .filter(|column| *column != key_col && *column != opposite_col), ); - // A `key IN (keys)` filter reaches Lance only where a BTREE on the key - // column turns it into index lookups. Elsewhere Lance scans every fragment - // anyway and re-plans the pushed list for each one (a clone and a hash - // set of every key per fragment), so the engine sieves the batches - // itself through one set of the keys. + // A `key IN (keys)` filter reaches Lance where a BTREE on the key column + // turns it into index lookups, and where the list is small: Lance 13 + // re-plans a pushed list for every fragment (a clone and a hash set of + // every key per fragment), but a pushed filter also lets it read a + // fragment's other columns only for the rows the key column admits, one + // read fewer per fragment no key is in. Up to `PUSHDOWN_KEYS_MAX` keys + // the per-fragment replan is cheaper than that read; past it the engine + // sieves the batches itself through one set of the keys. let pushdown = match pushdown { - Some(known) => known, + Some(true) => true, + _ if keys.len() <= PUSHDOWN_KEYS_MAX => true, + Some(false) => false, None => matches!( omnigraph_core::dataset_index::key_column_index_coverage(ds, key_col).await, Ok(omnigraph_core::dataset_index::IndexCoverage::Indexed) diff --git a/crates/omnigraph/src/engine/operators/single_hop.rs b/crates/omnigraph/src/engine/operators/single_hop.rs index 151aec348..1017d41cc 100644 --- a/crates/omnigraph/src/engine/operators/single_hop.rs +++ b/crates/omnigraph/src/engine/operators/single_hop.rs @@ -280,7 +280,9 @@ impl Streamed for SingleHop { /// One batch's neighbour source: CSR slices borrowed from the index, or the /// edge dataset a batch-local scan reads. enum Source<'g> { - Csr { csr: CsrSource<'g> }, + Csr { + csr: CsrSource<'g>, + }, Indexed { edge_ds: &'g lance::Dataset, pushdown: bool, diff --git a/docs/dev/execution.md b/docs/dev/execution.md index 7553fc097..a251b0ae2 100644 --- a/docs/dev/execution.md +++ b/docs/dev/execution.md @@ -75,7 +75,14 @@ not force a graph-wide scan and a lazy branch may reuse an identical inherited table view. An explicitly bound edge must scan its edge table because topology alone does -not contain edge properties. It preserves incoming row/rank order and carries a +not contain edge properties. Every edge probe (`graph.rs::scan_edges_stream`: +the bound-edge producer, an indexed hop) pushes its `key IN (keys)` list into +Lance when the key column has usable BTREE coverage, which turns it into +index lookups, or when the list holds at most 1,024 keys; a longer list over +an unindexed column is not pushed, because Lance 13 clones and plans the +list again for every fragment, so that scan reads unfiltered and the engine +keeps the rows whose key is in one set of the keys (one more column read per +fragment no key is in, which only a long list pays for). It preserves incoming row/rank order and carries a deterministic edge tie-break. A correlated block (`not { ... }`, `exists { ... }`, `count { ... } > 2`, `sum($d.size) { ... } > 100`) is one anti-join over its typed inner pipeline, keeping each outer row by a From b8f488bceb9c5a8ebca4859dc322dad75931731b Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 20:20:37 +0000 Subject: [PATCH 09/13] style(engine): rustfmt the traversal changes and add their release note Co-Authored-By: Claude Fable 5.1 --- changelog.d/traversal-per-row-work.changed.md | 1 + crates/omnigraph/src/engine/graph.rs | 8 ++++---- crates/omnigraph/src/engine/operators/hash_join.rs | 2 +- .../omnigraph/src/engine/operators/subquery_aggregate.rs | 3 ++- 4 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 changelog.d/traversal-per-row-work.changed.md diff --git a/changelog.d/traversal-per-row-work.changed.md b/changelog.d/traversal-per-row-work.changed.md new file mode 100644 index 000000000..aea3b127f --- /dev/null +++ b/changelog.d/traversal-per-row-work.changed.md @@ -0,0 +1 @@ +- Traversals allocate less per row: expand row sizing in one pass, bulk-hashed destination joins, one visited table per multi-hop walk, engine-side key sieving for long unindexed edge probes, and block predicates evaluated once over all outer rows. Results are unchanged. diff --git a/crates/omnigraph/src/engine/graph.rs b/crates/omnigraph/src/engine/graph.rs index fab0e679f..ba3849028 100644 --- a/crates/omnigraph/src/engine/graph.rs +++ b/crates/omnigraph/src/engine/graph.rs @@ -497,9 +497,7 @@ fn destination_column(dense: &[u32], ids: &[String], memory: &WorkMemory) -> Res .iter() .map(|&d| ids[d as usize].len()) .fold(0usize, usize::saturating_add); - memory - .string(bytes) - .map_err(|error| memory.error(error))?; + memory.string(bytes).map_err(|error| memory.error(error))?; memory .grow((dense.len() + 1).saturating_mul(std::mem::size_of::())) .map_err(|error| memory.error(error))?; @@ -1851,7 +1849,9 @@ async fn scan_edges_stream( .column_by_name(&key_col) .and_then(|column| column.as_any().downcast_ref::()) .ok_or_else(|| { - datafusion::common::DataFusionError::Internal(format!("edge batch has no Utf8 '{key_col}'")) + datafusion::common::DataFusionError::Internal(format!( + "edge batch has no Utf8 '{key_col}'" + )) })?; let mask = arrow_array::BooleanArray::from_iter( keys.iter() diff --git a/crates/omnigraph/src/engine/operators/hash_join.rs b/crates/omnigraph/src/engine/operators/hash_join.rs index 6d6cd55bc..e2399394d 100644 --- a/crates/omnigraph/src/engine/operators/hash_join.rs +++ b/crates/omnigraph/src/engine/operators/hash_join.rs @@ -15,7 +15,6 @@ use arrow_array::{Array, ArrayRef, RecordBatch, StringArray, UInt32Array}; use arrow_schema::SchemaRef; use datafusion::common::hash_utils::{RandomState, create_hashes}; use datafusion::common::{DataFusionError, Result as DfResult}; -use hashbrown::HashTable; use datafusion::execution::TaskContext; use datafusion::physical_plan::metrics::{ Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, MetricsSet, @@ -24,6 +23,7 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream, }; use futures::future::BoxFuture; +use hashbrown::HashTable; use omnigraph_compiler::catalog::Catalog; use omnigraph_compiler::ir::{IRExpr, ParamMap}; use omnigraph_planner::AccessPath; diff --git a/crates/omnigraph/src/engine/operators/subquery_aggregate.rs b/crates/omnigraph/src/engine/operators/subquery_aggregate.rs index 9fc3ccae9..2525097a1 100644 --- a/crates/omnigraph/src/engine/operators/subquery_aggregate.rs +++ b/crates/omnigraph/src/engine/operators/subquery_aggregate.rs @@ -184,7 +184,8 @@ fn count_values(counts: &[u64]) -> Result { let counts = counts .iter() .map(|&count| { - i64::try_from(count).map_err(|_| OmniError::manifest("block count exceeds the I64 range")) + i64::try_from(count) + .map_err(|_| OmniError::manifest("block count exceeds the I64 range")) }) .collect::>>()?; Ok(Arc::new(Int64Array::from(counts))) From abc35505d07da0f178c616976556a9e8d40d98f8 Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 20:28:57 +0000 Subject: [PATCH 10/13] docs(release): shorten the traversal release note to the limit Co-Authored-By: Claude Fable 5.1 --- changelog.d/traversal-per-row-work.changed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/traversal-per-row-work.changed.md b/changelog.d/traversal-per-row-work.changed.md index aea3b127f..22dd5624b 100644 --- a/changelog.d/traversal-per-row-work.changed.md +++ b/changelog.d/traversal-per-row-work.changed.md @@ -1 +1 @@ -- Traversals allocate less per row: expand row sizing in one pass, bulk-hashed destination joins, one visited table per multi-hop walk, engine-side key sieving for long unindexed edge probes, and block predicates evaluated once over all outer rows. Results are unchanged. +- Traversals allocate less per row: one-pass expand sizing, bulk-hashed joins, one visited table per walk, engine-side key sieving, block predicates run once. Results are unchanged. From d8d4c08c2266562329b8285619791d2de729a96a Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 22:46:24 +0000 Subject: [PATCH 11/13] fix(engine): flush the pending expand chunk before the walk swaps to the CSR The Indexed-to-CSR swap in execute_expand_bfs translated the frontiers, the visited set and the emitted set, but not the pending emission buffer, whose dense ids belong to the interner; the chunk then resolved them through the CSR dictionary and returned other nodes' ids (p5 for p0 from a later seed). The pending chunk now leaves as strings before the dictionaries change, a dense id outside its dictionary is an error instead of a panic, and the later-seed case sits beside adaptive_switch_returns_identical_rows. --- crates/omnigraph/src/engine/graph.rs | 104 +++++++++++++----- .../tests/integration/traversal_adaptive.rs | 38 +++++++ 2 files changed, 116 insertions(+), 26 deletions(-) diff --git a/crates/omnigraph/src/engine/graph.rs b/crates/omnigraph/src/engine/graph.rs index ba3849028..dfdba5eb2 100644 --- a/crates/omnigraph/src/engine/graph.rs +++ b/crates/omnigraph/src/engine/graph.rs @@ -493,17 +493,53 @@ fn source_node_key(source: usize, node: u32) -> u64 { /// The destination column of one chunk: `dense` resolved through `ids`, the /// dictionary of the walk's destination space, charged to `memory` first. fn destination_column(dense: &[u32], ids: &[String], memory: &WorkMemory) -> Result { - let bytes = dense + let strings = dense .iter() - .map(|&d| ids[d as usize].len()) + .map(|&d| { + ids.get(d as usize).map(String::as_str).ok_or_else(|| { + OmniError::manifest_internal(format!( + "dense destination id {d} is outside its dictionary of {} ids", + ids.len() + )) + }) + }) + .collect::>>()?; + let bytes = strings + .iter() + .map(|id| id.len()) .fold(0usize, usize::saturating_add); memory.string(bytes).map_err(|error| memory.error(error))?; memory .grow((dense.len() + 1).saturating_mul(std::mem::size_of::())) .map_err(|error| memory.error(error))?; - Ok(Arc::new(StringArray::from_iter_values( - dense.iter().map(|&d| ids[d as usize].as_str()), - ))) + Ok(Arc::new(StringArray::from_iter_values(strings))) +} + +/// The pending chunk as pairs ready to emit: its destinations resolved +/// through `ids`, the dictionary of the source that found them, the buffers +/// and the emission charge swapped for fresh ones. +fn take_pending_pairs( + emitted_src: &mut Vec, + emitted_dst: &mut Vec, + emission_memory: &mut WorkMemory, + ids: &[String], + memory: &WorkMemory, + chunk_rows: usize, +) -> Result { + let next_emission = memory + .child("expand emissions") + .map_err(|error| memory.error(error))?; + next_emission + .entries::<(u32, u32)>(chunk_rows) + .map_err(|error| memory.error(error))?; + memory.metric("expand_pairs", emitted_src.len()); + let destination = destination_column(emitted_dst, ids, emission_memory)?; + emitted_dst.clear(); + Ok(ExpandedPairs { + source_rows: std::mem::replace(emitted_src, Vec::with_capacity(chunk_rows)), + destination, + _memory: std::mem::replace(emission_memory, next_emission), + }) } pub(super) struct PreparedEdge { @@ -1355,6 +1391,23 @@ where let gi = graph_index.get().await?.ok_or_else(|| { OmniError::manifest("graph index required for CSR traversal".to_string()) })?; + // The pending chunk's destinations are dense ids of the + // interner: they leave now, as strings, before the + // dictionaries change. A `Utf8` column has no id space. + if !emitted_src.is_empty() { + let ActiveExpandSource::Indexed(src) = &active else { + unreachable!("switch only fires while the Indexed source is active"); + }; + let pairs = take_pending_pairs( + &mut emitted_src, + &mut emitted_dst, + &mut emission_memory, + src.interner.ids(), + memory, + chunk_rows, + )?; + emit(pairs).await?; + } let csr_source = ActiveExpandSource::Csr(resolve_csr( gi, edge_def, @@ -1479,41 +1532,40 @@ where .map_err(|error| memory.error(error))?; } let is_self = same_type && hop == 1 && neighbor == node; - if !is_self && same_type && !visited.insert(source_node_key(i, neighbor)) { - continue; + if !is_self && same_type { + if !visited.insert(source_node_key(i, neighbor)) { + continue; + } + added = added.saturating_add(WorkMemory::entry_bytes::()); } if !is_self { next.push(neighbor); + added = added.saturating_add(WorkMemory::entry_bytes::()); } if hop >= min_hops && (neighbor as usize) < ids.len() && seen_dst.insert(source_node_key(i, neighbor)) { + added = added.saturating_add( + WorkMemory::entry_bytes::() + .saturating_add(WorkMemory::entry_bytes::<(u32, u32)>()), + ); emitted_src.push(i as u32); emitted_dst.push(neighbor); if emitted_src.len() >= chunk_rows { - let next_emission = memory - .child("expand emissions") - .map_err(|error| memory.error(error))?; - next_emission - .entries::<(u32, u32)>(chunk_rows) - .map_err(|error| memory.error(error))?; - memory.metric("expand_pairs", emitted_src.len()); - let destination = - destination_column(&emitted_dst, ids, &emission_memory)?; - emitted_dst.clear(); - emit(ExpandedPairs { - source_rows: std::mem::replace( - &mut emitted_src, - Vec::with_capacity(chunk_rows), - ), - destination, - _memory: std::mem::replace(&mut emission_memory, next_emission), - }) - .await?; + let pairs = take_pending_pairs( + &mut emitted_src, + &mut emitted_dst, + &mut emission_memory, + ids, + memory, + chunk_rows, + )?; + emit(pairs).await?; } } } + memory.shrink_to(before.saturating_add(added)); memory.check().map_err(|error| memory.error(error))?; } *frontier = next; diff --git a/crates/omnigraph/tests/integration/traversal_adaptive.rs b/crates/omnigraph/tests/integration/traversal_adaptive.rs index dc3f70ce3..bbfcf5766 100644 --- a/crates/omnigraph/tests/integration/traversal_adaptive.rs +++ b/crates/omnigraph/tests/integration/traversal_adaptive.rs @@ -131,6 +131,44 @@ async fn adaptive_switch_returns_identical_rows() { // The switch under an UNDIRECTED traversal (Direction::Both — two endpoint // probes per hop, the probe-factor case, CSR+CSC merge after the swap). Same // clique, so the hop-2 policy check sees the same explosive growth. +/// The same switch from a seed whose dense id differs between the walk's +/// interner and the graph index's dictionary: a hop-1 chunk still buffered +/// when hop 2 swaps the source must leave as the ids it was found under. +#[tokio::test] +async fn adaptive_switch_from_a_later_seed_returns_identical_rows() { + let dir = tempfile::tempdir().unwrap(); + let db = clique_db(&dir).await; + let p = params(&[("$name", "p5")]); + + let csr_db = with_traversal(&db, Traversal::Csr); + let indexed_db = with_traversal(&db, Traversal::Indexed); + let csr = first_column_sorted(&query_main(&csr_db, REACH_3, "reach", &p).await.unwrap()); + let indexed = + first_column_sorted(&query_main(&indexed_db, REACH_3, "reach", &p).await.unwrap()); + let switches = Arc::new(AtomicU64::new(0)); + let probes = QueryIoProbes { + traversal_mid_switches: Arc::clone(&switches), + ..Default::default() + }; + let auto = first_column_sorted( + &with_query_io_probes(probes, query_main(&db, REACH_3, "reach", &p)) + .await + .unwrap(), + ); + + assert_eq!(csr.len(), 49, "p5 reaches every other clique member"); + assert!(csr.iter().any(|name| name == "p0") && !csr.iter().any(|name| name == "p5")); + assert_eq!(indexed, csr, "forced modes agree"); + assert_eq!( + auto, csr, + "auto (dispatch + mid-traversal re-decision) must match the forced paths" + ); + assert!( + switches.load(Ordering::Relaxed) >= 1, + "the mid-traversal Indexed→CSR switch must actually fire on this shape" + ); +} + #[tokio::test] async fn adaptive_switch_undirected_returns_identical_rows() { const REACH_3_UNDIRECTED: &str = r#" From 611be12c961e299e1152057c54f4c74cdb46f78e Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 22:46:25 +0000 Subject: [PATCH 12/13] perf(engine): settle the walk's and the join's pool charges to what is live A node admits the bound for all its neighbours and settles to what the walk added once its slice is done; the join releases its hash buffer's share once the build table stands. The join's first-row rule under a hash collision and a duplicate id has a unit test, and the execution guide says what Lance 13 does with a pushed list per fragment: a clone, with the physical expression built once per distinct expression. --- crates/omnigraph/src/engine/graph.rs | 22 ++++-- .../src/engine/operators/hash_join.rs | 74 +++++++++++++------ .../omnigraph/src/engine/operators/memory.rs | 5 ++ docs/dev/execution.md | 9 ++- 4 files changed, 77 insertions(+), 33 deletions(-) diff --git a/crates/omnigraph/src/engine/graph.rs b/crates/omnigraph/src/engine/graph.rs index dfdba5eb2..7424eee91 100644 --- a/crates/omnigraph/src/engine/graph.rs +++ b/crates/omnigraph/src/engine/graph.rs @@ -1336,7 +1336,10 @@ where let mut emitted_dst: Vec = Vec::with_capacity(chunk_rows); let mut prev_union_len: usize = 0; // What one examined neighbour may add: a visited entry, a frontier - // entry, an emitted-set entry and a pair; charged per node, not per edge. + // entry, an emitted-set entry and a pair. A node admits that bound for + // all its neighbours before its slice is walked and settles to what the + // walk added once it is done: two pool calls per node, none per edge, + // and the held charge is the live state. let per_neighbour_bytes = WorkMemory::entry_bytes::() .saturating_mul(2) .saturating_add(WorkMemory::entry_bytes::()) @@ -1522,9 +1525,11 @@ where ActiveExpandSource::Csr(CsrSource { dst_idx, .. }) => dst_idx.ids(), }; // Admit what this node's neighbours can add before the walk. + let before = memory.charged(); memory .grow((fwd.len() + rev.len()).saturating_mul(per_neighbour_bytes)) .map_err(|error| memory.error(error))?; + let mut added = 0usize; for &neighbor in fwd.iter().chain(rev) { if budgeted { memory @@ -1814,7 +1819,7 @@ fn physical_scan_rows( } /// The longest key list an unindexed edge scan still pushes into Lance; see -/// the pushdown rule in [`scan_edges_stream`]. The indexed expand path's +/// the pushdown rule in `scan_edges_stream`. The indexed expand path's /// frontier ceiling (`OMNIGRAPH_EXPAND_INDEXED_MAX_FRONTIER`, 1,024 by /// default) is this size: a frontier the plan sends to per-hop scans stays /// pushed, a bound edge over a larger source set is sieved. @@ -1857,12 +1862,13 @@ async fn scan_edges_stream( ); // A `key IN (keys)` filter reaches Lance where a BTREE on the key column // turns it into index lookups, and where the list is small: Lance 13 - // re-plans a pushed list for every fragment (a clone and a hash set of - // every key per fragment), but a pushed filter also lets it read a - // fragment's other columns only for the rows the key column admits, one - // read fewer per fragment no key is in. Up to `PUSHDOWN_KEYS_MAX` keys - // the per-fragment replan is cheaper than that read; past it the engine - // sieves the batches itself through one set of the keys. + // clones a pushed list for every fragment it reads (`filtered_read.rs` + // clones `full_filter` per fragment; the physical expression is built + // once per distinct expression), but a pushed filter also lets it read + // a fragment's other columns only for the rows the key column admits, + // one read fewer per fragment no key is in. Up to `PUSHDOWN_KEYS_MAX` + // keys the per-fragment clone is cheaper than that read; past it the + // engine sieves the batches itself through one set of the keys. let pushdown = match pushdown { Some(true) => true, _ if keys.len() <= PUSHDOWN_KEYS_MAX => true, diff --git a/crates/omnigraph/src/engine/operators/hash_join.rs b/crates/omnigraph/src/engine/operators/hash_join.rs index e2399394d..a2abcaadc 100644 --- a/crates/omnigraph/src/engine/operators/hash_join.rs +++ b/crates/omnigraph/src/engine/operators/hash_join.rs @@ -145,12 +145,32 @@ struct Built { impl Built { /// The first build row holding `id`, hashed as `hash`. fn row_of(&self, hash: u64, id: &str) -> Option { - self.rows - .find(hash, |&(entry_hash, row)| { - entry_hash == hash && self.ids.value(row as usize) == id - }) - .map(|&(_, row)| row) + find_row(&self.rows, &self.ids, hash, id) + } +} + +/// The build table over `ids` hashed as `hashes`: one `(hash, row)` entry +/// per distinct non-null id, the first row of a duplicate kept; two ids +/// with one hash keep separate entries, told apart by their strings. +fn index_rows(ids: &StringArray, hashes: &[u64]) -> HashTable<(u64, u32)> { + let mut rows: HashTable<(u64, u32)> = HashTable::with_capacity(ids.len()); + for (row, &hash) in hashes.iter().enumerate() { + if ids.is_null(row) { + continue; + } + if find_row(&rows, ids, hash, ids.value(row)).is_none() { + rows.insert_unique(hash, (hash, row as u32), |&(entry_hash, _)| entry_hash); + } } + rows +} + +/// The build row of `id`, hashed as `hash`, in `rows` over `ids`. +fn find_row(rows: &HashTable<(u64, u32)>, ids: &StringArray, hash: u64, id: &str) -> Option { + rows.find(hash, |&(entry_hash, row)| { + entry_hash == hash && ids.value(row as usize) == id + }) + .map(|&(_, row)| row) } /// `hashes` filled with the hash of every value of `ids` (nulls included, @@ -183,28 +203,19 @@ async fn build_side( "destination scan exceeds row index range", )) })?; - // The table's entries plus the hash buffer, one `u64` per row. + // The table's entries plus the hash buffer, one `u64` per row; the + // buffer's share is released once the table is built. + let before = work.charged(); + let table_bytes = WorkMemory::entry_bytes::<(u64, u32)>().saturating_mul(ids.len()); work.entries::<(u64, u32)>(ids.len())?; work.entries::(ids.len())?; let state = RandomState::with_seed(HASH_SEED); let id_array: ArrayRef = Arc::new(ids.clone()); let mut hashes = Vec::new(); hash_ids(&id_array, &state, &mut hashes)?; - let mut rows: HashTable<(u64, u32)> = HashTable::with_capacity(ids.len()); - for (row, &hash) in hashes.iter().enumerate() { - if ids.is_null(row) { - continue; - } - let id = ids.value(row); - let occupied = rows - .find(hash, |&(entry_hash, entry_row)| { - entry_hash == hash && ids.value(entry_row as usize) == id - }) - .is_some(); - if !occupied { - rows.insert_unique(hash, (hash, row as u32), |&(entry_hash, _)| entry_hash); - } - } + let rows = index_rows(ids, &hashes); + drop(hashes); + work.shrink_to(before.saturating_add(table_bytes)); let ids = ids.clone(); Ok(Built { batch, @@ -429,3 +440,24 @@ impl Streamed for Probe { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// The build table keeps the first row of a duplicate id, tells two ids + /// with one hash apart by their strings, and never indexes a null. + #[test] + fn build_table_keeps_the_first_row_and_resolves_a_collision() { + let ids = StringArray::from(vec![Some("a"), Some("b"), Some("a"), None, Some("c")]); + // `a` and `b` collide; the null shares `c`'s hash and must stay out. + let hashes = [7, 7, 7, 9, 9]; + let rows = index_rows(&ids, &hashes); + assert_eq!(rows.len(), 3); + assert_eq!(find_row(&rows, &ids, 7, "a"), Some(0)); + assert_eq!(find_row(&rows, &ids, 7, "b"), Some(1)); + assert_eq!(find_row(&rows, &ids, 9, "c"), Some(4)); + assert_eq!(find_row(&rows, &ids, 7, "c"), None); + assert_eq!(find_row(&rows, &ids, 9, "a"), None); + } +} diff --git a/crates/omnigraph/src/engine/operators/memory.rs b/crates/omnigraph/src/engine/operators/memory.rs index 2ba073b48..e315dc31c 100644 --- a/crates/omnigraph/src/engine/operators/memory.rs +++ b/crates/omnigraph/src/engine/operators/memory.rs @@ -732,6 +732,11 @@ impl WorkMemory { /// Lower this work's charge to `bytes` once its peak has passed; a charge /// already at or under `bytes`, or freed meanwhile, stays as it is. + /// This work's current scratch charge, the number `shrink_to` lowers. + pub(in crate::engine) fn charged(&self) -> usize { + self.scratch.size() + } + pub(in crate::engine) fn shrink_to(&self, bytes: usize) { let excess = self.scratch.size().saturating_sub(bytes); if self.scratch.try_shrink(excess).is_ok() { diff --git a/docs/dev/execution.md b/docs/dev/execution.md index a251b0ae2..bdeffc8c0 100644 --- a/docs/dev/execution.md +++ b/docs/dev/execution.md @@ -79,10 +79,11 @@ not contain edge properties. Every edge probe (`graph.rs::scan_edges_stream`: the bound-edge producer, an indexed hop) pushes its `key IN (keys)` list into Lance when the key column has usable BTREE coverage, which turns it into index lookups, or when the list holds at most 1,024 keys; a longer list over -an unindexed column is not pushed, because Lance 13 clones and plans the -list again for every fragment, so that scan reads unfiltered and the engine -keeps the rows whose key is in one set of the keys (one more column read per -fragment no key is in, which only a long list pays for). It preserves incoming row/rank order and carries a +an unindexed column is not pushed, because Lance 13 clones the pushed list +for every fragment it reads (it builds the physical expression once per +distinct expression), so that scan reads unfiltered and the engine keeps the +rows whose key is in one set of the keys (one more column read per fragment +no key is in, which only a long list pays for). It preserves incoming row/rank order and carries a deterministic edge tie-break. A correlated block (`not { ... }`, `exists { ... }`, `count { ... } > 2`, `sum($d.size) { ... } > 100`) is one anti-join over its typed inner pipeline, keeping each outer row by a From 5a473f82e748fb30ac72b5b93b257ad259036fbf Mon Sep 17 00:00:00 2001 From: Ragnor Comerford Date: Tue, 3 Nov 2026 23:26:25 +0000 Subject: [PATCH 13/13] perf(engine): check a chunk's dense ids without collecting them The guarded destination column collected the chunk's strings into a vector before building the Arrow column, one allocation per chunk that the BFS counters showed as +10% bytes; one pass now checks the ids and sums the bytes, and the column is built straight from the dictionary. --- crates/omnigraph/src/engine/graph.rs | 31 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/crates/omnigraph/src/engine/graph.rs b/crates/omnigraph/src/engine/graph.rs index 7424eee91..75276b39a 100644 --- a/crates/omnigraph/src/engine/graph.rs +++ b/crates/omnigraph/src/engine/graph.rs @@ -493,26 +493,25 @@ fn source_node_key(source: usize, node: u32) -> u64 { /// The destination column of one chunk: `dense` resolved through `ids`, the /// dictionary of the walk's destination space, charged to `memory` first. fn destination_column(dense: &[u32], ids: &[String], memory: &WorkMemory) -> Result { - let strings = dense - .iter() - .map(|&d| { - ids.get(d as usize).map(String::as_str).ok_or_else(|| { - OmniError::manifest_internal(format!( - "dense destination id {d} is outside its dictionary of {} ids", - ids.len() - )) - }) - }) - .collect::>>()?; - let bytes = strings - .iter() - .map(|id| id.len()) - .fold(0usize, usize::saturating_add); + // One pass checks every id against the dictionary and sums the bytes; + // the column is then built straight from the dictionary. + let mut bytes = 0usize; + for &d in dense { + let id = ids.get(d as usize).ok_or_else(|| { + OmniError::manifest_internal(format!( + "dense destination id {d} is outside its dictionary of {} ids", + ids.len() + )) + })?; + bytes = bytes.saturating_add(id.len()); + } memory.string(bytes).map_err(|error| memory.error(error))?; memory .grow((dense.len() + 1).saturating_mul(std::mem::size_of::())) .map_err(|error| memory.error(error))?; - Ok(Arc::new(StringArray::from_iter_values(strings))) + Ok(Arc::new(StringArray::from_iter_values( + dense.iter().map(|&d| ids[d as usize].as_str()), + ))) } /// The pending chunk as pairs ready to emit: its destinations resolved