Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/export-hydration-keyset-cpu.performance.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Export copies each ordered chunk once instead of twice, hydration sizes each row once, and `not { }` blocks, bound edges and hash joins hash keys in one pass. Results are unchanged.
56 changes: 44 additions & 12 deletions crates/omnigraph/src/engine/graph.rs
Original file line number Diff line number Diff line change
Expand Up @@ -931,26 +931,52 @@ where
.ok_or_else(|| {
OmniError::manifest(format!("wide batch has no Utf8 '{src_name}' column"))
})?;
u32::try_from(src_ids.len())
.map_err(|_| OmniError::manifest_internal("expand source ordinal exceeds UInt32"))?;
// A bound source is a node the frontier holds, and a node's id is never
// null: a null here is an integrity fault, refused rather than dropped.
if src_ids.null_count() > 0 {
return Err(OmniError::manifest_internal(format!(
"bound expand frontier has a null '{src_name}'"
)));
}
// The source groups, each group's wide rows (`group_starts` into
// `group_rows`, in row order) and one owned key per group.
super::operators::KeyGroups::charge(&source_memory, src_ids.len())
.map_err(|error| memory.error(error))?;
source_memory
.entries::<(&str, Vec<u32>, u32, String)>(src_ids.len())
.entries::<(u32, u32, String)>(src_ids.len())
.map_err(|error| memory.error(error))?;
source_memory
.string(src_ids.value_data().len())
.map_err(|error| memory.error(error))?;
let mut rows_by_src: HashMap<&str, Vec<u32>> = HashMap::new();
source_memory
.charge_traversal(src_ids.len() as u64)
.map_err(|error| memory.error(error))?;
for row in 0..src_ids.len() {
source_memory.check().map_err(|error| memory.error(error))?;
let ordinal = u32::try_from(row)
.map_err(|_| OmniError::manifest_internal("expand source ordinal exceeds UInt32"))?;
rows_by_src
.entry(src_ids.value(row))
.or_default()
.push(ordinal);
source_memory.check().map_err(|error| memory.error(error))?;
let (src_groups, row_groups) = super::operators::KeyGroups::build_numbered(src_ids)
.map_err(|error| memory.error(error))?;
let mut group_starts = vec![0u32; src_groups.len() + 1];
for group in row_groups.iter().flatten() {
group_starts[*group as usize + 1] += 1;
}
for group in 0..src_groups.len() {
group_starts[group + 1] += group_starts[group];
}
let keys: Vec<String> = rows_by_src.keys().map(|key| (*key).to_owned()).collect();
let mut group_rows = vec![0u32; group_starts[src_groups.len()] as usize];
let mut next = group_starts.clone();
for (row, group) in row_groups.iter().enumerate() {
if let Some(group) = group {
let slot = &mut next[*group as usize];
group_rows[*slot as usize] = row as u32;
*slot += 1;
}
}
drop((row_groups, next));
let keys: Vec<String> = (0..src_groups.len() as u32)
.map(|group| src_groups.key(group).to_owned())
.collect();
let (mut probe_hashes, mut probe_groups) = (Vec::new(), Vec::new());
let attach_columns: Vec<&str> = schema.fields()[2..]
.iter()
.map(|field| field.name().as_str())
Expand Down Expand Up @@ -1072,16 +1098,22 @@ where
.child("bound edge pairs")
.map_err(|error| memory.error(error))?,
);
src_groups
.probe(keys, &mut probe_hashes, &mut probe_groups)
.map_err(|error| memory.error(error))?;
for (row, attached_bytes) in attach_bytes.into_iter().enumerate() {
memory
.charge_traversal(1)
.map_err(|error| memory.error(error))?;
if probe == 1 && keys.value(row) == opposites.value(row) {
continue;
}
let Some(wide_rows) = rows_by_src.get(keys.value(row)) else {
let Some(group) = probe_groups[row] else {
continue;
};
let group = group as usize;
let wide_rows =
&group_rows[group_starts[group] as usize..group_starts[group + 1] as usize];
let edge_row = u32::try_from(row).map_err(|_| {
OmniError::manifest_internal("edge batch ordinal exceeds UInt32")
})?;
Expand Down
26 changes: 8 additions & 18 deletions crates/omnigraph/src/engine/operators/anti_join.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
//! they stream, and each outer row takes its key's verdict (`key_set_mask`).

use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet};
use std::collections::HashMap;
use std::fmt;
use std::sync::{Arc, Mutex};

Expand All @@ -27,6 +26,7 @@ use omnigraph_compiler::ir::{ParamMap, SubqueryPredicate};
use omnigraph_compiler::types::Direction;
use omnigraph_planner::{AggregateSpec, AntiJoinStrategy};

use super::key_groups::KeyGroups;
use super::memory::WorkMemory;
use super::shape::{Breaker, work};
use super::subquery_aggregate::{
Expand Down Expand Up @@ -406,16 +406,8 @@ async fn key_set_mask(
) -> DfResult<BooleanArray> {
let outer_keys = key_array(wide, key_column).map_err(external)?;
let num_rows = wide.num_rows();
reservation.entries::<(&str, u32)>(num_rows)?;
reservation.entries::<u32>(num_rows)?;
let mut groups: HashMap<&str, u32> = HashMap::new();
let mut row_groups = Vec::with_capacity(num_rows);
for row in 0..num_rows {
row_groups.push(outer_keys.is_valid(row).then(|| {
let next = groups.len() as u32;
*groups.entry(outer_keys.value(row)).or_insert(next)
}));
}
KeyGroups::charge(reservation, num_rows)?;
let (groups, row_groups) = KeyGroups::build_numbered(outer_keys)?;
let unmatched = groups.len() as u32;
let discarded = unmatched + 1;
let group_count = groups.len() + 2;
Expand All @@ -425,20 +417,18 @@ async fn key_set_mask(
}
let mut aggregate =
SubqueryAggregate::new(predicate, aggregate_spec, params, group_count).map_err(external)?;
let (mut hashes, mut inner_groups) = (Vec::new(), Vec::new());
while let Some(batch) = inner.next().await {
let batch = batch?;
reservation.metric("input_rows", batch.num_rows());
if batch.num_rows() == 0 {
continue;
}
let inner_keys = key_array(&batch, key_column).map_err(external)?;
let tags = UInt32Array::from_iter_values((0..batch.num_rows()).map(|row| {
inner_keys
.is_valid(row)
.then(|| groups.get(inner_keys.value(row)).copied())
.flatten()
.unwrap_or(discarded)
}));
groups.probe(inner_keys, &mut hashes, &mut inner_groups)?;
let tags = UInt32Array::from_iter_values(
inner_groups.iter().map(|group| group.unwrap_or(discarded)),
);
let values = predicate
.left
.arg()
Expand Down
48 changes: 48 additions & 0 deletions crates/omnigraph/src/engine/operators/expand_stream/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -553,3 +553,51 @@ async fn bound_pairs_size_a_list_property_by_its_own_elements_issue_1251() {
[EDGES]
);
}

/// A bound frontier holds nodes, whose ids are never null, so a null
/// source id is an integrity fault: the pairs refuse it instead of dropping
/// the row, and emit nothing.
#[tokio::test]
async fn bound_pairs_refuse_a_null_source_id() {
let (_dir, env) = graph_environment_for(
"node Person { name: String @key }\nedge Knows: Person -> Person",
people(),
)
.await;
let ctx = QueryContext::new(64 * 1024 * 1024).unwrap();
let memory = Arc::new(WorkMemory::new(ctx.task_ctx(), "bound pair nulls").unwrap());
let member = EdgeMember {
edge_type: "Knows".into(),
direction: Direction::Out,
};
let schema =
bound_edge_pair_schema(&env.catalog, std::slice::from_ref(&member), false).unwrap();
let wide = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new(
"p.__id",
DataType::Utf8,
true,
)])),
vec![Arc::new(StringArray::from(vec![Some("a"), None]))],
)
.unwrap();
let mut emitted = 0;
let error = produce_bound_edge_pairs(
&wide,
&env.snapshot,
&env.catalog,
"p",
&member,
None,
&schema,
&memory,
|_batch, _lease| {
emitted += 1;
std::future::ready(Ok(()))
},
)
.await
.unwrap_err();
assert!(error.to_string().contains("null 'p.__id'"), "{error}");
assert_eq!(emitted, 0);
}
107 changes: 15 additions & 92 deletions crates/omnigraph/src/engine/operators/hash_join.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@
use std::fmt;
use std::sync::Arc;

use arrow_array::{Array, ArrayRef, RecordBatch, StringArray, UInt32Array};
use arrow_array::{Array, RecordBatch, StringArray, UInt32Array};
use arrow_schema::SchemaRef;
use datafusion::common::hash_utils::{RandomState, create_hashes};
use datafusion::common::{DataFusionError, Result as DfResult};
use datafusion::execution::TaskContext;
use datafusion::physical_plan::metrics::{
Expand All @@ -23,11 +22,11 @@ use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream,
};
use futures::future::BoxFuture;
use hashbrown::HashTable;
use omnigraph_compiler::catalog::Catalog;
use omnigraph_compiler::ir::{IRExpr, ParamMap};
use omnigraph_planner::AccessPath;

use super::key_groups::KeyGroups;
use super::memory::WorkMemory;
use super::scan::Lookup;
use super::shape::{BatchSender, CollectedSide, Flow, Streamed, Streaming, work};
Expand Down Expand Up @@ -125,63 +124,16 @@ impl DisplayAs for HashJoinExec {
}
}

/// The seed of the join's hash function: fixed, so a build hashes the same
/// way on every run.
const HASH_SEED: u64 = 0x6f6d_6e69_6a6f_696e;

/// The drained build side: its rows and, per distinct non-null id, the first
/// row holding it, as `(hash, row)` entries in a table that borrows nothing
/// (an entry compares through the id column of `batch`), held under
/// `memory` for as long as the probe runs. The ids are hashed in bulk with
/// DataFusion's `create_hashes`; no id is copied out of the Arrow column.
/// row holding it, as the [`KeyGroups`] of its id column (one vectorized
/// `create_hashes` pass, no id copied out of the Arrow column), held under
/// `memory` for as long as the probe runs.
struct Built {
batch: RecordBatch,
ids: StringArray,
rows: HashTable<(u64, u32)>,
state: RandomState,
ids: KeyGroups,
_memory: WorkMemory,
}

impl Built {
/// The first build row holding `id`, hashed as `hash`.
fn row_of(&self, hash: u64, id: &str) -> Option<u32> {
find_row(&self.rows, &self.ids, hash, id)
}
}

/// The build table over `ids` hashed as `hashes`: one `(hash, row)` entry
/// per distinct non-null id, the first row of a duplicate kept; two ids
/// with one hash keep separate entries, told apart by their strings.
fn index_rows(ids: &StringArray, hashes: &[u64]) -> HashTable<(u64, u32)> {
let mut rows: HashTable<(u64, u32)> = HashTable::with_capacity(ids.len());
for (row, &hash) in hashes.iter().enumerate() {
if ids.is_null(row) {
continue;
}
if find_row(&rows, ids, hash, ids.value(row)).is_none() {
rows.insert_unique(hash, (hash, row as u32), |&(entry_hash, _)| entry_hash);
}
}
rows
}

/// The build row of `id`, hashed as `hash`, in `rows` over `ids`.
fn find_row(rows: &HashTable<(u64, u32)>, ids: &StringArray, hash: u64, id: &str) -> Option<u32> {
rows.find(hash, |&(entry_hash, row)| {
entry_hash == hash && ids.value(row as usize) == id
})
.map(|&(_, row)| row)
}

/// `hashes` filled with the hash of every value of `ids` (nulls included,
/// which the callers skip) under `state`.
fn hash_ids(ids: &ArrayRef, state: &RandomState, hashes: &mut Vec<u64>) -> DfResult<()> {
hashes.clear();
hashes.resize(ids.len(), 0);
create_hashes([ids], state, hashes)?;
Ok(())
}

async fn build_side(
build: &Arc<dyn ExecutionPlan>,
id_column: &str,
Expand All @@ -203,25 +155,17 @@ async fn build_side(
"destination scan exceeds row index range",
))
})?;
// The table's entries plus the hash buffer, one `u64` per row; the
// buffer's share is released once the table is built.
// The groups plus the hash buffer, one `u64` per row; the buffer's share
// is released once the groups are built.
let before = work.charged();
let table_bytes = WorkMemory::entry_bytes::<(u64, u32)>().saturating_mul(ids.len());
work.entries::<(u64, u32)>(ids.len())?;
let held = KeyGroups::group_bytes(ids.len());
work.grow(held)?;
work.entries::<u64>(ids.len())?;
let state = RandomState::with_seed(HASH_SEED);
let id_array: ArrayRef = Arc::new(ids.clone());
let mut hashes = Vec::new();
hash_ids(&id_array, &state, &mut hashes)?;
let rows = index_rows(ids, &hashes);
drop(hashes);
work.shrink_to(before.saturating_add(table_bytes));
let ids = ids.clone();
let ids = KeyGroups::build(ids)?;
work.shrink_to(before.saturating_add(held));
Ok(Built {
batch,
ids,
rows,
state,
_memory: work,
})
}
Expand Down Expand Up @@ -380,16 +324,16 @@ impl Probe {
))
})?;
let mut hashes = Vec::new();
hash_ids(batch.column(id_index), &built.state, &mut hashes)?;
built.ids.hash(ids, &mut hashes)?;
let mut probe_rows = Vec::with_capacity(ids.len());
let mut build_rows = Vec::with_capacity(ids.len());
for (row, &hash) in hashes.iter().enumerate() {
if ids.is_null(row) {
continue;
}
if let Some(build_row) = built.row_of(hash, ids.value(row)) {
if let Some(group) = built.ids.group_of(hash, ids.value(row)) {
probe_rows.push(row as u32);
build_rows.push(build_row);
build_rows.push(built.ids.first_row(group));
}
}
if probe_rows.is_empty() {
Expand Down Expand Up @@ -440,24 +384,3 @@ impl Streamed for Probe {
})
}
}

#[cfg(test)]
mod tests {
use super::*;

/// The build table keeps the first row of a duplicate id, tells two ids
/// with one hash apart by their strings, and never indexes a null.
#[test]
fn build_table_keeps_the_first_row_and_resolves_a_collision() {
let ids = StringArray::from(vec![Some("a"), Some("b"), Some("a"), None, Some("c")]);
// `a` and `b` collide; the null shares `c`'s hash and must stay out.
let hashes = [7, 7, 7, 9, 9];
let rows = index_rows(&ids, &hashes);
assert_eq!(rows.len(), 3);
assert_eq!(find_row(&rows, &ids, 7, "a"), Some(0));
assert_eq!(find_row(&rows, &ids, 7, "b"), Some(1));
assert_eq!(find_row(&rows, &ids, 9, "c"), Some(4));
assert_eq!(find_row(&rows, &ids, 7, "c"), None);
assert_eq!(find_row(&rows, &ids, 9, "a"), None);
}
}
Loading
Loading