Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 0 additions & 42 deletions crates/omnigraph/src/changes/model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,22 +37,6 @@ pub const CHANGE_FEED_DEFAULT_COMMITS_PER_POLL: usize = 128;
/// Public ceiling on graph commits examined by one feed poll.
pub const CHANGE_FEED_MAX_COMMITS_PER_POLL: usize = 512;

/// Engine mirror of the compiler's reserved Lance virtual column list. The
/// compiler's `is_reserved_storage_system_column` is crate-private there, so
/// this exact five-name list is pinned by a unit test below. Matching is
/// exact: a legal user property such as `_rowdy` or `_row_custom` is never
/// hidden by a prefix heuristic.
pub(crate) fn is_reserved_storage_system_column(name: &str) -> bool {
matches!(
name,
"_rowid"
| "_rowaddr"
| "_rowoffset"
| "_row_created_at_version"
| "_row_last_updated_at_version"
)
}

/// Logical entity namespace of one change.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
#[serde(rename_all = "snake_case")]
Expand Down Expand Up @@ -356,32 +340,6 @@ pub struct ChangeBaseline {
mod tests {
use super::*;

/// Pins the exact five reserved Lance virtual column names. If Lance adds
/// a sixth, the compiler's reserved list and this mirror must move
/// together in one audited change.
#[test]
fn reserved_storage_system_columns_are_exactly_the_five_lance_names() {
for name in [
"_rowid",
"_rowaddr",
"_rowoffset",
"_row_created_at_version",
"_row_last_updated_at_version",
] {
assert!(is_reserved_storage_system_column(name), "{name}");
}
for name in [
"_row",
"_rowdy",
"_row_custom",
"_row_created_at",
"id",
"rowid",
] {
assert!(!is_reserved_storage_system_column(name), "{name}");
}
}

#[test]
fn operation_ranks_are_frozen() {
assert_eq!(ChangeOpKind::Insert.rank(), 0);
Expand Down
9 changes: 5 additions & 4 deletions crates/omnigraph/src/changes/row_compare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,12 @@ use std::collections::{HashMap, HashSet};
use arrow_array::{Array, RecordBatch, StringArray, StructArray, UInt64Array};
use datafusion::prelude::{Expr, col, lit};
use lance::Dataset;
use lance_core::is_system_column;
use lance_table::format::Fragment;

use omnigraph_compiler::{SYSTEM_COLUMNS_META, SystemColumns};

use super::model::{COMMIT_CHANGES_MAX_BYTES, is_reserved_storage_system_column};
use super::model::COMMIT_CHANGES_MAX_BYTES;
use crate::blob::{BlobDescriptor, BlobDescriptorDecoder};
use crate::db::{
LogicalBlobValue, RangedExternalBlobs, STABLE_PROPERTY_ID_METADATA_KEY, row_blob_values,
Expand All @@ -47,7 +48,7 @@ pub(crate) fn user_schema_fingerprint(
.schema()
.fields
.iter()
.filter(|field| !is_reserved_storage_system_column(&field.name))
.filter(|field| !is_system_column(&field.name))
.map(|field| {
(
column_name_at_vintage(&field.name, system_columns, SYSTEM_COLUMNS_META, is_edge)
Expand Down Expand Up @@ -198,7 +199,7 @@ impl BatchCursor {
.zip(batch.columns())
.enumerate()
{
if is_reserved_storage_system_column(field.name()) {
if is_system_column(field.name()) {
continue;
}
let lance_field = lance::datatypes::Field::try_from(field.as_ref())
Expand Down Expand Up @@ -545,7 +546,7 @@ async fn rows_equal_by_column(
.zip(left.slice.columns())
{
let name = field.name();
if is_reserved_storage_system_column(name) || blob_columns.contains(name.as_str()) {
if is_system_column(name) || blob_columns.contains(name.as_str()) {
continue;
}
let other = right
Expand Down
4 changes: 2 additions & 2 deletions crates/omnigraph/src/db/omnigraph/export.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use super::*;
use crate::changes::model::is_reserved_storage_system_column;
use crate::ordered_cursor::{KeyOrder, OrderedRowCursor, WalkSubject};
use futures::TryStreamExt;
use lance_core::is_system_column;
use omnigraph_compiler::SYSTEM_COLUMNS_META;
use std::future::Future;

Expand Down Expand Up @@ -901,7 +901,7 @@ pub(crate) async fn logical_row_image(
.schema()
.fields()
.iter()
.filter(|field| !is_reserved_storage_system_column(field.name()))
.filter(|field| !is_system_column(field.name()))
.map(|field| field.name().clone())
.collect::<Vec<_>>();
let lines = row_json_lines(&row_batch, &fields, blob_values.as_ref(), 0, id_col)?;
Expand Down
4 changes: 2 additions & 2 deletions crates/omnigraph/src/engine/push/compare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use super::chunk::{Chunk, prefixed, side_batch, with_column};
use super::context::{EngineHooks, ExecContext, SideRow, side_name};
use super::error::Result;
use super::roles::{Operator, OperatorResult};
use crate::changes::model::is_reserved_storage_system_column;
use lance_core::is_system_column;

use crate::changes::row_compare::column_name_at_vintage;

Expand Down Expand Up @@ -102,7 +102,7 @@ pub async fn rows_differ(
let Some(name) = field.name().strip_prefix(&prefix) else {
continue;
};
if is_reserved_storage_system_column(name) {
if is_system_column(name) {
continue;
}
if is_blob(field) {
Expand Down
10 changes: 1 addition & 9 deletions crates/omnigraph/src/loader/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2846,15 +2846,7 @@ fn validate_strict_input_fields(
}

fn is_reserved_physical_input_field(field: &str) -> bool {
matches!(
field,
"_tombstone"
| "_rowid"
| "_rowaddr"
| "_rowoffset"
| "_row_created_at_version"
| "_row_last_updated_at_version"
)
field == "_tombstone" || lance_core::is_system_column(field)
}

fn validate_optional_row_id(
Expand Down
13 changes: 13 additions & 0 deletions crates/omnigraph/tests/lance_surface_guards.rs
Original file line number Diff line number Diff line change
Expand Up @@ -706,6 +706,19 @@ fn compiler_rejects_five_surveyed_lance_virtual_system_columns() {
assert!(error.contains("reserved"), "unexpected error: {error}");
assert!(error.contains(name), "unexpected error: {error}");
}
// The engine drops exactly the names `is_system_column` matches from
// change images, export and push comparison. The match must stay exact,
// so a property a persisted contract may still declare is never hidden.
for name in [
"_row",
"_rowdy",
"_row_custom",
"_row_created_at",
"id",
"rowid",
] {
assert!(!is_system_column(name), "Lance now reserves {name}");
}
}

/// Guard: a rename-only `Operation::Project` (the RFC 0040 system-column
Expand Down
2 changes: 2 additions & 0 deletions docs/dev/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ and invariants rather than duplicating those details.

- [RFC process and registry](../rfcs/README.md)
- [Release notes](../releases/)
- [Simplification candidates, 2026-11-09](../simplify/20261109.md): code
ranked for removal, with what was taken and why the rest was not
- Git history for superseded implementation plans and handoff notes

An implemented RFC records why a decision was made; it does not replace the
Expand Down
174 changes: 174 additions & 0 deletions docs/simplify/20261109.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
# Simplification candidates, 2026-11-09

**Audience:** maintainers who review code-removal work
**Base:** `main` at `f0df46736c3598b6f70d30a6cb01abfa6320d788`

This report lists the candidates from one search for code that can be deleted
to lower maintenance cost. One candidate became a pull request. The others are
ranked below, with what was examined and why they were not taken.

Liability is code that a maintainer must read, keep correct and change
together. Fewer lines are evidence of less liability, not proof.

## Taken

### 1. One list of Lance's virtual columns, not three

Kind: a special case that has a general owner. It is also a hand-written copy
of a Lance function.

The Lance scanner adds five columns to a scan on request: `_rowid`,
`_rowaddr`, `_rowoffset`, `_row_created_at_version` and
`_row_last_updated_at_version`. Lance names them in one function,
`lance_core::is_system_column` (`lance-core-13.0.0/src/lib.rs:93-98`). Before
this change, the workspace had three more hand-written copies of the five
names:

| Copy | Use |
|---|---|
| `omnigraph-compiler` `schema::is_reserved_storage_system_column` | Refuse a property with one of these names |
| engine `changes::model::is_reserved_storage_system_column`, with its own test | Drop these columns from change images, the schema fingerprint, export and push comparison |
| engine `loader::is_reserved_physical_input_field` (the five names and `_tombstone`) | Refuse load input fields with these names |

The engine copy existed only because the compiler copy is private to its
crate. The comment on the engine copy said so. Its test said that a new Lance
column must go into the compiler list and into this copy "in one audited
change".

The engine uses the columns that Lance adds, so the Lance function is the
correct owner there. The change does three things:

1. It deletes the engine copy and its test. The five call sites call
`lance_core::is_system_column`.
2. It reduces the loader predicate to
`field == "_tombstone" || lance_core::is_system_column(field)`.
3. It moves the negative cases of the deleted test into the existing guard
`lance_surface_guards.rs::compiler_rejects_five_surveyed_lance_virtual_system_columns`.
The cases are names such as `_rowdy` and `_row_custom`, which are not
system columns. That guard already tests that Lance reserves the five names
and that the compiler refuses them.

The compiler keeps its copy. The compiler does not depend on Lance, and the
guard ties its list to the Lance list.

Liability removed: two copies of a list that had to change together by hand,
and one test that tested only a copy.

Liability added: five call sites name a `lance_core` function directly. The
engine already depends on `lance-core` and uses its column constants in other
places. If a Lance upgrade adds a sixth column, the engine now drops it
automatically, which is what the engine needs. The compiler copy still needs a
manual change, as before.

Behavior: unchanged. At the pinned Lance 13.0.0, the function matches exactly
the same five names, with no prefix rule.

Lines: the engine definitions and the test went from 52 lines to 3, and the
guard gained 13. The code change adds 23 lines and removes 59.

## Not taken

The list is ranked by liability removed against risk.

### 2. Unused direct dependencies

Kind: dead code (dependencies). Outcome: not taken in this run. It is a good
next candidate.

No Rust source in the declaring crate names these dependencies:

- `dashmap`, `tower-http` and the `lance-index` dev-dependency in
`omnigraph-server`
- The `lance-index` dev-dependency in `omnigraph-cli`
- `ahash`, `arrow-ord` and `arrow-cast` in `omnigraph-compiler`
- The `lance-namespace-impls` dev-dependency in `omnigraph-engine`
- `lance-core` in `omnigraph-dst`
- The `serial_test` dev-dependency in `omnigraph-core`

`cargo tree -i` shows that each one stays in the build through another crate.
Removal makes the manifests shorter but not the dependency tree, and it can
change feature unification. For that reason it ranks below the taken
candidate. `arrow-ipc` in `omnigraph-catalog` is left out, because open pull
requests change that crate.

### 3. A cost counter that nothing increments

Kind: dead code. Outcome: not taken. A reviewer must agree on what the
remaining limits cover.

`omnigraph-core` `instrumentation::record_projection_identity_rows` has no
caller. #863 removed its only call. The counter it feeds stays at zero, so the
assertions on it in `tests/integration/merge_projection_cache.rs` always pass.
Removal deletes test assertions, so a reviewer must agree that they test
nothing.

### 4. Small unused items

Kind: dead code. Outcome: not taken. The search reported these items, and no
build has tested their removal yet.

The items are `engine::push::hydrate::hydrated_columns`,
`MutationStaging::pending_row_count` (`#[allow(dead_code)]`), the test-only
`loader::preflight_strict_row_arrow_bytes` shim, and the `PreparedClone`
alias in `omnigraph-bench`. They are about 50 lines.

### 5. Hand-written hex helpers

Kind: bloat. Outcome: not taken.

Two hex-digit parsers (`blob.rs`, `omnigraph-storage`) can use
`char::to_digit(16)`. Two SHA-256 hex encoders (`omnigraph-storage`,
`omnigraph-cli` `skill.rs`) can use `format!("{:x}", Sha256::digest(..))`, as
the server already does. They are about 25 lines.

### 6. Copies in the CLI of functions that exist in other crates

Kind: special cases. Outcome: not taken.

The test-only `normalize_bearer_token` in `omnigraph-cli` copies the server
function and tests only itself. `graph_type_subject` in the CLI copies the
function in `omnigraph-core`. The CLI and the server spell
`EntityKind::as_str` and the API-to-engine `EntityKind` mapping inline in
several places. They are about 50 lines in total.

### 7. The `JavaScript` mode of `JsonParamMode`

Kind: dead code. Outcome: not taken, because it needs a design decision.

No production caller passes `JsonParamMode::JavaScript`. Only one in-source
test uses it. Removal deletes about 150 lines in
`omnigraph-compiler/src/query_input.rs`. But the enum is public in a crate
that is meant for publication, so removal is an API decision.

### 8. The retired benchmark pipeline

Kind: dead code. Outcome: not taken, because it needs a maintainer decision.

`omnigraph-bench` keeps `worker.rs`, `supervisor.rs`, `fixture_worker.rs` and
`real_graph_run.rs` (about 5,400 lines) behind `#[cfg(test)]`. #894 removed
the subcommands that they start. The
[testing guide](../dev/testing.md#cost-tests-and-benchmarks) still names the
retired runner as a test oracle, so the maintainers must decide its removal.

### 9. A hand-written list-literal builder

Kind: bloat. Outcome: not taken, because the risk is medium.

`exec/mutation.rs` `typed_list_literal_to_array` repeats one builder loop for
ten element types (about 120 lines). DataFusion
`ScalarValue::new_list(..).to_array_of_size(..)` can replace it. But the
change must keep the error text, the zero-row case and the order of errors
exactly.

### Examined and rejected

- The single-batch shortcuts before `concat_batches` are not redundant. With
one batch, `concat_batches` can panic where `RecordBatch::try_new` returns
an error.
- `make_list_value_builder` in the loader is not the Arrow `make_builder`. The
buffer sizes and the error for an unsupported element type are different.
- The `SnapshotScanner` pass-through methods are a sealing boundary, not a
layer that only forwards calls.
- The search skipped candidates in files that open pull requests change, for
example `schema_for_table_key` in `exec/merge.rs` and
`normalize_branch_name`.
Loading