diff --git a/crates/omnigraph/src/changes/model.rs b/crates/omnigraph/src/changes/model.rs index 8f4f0080a..12267490b 100644 --- a/crates/omnigraph/src/changes/model.rs +++ b/crates/omnigraph/src/changes/model.rs @@ -37,22 +37,6 @@ pub const CHANGE_FEED_DEFAULT_COMMITS_PER_POLL: usize = 128; /// Public ceiling on graph commits examined by one feed poll. pub const CHANGE_FEED_MAX_COMMITS_PER_POLL: usize = 512; -/// Engine mirror of the compiler's reserved Lance virtual column list. The -/// compiler's `is_reserved_storage_system_column` is crate-private there, so -/// this exact five-name list is pinned by a unit test below. Matching is -/// exact: a legal user property such as `_rowdy` or `_row_custom` is never -/// hidden by a prefix heuristic. -pub(crate) fn is_reserved_storage_system_column(name: &str) -> bool { - matches!( - name, - "_rowid" - | "_rowaddr" - | "_rowoffset" - | "_row_created_at_version" - | "_row_last_updated_at_version" - ) -} - /// Logical entity namespace of one change. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)] #[serde(rename_all = "snake_case")] @@ -356,32 +340,6 @@ pub struct ChangeBaseline { mod tests { use super::*; - /// Pins the exact five reserved Lance virtual column names. If Lance adds - /// a sixth, the compiler's reserved list and this mirror must move - /// together in one audited change. - #[test] - fn reserved_storage_system_columns_are_exactly_the_five_lance_names() { - for name in [ - "_rowid", - "_rowaddr", - "_rowoffset", - "_row_created_at_version", - "_row_last_updated_at_version", - ] { - assert!(is_reserved_storage_system_column(name), "{name}"); - } - for name in [ - "_row", - "_rowdy", - "_row_custom", - "_row_created_at", - "id", - "rowid", - ] { - assert!(!is_reserved_storage_system_column(name), "{name}"); - } - } - #[test] fn operation_ranks_are_frozen() { assert_eq!(ChangeOpKind::Insert.rank(), 0); diff --git a/crates/omnigraph/src/changes/row_compare.rs b/crates/omnigraph/src/changes/row_compare.rs index e43316528..5a2c07796 100644 --- a/crates/omnigraph/src/changes/row_compare.rs +++ b/crates/omnigraph/src/changes/row_compare.rs @@ -17,11 +17,12 @@ use std::collections::{HashMap, HashSet}; use arrow_array::{Array, RecordBatch, StringArray, StructArray, UInt64Array}; use datafusion::prelude::{Expr, col, lit}; use lance::Dataset; +use lance_core::is_system_column; use lance_table::format::Fragment; use omnigraph_compiler::{SYSTEM_COLUMNS_META, SystemColumns}; -use super::model::{COMMIT_CHANGES_MAX_BYTES, is_reserved_storage_system_column}; +use super::model::COMMIT_CHANGES_MAX_BYTES; use crate::blob::{BlobDescriptor, BlobDescriptorDecoder}; use crate::db::{ LogicalBlobValue, RangedExternalBlobs, STABLE_PROPERTY_ID_METADATA_KEY, row_blob_values, @@ -47,7 +48,7 @@ pub(crate) fn user_schema_fingerprint( .schema() .fields .iter() - .filter(|field| !is_reserved_storage_system_column(&field.name)) + .filter(|field| !is_system_column(&field.name)) .map(|field| { ( column_name_at_vintage(&field.name, system_columns, SYSTEM_COLUMNS_META, is_edge) @@ -198,7 +199,7 @@ impl BatchCursor { .zip(batch.columns()) .enumerate() { - if is_reserved_storage_system_column(field.name()) { + if is_system_column(field.name()) { continue; } let lance_field = lance::datatypes::Field::try_from(field.as_ref()) @@ -545,7 +546,7 @@ async fn rows_equal_by_column( .zip(left.slice.columns()) { let name = field.name(); - if is_reserved_storage_system_column(name) || blob_columns.contains(name.as_str()) { + if is_system_column(name) || blob_columns.contains(name.as_str()) { continue; } let other = right diff --git a/crates/omnigraph/src/db/omnigraph/export.rs b/crates/omnigraph/src/db/omnigraph/export.rs index b45603081..fac46fbb9 100644 --- a/crates/omnigraph/src/db/omnigraph/export.rs +++ b/crates/omnigraph/src/db/omnigraph/export.rs @@ -1,7 +1,7 @@ use super::*; -use crate::changes::model::is_reserved_storage_system_column; use crate::ordered_cursor::{KeyOrder, OrderedRowCursor, WalkSubject}; use futures::TryStreamExt; +use lance_core::is_system_column; use omnigraph_compiler::SYSTEM_COLUMNS_META; use std::future::Future; @@ -901,7 +901,7 @@ pub(crate) async fn logical_row_image( .schema() .fields() .iter() - .filter(|field| !is_reserved_storage_system_column(field.name())) + .filter(|field| !is_system_column(field.name())) .map(|field| field.name().clone()) .collect::>(); let lines = row_json_lines(&row_batch, &fields, blob_values.as_ref(), 0, id_col)?; diff --git a/crates/omnigraph/src/engine/push/compare.rs b/crates/omnigraph/src/engine/push/compare.rs index ad6dabb85..202eb22ef 100644 --- a/crates/omnigraph/src/engine/push/compare.rs +++ b/crates/omnigraph/src/engine/push/compare.rs @@ -15,7 +15,7 @@ use super::chunk::{Chunk, prefixed, side_batch, with_column}; use super::context::{EngineHooks, ExecContext, SideRow, side_name}; use super::error::Result; use super::roles::{Operator, OperatorResult}; -use crate::changes::model::is_reserved_storage_system_column; +use lance_core::is_system_column; use crate::changes::row_compare::column_name_at_vintage; @@ -102,7 +102,7 @@ pub async fn rows_differ( let Some(name) = field.name().strip_prefix(&prefix) else { continue; }; - if is_reserved_storage_system_column(name) { + if is_system_column(name) { continue; } if is_blob(field) { diff --git a/crates/omnigraph/src/loader/mod.rs b/crates/omnigraph/src/loader/mod.rs index 24f222087..7bf1e03d7 100644 --- a/crates/omnigraph/src/loader/mod.rs +++ b/crates/omnigraph/src/loader/mod.rs @@ -2846,15 +2846,7 @@ fn validate_strict_input_fields( } fn is_reserved_physical_input_field(field: &str) -> bool { - matches!( - field, - "_tombstone" - | "_rowid" - | "_rowaddr" - | "_rowoffset" - | "_row_created_at_version" - | "_row_last_updated_at_version" - ) + field == "_tombstone" || lance_core::is_system_column(field) } fn validate_optional_row_id( diff --git a/crates/omnigraph/tests/lance_surface_guards.rs b/crates/omnigraph/tests/lance_surface_guards.rs index f0f4fe817..48c8984ed 100644 --- a/crates/omnigraph/tests/lance_surface_guards.rs +++ b/crates/omnigraph/tests/lance_surface_guards.rs @@ -706,6 +706,19 @@ fn compiler_rejects_five_surveyed_lance_virtual_system_columns() { assert!(error.contains("reserved"), "unexpected error: {error}"); assert!(error.contains(name), "unexpected error: {error}"); } + // The engine drops exactly the names `is_system_column` matches from + // change images, export and push comparison. The match must stay exact, + // so a property a persisted contract may still declare is never hidden. + for name in [ + "_row", + "_rowdy", + "_row_custom", + "_row_created_at", + "id", + "rowid", + ] { + assert!(!is_system_column(name), "Lance now reserves {name}"); + } } /// Guard: a rename-only `Operation::Project` (the RFC 0040 system-column diff --git a/docs/dev/index.md b/docs/dev/index.md index f4bd669f2..c53dd9d21 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -47,6 +47,8 @@ and invariants rather than duplicating those details. - [RFC process and registry](../rfcs/README.md) - [Release notes](../releases/) +- [Simplification candidates, 2026-11-09](../simplify/20261109.md): code + ranked for removal, with what was taken and why the rest was not - Git history for superseded implementation plans and handoff notes An implemented RFC records why a decision was made; it does not replace the diff --git a/docs/simplify/20261109.md b/docs/simplify/20261109.md new file mode 100644 index 000000000..fd2a31d86 --- /dev/null +++ b/docs/simplify/20261109.md @@ -0,0 +1,174 @@ +# Simplification candidates, 2026-11-09 + +**Audience:** maintainers who review code-removal work +**Base:** `main` at `f0df46736c3598b6f70d30a6cb01abfa6320d788` + +This report lists the candidates from one search for code that can be deleted +to lower maintenance cost. One candidate became a pull request. The others are +ranked below, with what was examined and why they were not taken. + +Liability is code that a maintainer must read, keep correct and change +together. Fewer lines are evidence of less liability, not proof. + +## Taken + +### 1. One list of Lance's virtual columns, not three + +Kind: a special case that has a general owner. It is also a hand-written copy +of a Lance function. + +The Lance scanner adds five columns to a scan on request: `_rowid`, +`_rowaddr`, `_rowoffset`, `_row_created_at_version` and +`_row_last_updated_at_version`. Lance names them in one function, +`lance_core::is_system_column` (`lance-core-13.0.0/src/lib.rs:93-98`). Before +this change, the workspace had three more hand-written copies of the five +names: + +| Copy | Use | +|---|---| +| `omnigraph-compiler` `schema::is_reserved_storage_system_column` | Refuse a property with one of these names | +| engine `changes::model::is_reserved_storage_system_column`, with its own test | Drop these columns from change images, the schema fingerprint, export and push comparison | +| engine `loader::is_reserved_physical_input_field` (the five names and `_tombstone`) | Refuse load input fields with these names | + +The engine copy existed only because the compiler copy is private to its +crate. The comment on the engine copy said so. Its test said that a new Lance +column must go into the compiler list and into this copy "in one audited +change". + +The engine uses the columns that Lance adds, so the Lance function is the +correct owner there. The change does three things: + +1. It deletes the engine copy and its test. The five call sites call + `lance_core::is_system_column`. +2. It reduces the loader predicate to + `field == "_tombstone" || lance_core::is_system_column(field)`. +3. It moves the negative cases of the deleted test into the existing guard + `lance_surface_guards.rs::compiler_rejects_five_surveyed_lance_virtual_system_columns`. + The cases are names such as `_rowdy` and `_row_custom`, which are not + system columns. That guard already tests that Lance reserves the five names + and that the compiler refuses them. + +The compiler keeps its copy. The compiler does not depend on Lance, and the +guard ties its list to the Lance list. + +Liability removed: two copies of a list that had to change together by hand, +and one test that tested only a copy. + +Liability added: five call sites name a `lance_core` function directly. The +engine already depends on `lance-core` and uses its column constants in other +places. If a Lance upgrade adds a sixth column, the engine now drops it +automatically, which is what the engine needs. The compiler copy still needs a +manual change, as before. + +Behavior: unchanged. At the pinned Lance 13.0.0, the function matches exactly +the same five names, with no prefix rule. + +Lines: the engine definitions and the test went from 52 lines to 3, and the +guard gained 13. The code change adds 23 lines and removes 59. + +## Not taken + +The list is ranked by liability removed against risk. + +### 2. Unused direct dependencies + +Kind: dead code (dependencies). Outcome: not taken in this run. It is a good +next candidate. + +No Rust source in the declaring crate names these dependencies: + +- `dashmap`, `tower-http` and the `lance-index` dev-dependency in + `omnigraph-server` +- The `lance-index` dev-dependency in `omnigraph-cli` +- `ahash`, `arrow-ord` and `arrow-cast` in `omnigraph-compiler` +- The `lance-namespace-impls` dev-dependency in `omnigraph-engine` +- `lance-core` in `omnigraph-dst` +- The `serial_test` dev-dependency in `omnigraph-core` + +`cargo tree -i` shows that each one stays in the build through another crate. +Removal makes the manifests shorter but not the dependency tree, and it can +change feature unification. For that reason it ranks below the taken +candidate. `arrow-ipc` in `omnigraph-catalog` is left out, because open pull +requests change that crate. + +### 3. A cost counter that nothing increments + +Kind: dead code. Outcome: not taken. A reviewer must agree on what the +remaining limits cover. + +`omnigraph-core` `instrumentation::record_projection_identity_rows` has no +caller. #863 removed its only call. The counter it feeds stays at zero, so the +assertions on it in `tests/integration/merge_projection_cache.rs` always pass. +Removal deletes test assertions, so a reviewer must agree that they test +nothing. + +### 4. Small unused items + +Kind: dead code. Outcome: not taken. The search reported these items, and no +build has tested their removal yet. + +The items are `engine::push::hydrate::hydrated_columns`, +`MutationStaging::pending_row_count` (`#[allow(dead_code)]`), the test-only +`loader::preflight_strict_row_arrow_bytes` shim, and the `PreparedClone` +alias in `omnigraph-bench`. They are about 50 lines. + +### 5. Hand-written hex helpers + +Kind: bloat. Outcome: not taken. + +Two hex-digit parsers (`blob.rs`, `omnigraph-storage`) can use +`char::to_digit(16)`. Two SHA-256 hex encoders (`omnigraph-storage`, +`omnigraph-cli` `skill.rs`) can use `format!("{:x}", Sha256::digest(..))`, as +the server already does. They are about 25 lines. + +### 6. Copies in the CLI of functions that exist in other crates + +Kind: special cases. Outcome: not taken. + +The test-only `normalize_bearer_token` in `omnigraph-cli` copies the server +function and tests only itself. `graph_type_subject` in the CLI copies the +function in `omnigraph-core`. The CLI and the server spell +`EntityKind::as_str` and the API-to-engine `EntityKind` mapping inline in +several places. They are about 50 lines in total. + +### 7. The `JavaScript` mode of `JsonParamMode` + +Kind: dead code. Outcome: not taken, because it needs a design decision. + +No production caller passes `JsonParamMode::JavaScript`. Only one in-source +test uses it. Removal deletes about 150 lines in +`omnigraph-compiler/src/query_input.rs`. But the enum is public in a crate +that is meant for publication, so removal is an API decision. + +### 8. The retired benchmark pipeline + +Kind: dead code. Outcome: not taken, because it needs a maintainer decision. + +`omnigraph-bench` keeps `worker.rs`, `supervisor.rs`, `fixture_worker.rs` and +`real_graph_run.rs` (about 5,400 lines) behind `#[cfg(test)]`. #894 removed +the subcommands that they start. The +[testing guide](../dev/testing.md#cost-tests-and-benchmarks) still names the +retired runner as a test oracle, so the maintainers must decide its removal. + +### 9. A hand-written list-literal builder + +Kind: bloat. Outcome: not taken, because the risk is medium. + +`exec/mutation.rs` `typed_list_literal_to_array` repeats one builder loop for +ten element types (about 120 lines). DataFusion +`ScalarValue::new_list(..).to_array_of_size(..)` can replace it. But the +change must keep the error text, the zero-row case and the order of errors +exactly. + +### Examined and rejected + +- The single-batch shortcuts before `concat_batches` are not redundant. With + one batch, `concat_batches` can panic where `RecordBatch::try_new` returns + an error. +- `make_list_value_builder` in the loader is not the Arrow `make_builder`. The + buffer sizes and the error for an unsupported element type are different. +- The `SnapshotScanner` pass-through methods are a sealing boundary, not a + layer that only forwards calls. +- The search skipped candidates in files that open pull requests change, for + example `schema_for_table_key` in `exec/merge.rs` and + `normalize_branch_name`.