From 8024ea1d2cfb26d2af6ceb9a576293de38811604 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 01:12:30 -0400 Subject: [PATCH 01/36] bench(api): transact_upsert_replace measures upsert over existing values Nothing in the suite measured the upsert wave that retracts the current values of every (graph, subject, predicate) an upsert names: insert_formats upserts only new subjects and transact_commit measures inserts. Eight scenarios: default and named graph, each unindexed and indexed; language tags plus @list; 16 predicates per subject; an indexed base under 4x unrelated novelty; and new subjects as the #1549 control. Each scenario asserts its lane in setup and its retract/assert counts after every measured upsert, so a drifting fixture fails instead of timing the wrong path. --- BENCHMARKING.md | 1 + fluree-db-api/Cargo.toml | 4 + .../benches/transact_upsert_replace.rs | 384 ++++++++++++++++++ regression-budget.json | 5 + 4 files changed, 394 insertions(+) create mode 100644 fluree-db-api/benches/transact_upsert_replace.rs diff --git a/BENCHMARKING.md b/BENCHMARKING.md index 3a2cf7cc1e..ca7e326dcd 100644 --- a/BENCHMARKING.md +++ b/BENCHMARKING.md @@ -77,6 +77,7 @@ this table, so it is the one that rots. Add a row when you add a bench file. | `fluree-db-api` | `fulltext_query.rs` | Full-text query through novelty + index | | `fluree-db-api` | `import_bulk.rs` | Bulk Turtle import via `fluree.create(id).import(path).execute()`; single- vs default-threaded | | `fluree-db-api` | `transact_commit.rs` | Single-commit latency on a fresh and a populated ledger (`iter_batched` setup) | +| `fluree-db-api` | `transact_upsert_replace.rs` | Upsert that replaces existing values: default/named graph × novelty/indexed, lang + `@list`, 16-predicate subjects, novelty-heavy base, and the new-subject control | | `fluree-db-api` | `query_cold_reload.rs` | File-backed cold reload (load only, and load + first query) | | `fluree-db-api` | `reindex_full.rs` | `Fluree::reindex(...)` end-to-end against a single-txn populated ledger | | `fluree-db-api` | `reindex_incremental.rs` | Orchestrator's incremental path via `Fluree::trigger_index(...)` over delta novelty | diff --git a/fluree-db-api/Cargo.toml b/fluree-db-api/Cargo.toml index 8918aa160b..956cb30da8 100644 --- a/fluree-db-api/Cargo.toml +++ b/fluree-db-api/Cargo.toml @@ -590,6 +590,10 @@ harness = false name = "transact_filtered_delete" harness = false +[[bench]] +name = "transact_upsert_replace" +harness = false + [[bench]] name = "query_overlay_only_range" harness = false diff --git a/fluree-db-api/benches/transact_upsert_replace.rs b/fluree-db-api/benches/transact_upsert_replace.rs new file mode 100644 index 0000000000..e33fe11d1d --- /dev/null +++ b/fluree-db-api/benches/transact_upsert_replace.rs @@ -0,0 +1,384 @@ +//! Upsert staging latency when the payload replaces values the ledger holds. +//! +//! An upsert retracts every current value of each `(graph, subject, +//! predicate)` its payload names, then asserts the payload. That retraction +//! wave runs on every upsert of an existing entity, and nothing else in the +//! suite exercises it: `insert_formats` upserts only new subjects and +//! `transact_commit` measures inserts. +//! +//! ## Scenarios +//! +//! Each scenario upserts one payload into a freshly built ledger. N comes +//! from the scale (below); every subject named is replaced unless stated. +//! +//! 1. `default_novelty` — N subjects × 1 value, default graph, nothing +//! indexed. +//! 2. `default_indexed` — the same, indexed, novelty empty. +//! 3. `named_novelty` — the same in a named graph, nothing indexed. The +//! lane that walked the graph's whole novelty once per slot. +//! 4. `named_indexed` — the named graph, indexed. +//! 5. `lang_list` — each subject holds two language-tagged values and a +//! two-entry `@list`, indexed; the payload replaces both predicates. +//! Language tags and list positions are exactly what the wave must +//! retract as stored. +//! 6. `wide_predicates` — N/4 subjects × 16 predicates, indexed: the +//! per-slot cost shows up here, where one subject costs 16 lookups. +//! 7. `novelty_heavy` — the indexed base of (2) plus 4×N unrelated +//! novelty flakes. A lookup that translates or walks the whole overlay +//! per call (the #1722 floor) turns into a slope here. +//! 8. `new_subjects` — the payload names N subjects the ledger has never +//! seen: the #1549 absence skip, which must stay flat. +//! +//! Every scenario asserts in setup that it reached the lane it names +//! (indexed or not, graph registered, novelty size), and after each +//! measured upsert that the receipt's retract/assert counts are the ones +//! the scenario implies. A fixture that drifts fails instead of reporting +//! a number for the wrong path. +//! +//! ## Matrix +//! +//! inputs: BenchScale → N (Tiny=500, Small=2k, Medium=8k, Large=24k) +//! metric: ns/upsert (criterion default; no Throughput — the guard is +//! on the curve, like `transact_filtered_delete`) +//! +//! ## Running +//! +//! cargo bench -p fluree-db-api --bench transact_upsert_replace +//! cargo bench -p fluree-db-api --bench transact_upsert_replace -- --test +//! FLUREE_BENCH_SCALE=medium cargo bench -p fluree-db-api --bench transact_upsert_replace +//! +//! ## Cargo.toml + budget already wired (see fluree-db-api/Cargo.toml, +//! regression-budget.json). + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use fluree_bench_support::{ + bench_runtime, current_profile, current_scale, init_tracing_for_bench, next_ledger_alias, + BenchScale, +}; +use fluree_db_api::{ + CommitOpts, Fluree, FlureeBuilder, IndexConfig, LedgerState, ReindexOptions, TxnOpts, +}; +use serde_json::{json, Value as JsonValue}; + +/// Map BenchScale to N, the number of subjects the upsert replaces. +fn scale_n(scale: BenchScale) -> usize { + match scale { + // Keep tiny tiny so PR-gated runs finish quickly. + BenchScale::Tiny => 500, + BenchScale::Small => 2_000, + BenchScale::Medium => 8_000, + BenchScale::Large => 24_000, + } +} + +const CTX_PREFIX: &str = "http://example.org/ns/"; +const GRAPH: &str = "http://example.org/ns/bench-graph"; +/// Predicates per subject in `wide_predicates`. +const WIDE: usize = 16; +/// Subjects per base-load commit. +const BATCH: usize = 500; + +/// Novelty must survive the fixture: a reindex would fold it into the base. +fn index_config() -> IndexConfig { + IndexConfig { + reindex_min_bytes: 1 << 40, + reindex_max_bytes: 1 << 41, + } +} + +fn ctx() -> JsonValue { + json!({"ex": CTX_PREFIX}) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Scenario { + DefaultNovelty, + DefaultIndexed, + NamedNovelty, + NamedIndexed, + LangList, + WidePredicates, + NoveltyHeavy, + NewSubjects, +} + +impl Scenario { + const ALL: [Scenario; 8] = [ + Scenario::DefaultNovelty, + Scenario::DefaultIndexed, + Scenario::NamedNovelty, + Scenario::NamedIndexed, + Scenario::LangList, + Scenario::WidePredicates, + Scenario::NoveltyHeavy, + Scenario::NewSubjects, + ]; + + fn name(self) -> &'static str { + match self { + Scenario::DefaultNovelty => "default_novelty", + Scenario::DefaultIndexed => "default_indexed", + Scenario::NamedNovelty => "named_novelty", + Scenario::NamedIndexed => "named_indexed", + Scenario::LangList => "lang_list", + Scenario::WidePredicates => "wide_predicates", + Scenario::NoveltyHeavy => "novelty_heavy", + Scenario::NewSubjects => "new_subjects", + } + } + + fn indexed(self) -> bool { + !matches!(self, Scenario::DefaultNovelty | Scenario::NamedNovelty) + } + + fn graph(self) -> Option<&'static str> { + match self { + Scenario::NamedNovelty | Scenario::NamedIndexed => Some(GRAPH), + _ => None, + } + } + + /// Subjects in the base (and, except for `new_subjects`, in the upsert). + fn subjects(self, n: usize) -> usize { + match self { + Scenario::WidePredicates => (n / 4).max(1), + _ => n, + } + } + + /// Retractions and assertions the measured upsert must commit. + fn expected_counts(self, n: usize) -> (usize, usize) { + let subjects = self.subjects(n); + match self { + // Two tagged values + two list entries out, one of each in. + Scenario::LangList => (4 * subjects, 2 * subjects), + Scenario::WidePredicates => (WIDE * subjects, WIDE * subjects), + Scenario::NewSubjects => (0, subjects), + _ => (subjects, subjects), + } + } +} + +/// One node of the base load or of the measured upsert. +fn node(scenario: Scenario, i: usize, replacement: bool) -> JsonValue { + let id = if scenario == Scenario::NewSubjects && replacement { + format!("ex:new{i}") + } else { + format!("ex:s{i}") + }; + let mut node = match scenario { + Scenario::LangList if replacement => json!({ + "ex:label": {"@value": format!("b{i}"), "@language": "en"}, + "ex:items": {"@list": [format!("z{i}")]}, + }), + Scenario::LangList => json!({ + // Distinct lexical forms per tag, so the count check below + // reads the same whether or not the retractions keep the tag. + "ex:label": [ + {"@value": format!("a{i}"), "@language": "en"}, + {"@value": format!("c{i}"), "@language": "fr"}, + ], + "ex:items": {"@list": [format!("x{i}"), format!("y{i}")]}, + }), + Scenario::WidePredicates => { + let tag = if replacement { "n" } else { "o" }; + let mut obj = serde_json::Map::new(); + for k in 0..WIDE { + obj.insert(format!("ex:p{k}"), json!(format!("{tag}{i}-{k}"))); + } + JsonValue::Object(obj) + } + _ => { + let tag = if replacement { "new" } else { "old" }; + json!({"ex:v": format!("{tag}{i}")}) + } + }; + let obj = node.as_object_mut().expect("object"); + obj.insert("@id".to_string(), json!(id)); + if let Some(g) = scenario.graph() { + obj.insert("@graph".to_string(), json!(g)); + } + node +} + +fn payload(nodes: Vec) -> JsonValue { + json!({"@context": ctx(), "@graph": nodes}) +} + +/// The measured upsert. +fn upsert_payload(scenario: Scenario, n: usize) -> JsonValue { + payload( + (0..scenario.subjects(n)) + .map(|i| node(scenario, i, true)) + .collect(), + ) +} + +/// Everything one measured iteration consumes. The measured routine hands +/// the tempdir and the instance back to criterion, which drops them after +/// the clock stops: deleting the file-backed store costs more than the +/// upsert itself, and its cost follows the filesystem, not the code. +struct Fixture { + db_dir: tempfile::TempDir, + fluree: Fluree, + ledger: LedgerState, +} + +async fn insert_batches( + fluree: &Fluree, + mut ledger: LedgerState, + count: usize, + mut make: impl FnMut(usize) -> JsonValue, +) -> LedgerState { + let mut from = 0usize; + while from < count { + let to = (from + BATCH).min(count); + let r = fluree + .insert_with_opts( + ledger, + &payload((from..to).map(&mut make).collect()), + TxnOpts::default(), + CommitOpts::default(), + &index_config(), + ) + .await + .expect("fixture insert"); + ledger = r.ledger; + from = to; + } + ledger +} + +/// Load the base, index it when the scenario says so, and add the +/// unrelated novelty of `novelty_heavy`. Asserts the lane on the way out. +fn build_fixture(rt: &tokio::runtime::Runtime, scenario: Scenario, n: usize) -> Fixture { + rt.block_on(async { + let db_dir = tempfile::tempdir().expect("db tmpdir"); + let fluree = FlureeBuilder::file(db_dir.path().to_string_lossy().to_string()) + .with_ledger_cache_config(fluree_db_api::LedgerManagerConfig::default()) + .build() + .expect("build file-backed Fluree"); + let alias = next_ledger_alias("tur"); + let ledger = fluree.create_ledger(&alias).await.expect("create_ledger"); + let mut ledger = insert_batches(&fluree, ledger, scenario.subjects(n), |i| { + node(scenario, i, false) + }) + .await; + + if scenario.indexed() { + drop(ledger); + fluree + .reindex(&alias, ReindexOptions::default()) + .await + .expect("reindex"); + ledger = fluree.ledger(&alias).await.expect("reload after reindex"); + assert!( + ledger.snapshot.range_provider.is_some() && ledger.novelty.is_empty(), + "{}: fixture needs an indexed base and empty novelty", + scenario.name() + ); + } else { + assert!( + ledger.snapshot.range_provider.is_none(), + "{}: fixture must stay unindexed", + scenario.name() + ); + } + + if scenario == Scenario::NoveltyHeavy { + ledger = insert_batches( + &fluree, + ledger, + 4 * n, + |i| json!({"@id": format!("ex:u{i}"), "ex:w": format!("w{i}")}), + ) + .await; + assert!( + ledger.novelty.len() >= 4 * n, + "novelty_heavy: the unrelated novelty must survive the fixture" + ); + } + + if let Some(g) = scenario.graph() { + assert!( + ledger.snapshot.graph_registry.graph_id_for_iri(g).is_some(), + "{}: the named graph must be registered before the upsert", + scenario.name() + ); + } + + Fixture { + db_dir, + fluree, + ledger, + } + }) +} + +fn bench_transact_upsert_replace(c: &mut Criterion) { + init_tracing_for_bench(); + + let rt = bench_runtime(); + let scale = current_scale(); + let profile = current_profile(); + let n = scale_n(scale); + + eprintln!(" [transact_upsert_replace] scale={} n={n}", scale.as_str()); + + let mut group = c.benchmark_group("transact_upsert_replace"); + group.sample_size(profile.sample_size()); + // The fixture is rebuilt per iteration because the measured upsert + // changes it; Flat keeps criterion from assuming a cheap, repeatable + // routine. + group.sampling_mode(criterion::SamplingMode::Flat); + + for scenario in Scenario::ALL { + let upsert = upsert_payload(scenario, n); + let (retracts, asserts) = scenario.expected_counts(n); + group.bench_with_input( + BenchmarkId::new(scenario.name(), scale.as_str()), + &n, + |b, &n| { + b.iter_batched( + // Setup: base load (+ reindex, + novelty). NOT measured. + || build_fixture(&rt, scenario, n), + // Measured: one upsert over every subject named. The + // store goes back to criterion to drop untimed. + |fixture| { + let Fixture { + db_dir, + fluree, + ledger, + } = fixture; + let ledger = rt.block_on(async { + let result = fluree + .upsert_with_opts( + ledger, + &upsert, + TxnOpts::default(), + CommitOpts::default(), + &index_config(), + ) + .await + .expect("upsert"); + assert_eq!( + (result.receipt.retract_count, result.receipt.assert_count), + (retracts, asserts), + "{}: unexpected (retracts, asserts)", + scenario.name() + ); + result.ledger + }); + black_box((db_dir, fluree, ledger)) + }, + criterion::BatchSize::PerIteration, + ); + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, bench_transact_upsert_replace); +criterion_main!(benches); diff --git a/regression-budget.json b/regression-budget.json index 17b9e86965..9ff7d906a5 100644 --- a/regression-budget.json +++ b/regression-budget.json @@ -33,6 +33,11 @@ "small": 5.0, "medium": 5.0 }, + "transact_upsert_replace": { + "tiny": 10.0, + "small": 5.0, + "medium": 5.0 + }, "query_overlay_only_range": { "tiny": 10.0, "small": 5.0, From 11bcd0398044e0a6fab8e73bd2de766b6ef9f5bd Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 01:15:26 -0400 Subject: [PATCH 02/36] fix(query): a range probe through an empty overlay leaves the translation LRU alone binary_range_eq_v3 sent every probe through the cross-call translation LRU when the overlay reported a content version, and NoOverlay reports one. Staging reads the persisted index through NoOverlay once per retraction group, and the LRU key includes to_t, so each commit's first such probe per (graph, index order) inserted an empty entry into an eight-slot cache and could evict a live query's novelty translation. The translation step moves into probe_overlay, which returns an empty translation for an effectively empty overlay before touching the cache. --- fluree-db-query/src/binary_range.rs | 224 +++++++++++++++++++--------- 1 file changed, 157 insertions(+), 67 deletions(-) diff --git a/fluree-db-query/src/binary_range.rs b/fluree-db-query/src/binary_range.rs index f064ce9be4..0fe0c2f4b6 100644 --- a/fluree-db-query/src/binary_range.rs +++ b/fluree-db-query/src/binary_range.rs @@ -416,6 +416,111 @@ fn raw_fact_may_match(f: &Flake, match_val: &RangeMatch) -> bool { && match_val.o.as_ref().is_none_or(|o| f.o == *o) } +/// The overlay side of one V3 range probe: the translated ops the cursor +/// merges, the untranslatable flakes that may match the probe, and the +/// ephemeral predicate ids the ops use. +struct ProbeOverlay { + ops: Arc<[fluree_db_binary_index::OverlayOp]>, + untranslated: Vec, + ephemeral_p_id_to_sid: Arc>, +} + +/// Translate `overlay` for one probe. +/// +/// An effectively empty overlay (`NoOverlay`, or a novelty holding no +/// flakes) contributes nothing, so it returns before touching the +/// translation LRU. Staging reads the persisted index through `NoOverlay` +/// once per retraction group, and `NoOverlay` vouches for a content version, +/// so without this check every new `to_t` put an empty entry into the +/// process-wide LRU — eight slots, keyed on `to_t` — and evicted a live +/// query's novelty translation once per commit. +/// +/// Otherwise unfiltered translations are served from the cross-call LRU when +/// the overlay reports a content version (raw `Novelty` does) — see +/// `range_translation_cache` for why fresh per-call translation makes +/// point-lookup loops quadratic in novelty size. When the caller supplied a +/// projection-predicate allow-list, the translation is fresh and filtered on +/// `flake.p`: the allow-list changes both the translated set and the +/// raw-fallback set (it must not smuggle non-selected predicates back in), +/// so that product is not cacheable under the unfiltered key. Sid match (vs +/// persisted p_id) so novel predicates still pass. +#[allow(clippy::too_many_arguments)] +fn probe_overlay( + overlay: &dyn OverlayProvider, + g_id: GraphId, + index: IndexType, + effective_to_t: i64, + store: &Arc, + dict_novelty: &Arc, + runtime_small_dicts: &Arc, + match_val: &RangeMatch, + predicate_filter: Option<&[Sid]>, +) -> ProbeOverlay { + if overlay.is_effectively_empty() { + return ProbeOverlay { + ops: Arc::from(Vec::new()), + untranslated: Vec::new(), + ephemeral_p_id_to_sid: Arc::new(HashMap::new()), + }; + } + let cached = if predicate_filter.is_none() { + cached_overlay_translation( + overlay, + g_id, + index, + effective_to_t, + store, + dict_novelty, + runtime_small_dicts, + "V3 range", + ) + } else { + None + }; + if let Some(entry) = cached { + return ProbeOverlay { + ops: Arc::clone(&entry.ops), + untranslated: raw_window(&entry.raw, index, match_val) + .iter() + .filter(|f| raw_fact_may_match(f, match_val)) + .cloned() + .collect(), + ephemeral_p_id_to_sid: Arc::clone(&entry.ephemeral_p_id_to_sid), + }; + } + let predicate_filter_sids = predicate_filter.map(<[Sid]>::to_vec); + let OverlayTranslateV3Result { + mut ops, + raw, + ephemeral_p_id_to_sid, + failed: _overlay_failed_translation, + } = translate_overlay_ops_v3_with_raw( + overlay, + g_id, + index, + effective_to_t, + store, + dict_novelty, + runtime_small_dicts, + move |flake| match &predicate_filter_sids { + Some(allow) => allow.iter().any(|p| p == &flake.p), + None => true, + }, + "V3 range", + ); + let order = index_type_to_sort_order(index); + fluree_db_binary_index::read::types::sort_overlay_ops(&mut ops, order); + fluree_db_binary_index::read::types::resolve_overlay_ops(&mut ops); + ProbeOverlay { + ops: Arc::from(ops), + untranslated: raw + .into_iter() + .filter(|f| raw_fact_may_match(f, match_val)) + .collect(), + ephemeral_p_id_to_sid: Arc::new(ephemeral_p_id_to_sid), + } +} + /// V3 equality range query: scan the appropriate index order with filters, /// decode each row to a `Flake`, apply overlay merge. #[allow(clippy::too_many_arguments)] @@ -642,73 +747,21 @@ fn binary_range_eq_v3( }) .collect(); - // Overlay translation. Unfiltered translations are served from the - // cross-call LRU when the overlay reports a content version (raw - // `Novelty` does) — see `range_translation_cache` for why fresh per-call - // translation makes point-lookup loops quadratic in novelty size. - // - // When the caller supplied a projection-predicate allow-list, translate - // fresh with the `flake.p` filter as before — the allow-list changes - // both the translated set and the raw-fallback set (it must not smuggle - // non-selected predicates back in), so that product is not cacheable - // under the unfiltered key. Sid match (vs persisted p_id) so novel - // predicates still pass. - let cached = if opts.predicate_filter.is_none() { - cached_overlay_translation( - overlay, - g_id, - index, - effective_to_t, - store, - dict_novelty, - runtime_small_dicts, - "V3 range", - ) - } else { - None - }; - let (overlay_ops, untranslated, ephemeral_p_id_to_sid) = match cached { - Some(entry) => ( - Arc::clone(&entry.ops), - raw_window(&entry.raw, index, match_val) - .iter() - .filter(|f| raw_fact_may_match(f, match_val)) - .cloned() - .collect::>(), - Arc::clone(&entry.ephemeral_p_id_to_sid), - ), - None => { - let predicate_filter_sids = opts.predicate_filter.clone(); - let OverlayTranslateV3Result { - mut ops, - raw, - ephemeral_p_id_to_sid, - failed: _overlay_failed_translation, - } = translate_overlay_ops_v3_with_raw( - overlay, - g_id, - index, - effective_to_t, - store, - dict_novelty, - runtime_small_dicts, - move |flake| match &predicate_filter_sids { - Some(allow) => allow.iter().any(|p| p == &flake.p), - None => true, - }, - "V3 range", - ); - fluree_db_binary_index::read::types::sort_overlay_ops(&mut ops, order); - fluree_db_binary_index::read::types::resolve_overlay_ops(&mut ops); - ( - Arc::<[fluree_db_binary_index::OverlayOp]>::from(ops), - raw.into_iter() - .filter(|f| raw_fact_may_match(f, match_val)) - .collect(), - Arc::new(ephemeral_p_id_to_sid), - ) - } - }; + let ProbeOverlay { + ops: overlay_ops, + untranslated, + ephemeral_p_id_to_sid, + } = probe_overlay( + overlay, + g_id, + index, + effective_to_t, + store, + dict_novelty, + runtime_small_dicts, + match_val, + opts.predicate_filter.as_deref(), + ); if !untranslated.is_empty() { tracing::trace!( raw_merged = untranslated.len(), @@ -2073,4 +2126,41 @@ mod tests { 2 ); } + + /// A probe through an overlay with nothing in it — `NoOverlay`, which + /// staging uses for every base-only read — translates nothing and must + /// leave the process-wide translation LRU alone. `NoOverlay` reports a + /// content version, so each new `to_t` used to add an empty entry, and + /// with eight slots a few commits evicted a live query's translation. + #[test] + fn an_empty_overlay_probe_leaves_the_translation_lru_alone() { + let store = Arc::new(BinaryIndexStore::empty(std::env::temp_dir())); + let dict_novelty = Arc::new(DictNovelty::new_uninitialized()); + let small_dicts = Arc::new(RuntimeSmallDicts::new()); + let probe = RangeMatch::subject_predicate(s("a"), s("weight")); + for to_t in 1..=3 { + let out = probe_overlay( + &fluree_db_core::NoOverlay, + 0, + IndexType::Spot, + to_t, + &store, + &dict_novelty, + &small_dicts, + &probe, + None, + ); + assert!(out.ops.is_empty() && out.untranslated.is_empty()); + } + let entries_for_store = range_translation_cache() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .filter(|(key, _)| key.store_id == store.store_id()) + .count(); + assert_eq!( + entries_for_store, 0, + "an empty overlay must not occupy translation LRU slots" + ); + } } From 73f5223aeea66eec1d1a2dec8a01cd42afea2cde Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 01:18:59 -0400 Subject: [PATCH 03/36] feat(transact): CurrentFacts reads the facts a retraction may name A retraction staged for commit must name a fact that is currently stored. Today staging rebuilds retractions from query bindings, templates and edge keys, which drops language tags and list positions and invents retractions of facts that do not exist. This adds the read those sites will share: per (graph, subject, predicate) slot, one seek into the persisted index through NoOverlay plus a Sid-space novelty seek bounded by the slot's sentinels, lifecycle-resolved, graph stamped. A graph whose novelty is small next to the slots asked of it is walked once instead. StoredFact (private fields, built only here) and Retraction (private field, built from StoredFact::retract) carry the invariant in the types; the next commits route staging through them. --- fluree-db-transact/src/current_facts.rs | 590 ++++++++++++++++++++++++ fluree-db-transact/src/lib.rs | 2 + 2 files changed, 592 insertions(+) create mode 100644 fluree-db-transact/src/current_facts.rs diff --git a/fluree-db-transact/src/current_facts.rs b/fluree-db-transact/src/current_facts.rs new file mode 100644 index 0000000000..ba4c304f72 --- /dev/null +++ b/fluree-db-transact/src/current_facts.rs @@ -0,0 +1,590 @@ +//! Current facts: the one read every staged retraction comes from. +//! +//! **Invariant.** A retraction staged for commit names a fact that is +//! currently stored: its `(g, s, p, o, dt, m)` is read from storage (the +//! persisted index plus novelty, lifecycle-resolved), never rebuilt from a +//! query binding, a template, or an edge key. An intent that matches no +//! stored fact stages nothing. +//! +//! The second sentence is what makes the accumulator's cancellation sound. +//! It cancels a retraction against a same-transaction assertion of the same +//! fact, which implements SPARQL 1.1 Update's `(DS − D) ∪ I` only if every +//! retraction cancels a stored fact: a retraction of an absent `x` would +//! cancel an `INSERT x` in the same transaction and `x` would never be +//! written. +//! +//! The types carry the invariant. [`CurrentFacts`] is the only producer of +//! [`StoredFact`], whose fields are private, and [`StoredFact::retract`] is +//! the way staging turns one into a [`Retraction`] — the only item type +//! [`FlakeAccumulator::push_retractions`](crate::generate::FlakeAccumulator::push_retractions) +//! accepts. +//! +//! ## The read +//! +//! Per `(graph, subject, predicate)` slot (or subject): one seek into the +//! persisted index through `NoOverlay` (no novelty translation, no shared +//! LRU entry), plus a Sid-space seek into novelty bounded by the slot's +//! sentinels, then [`resolve_current_flakes`]: the newest op per fact wins +//! and retractions drop out. Index rows come back without their graph, so +//! every survivor is stamped with the slot's graph Sid. +//! +//! When a graph's novelty is small relative to the number of slots asked +//! about in it, one filtered walk over that graph's novelty answers every +//! slot at once instead (see [`NOVELTY_WALK_RATIO`]). +//! +//! Not view-policy filtered, matching the whole-graph scans of sync and +//! CLEAR: modify policy still runs on the staged flakes. + +use crate::error::{Result, TransactError}; +use fluree_db_core::comparator::IndexType; +use fluree_db_core::range::resolve_current_flakes; +use fluree_db_core::{ + range_with_overlay, Flake, GraphId, NoOverlay, OverlayProvider, RangeMatch, RangeOptions, + RangeTest, Sid, Tracker, +}; +use fluree_db_ledger::LedgerState; +use std::collections::{HashMap, HashSet}; + +/// Seek-versus-walk break-even for a graph's novelty. Slots are answered by +/// one filtered walk over the graph's novelty when +/// `slots × segments × NOVELTY_WALK_RATIO > novelty flakes`, and by one +/// bounded seek per slot otherwise. A seek costs `O(segments · log N)` +/// comparisons, a walk `O(N)` hash probes; 16 puts the switch where a +/// seek's log factor and per-call overhead catch up with a probe per flake. +pub const NOVELTY_WALK_RATIO: usize = 16; + +/// Where a stored fact's surviving op was read from. +/// +/// Index rows are decodes, and some decodes are lossy — a big integer's +/// declared XSD subtype is not stored, so it decodes as `xsd:integer` or +/// `xsd:decimal`. Recording the origin keeps any value-only identity rule +/// for those rows in this one type. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Origin { + /// The persisted index. + Index, + /// Novelty (commits since the last index). + Novelty, +} + +/// A fact as currently stored in its graph, graph Sid stamped (`None` for +/// the default graph). Built only by [`CurrentFacts`]. +#[derive(Clone, Debug)] +pub struct StoredFact { + flake: Flake, + origin: Origin, +} + +impl StoredFact { + /// The stored fact (an assertion; `t` is the transaction that wrote it). + pub fn flake(&self) -> &Flake { + &self.flake + } + + /// Where the fact was read from. + pub fn origin(&self) -> Origin { + self.origin + } + + /// Retract this fact at `t`. The one way staging retracts a stored fact. + pub fn retract(self, t: i64) -> Retraction { + Retraction(Flake { + t, + op: false, + ..self.flake + }) + } +} + +/// A retraction of a currently stored fact. +/// +/// The field is private to this module: a `Retraction` comes from +/// [`StoredFact::retract`] or from a WHERE row proven to be the decode of a +/// stored fact (see `resolve_intents`), and from nowhere else. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Retraction(Flake); + +impl Retraction { + /// The retraction flake. + pub fn flake(&self) -> &Flake { + &self.0 + } + + /// Unwrap into the retraction flake, for the staged flake set. + pub fn into_flake(self) -> Flake { + self.0 + } +} + +/// A `(graph, subject, predicate)` slot to read. +#[derive(Clone, Debug)] +pub struct Slot { + /// Ledger graph id to read (0 = default graph). + pub g_id: GraphId, + /// Graph Sid stamped on every fact read (`None` = default graph). + pub g_sid: Option, + /// Subject. + pub s: Sid, + /// Predicate. + pub p: Sid, +} + +/// Reads a ledger's current facts as of its current `t`. +pub struct CurrentFacts<'l> { + ledger: &'l LedgerState, + to_t: i64, +} + +impl<'l> CurrentFacts<'l> { + /// Current facts of `ledger` as of `ledger.t()`. + pub fn new(ledger: &'l LedgerState) -> Self { + Self { + ledger, + to_t: ledger.t(), + } + } + + /// The current facts of every slot, in slot order: one `Vec` per slot, + /// each in SPOT order. + /// + /// Duplicate slots are answered once and cloned. Slots of one graph + /// share one decision between per-slot novelty seeks and a single + /// filtered novelty walk. + pub async fn of_slots(&self, slots: &[Slot]) -> Result>> { + let novelty = NoveltyPlan::for_slots(self.ledger, slots, self.to_t); + let mut out = Vec::with_capacity(slots.len()); + for slot in slots { + let rm = RangeMatch::subject_predicate(slot.s.clone(), slot.p.clone()); + let mut flakes = self.base(slot.g_id, rm).await?; + novelty.collect_slot(self.ledger, slot, self.to_t, &mut flakes); + out.push(self.resolve(flakes, slot.g_sid.as_ref())); + } + Ok(out) + } + + /// The current facts of one subject in graph `g_id`, in SPOT order. + pub async fn of_subject( + &self, + g_id: GraphId, + g_sid: Option<&Sid>, + s: &Sid, + ) -> Result> { + let mut flakes = self + .base(g_id, RangeMatch::new().with_subject(s.clone())) + .await?; + let lo = Flake::min_for_subject(s.clone()); + let hi = Flake::max_for_subject(s.clone()); + self.ledger.novelty.for_each_overlay_flake( + g_id, + IndexType::Spot, + Some(&lo), + Some(&hi), + false, + self.to_t, + &mut |f| flakes.push(f.clone()), + ); + Ok(self.resolve(flakes, g_sid)) + } + + /// Every current fact of graph `g_id`, graph Sid stamped. + /// + /// The whole-graph verbs (graph sync, CLEAR, DROP, COPY, MOVE) read + /// through this. `limit` bounds what is materialized, not just what is + /// returned: the provider's drain loop stops at `limit + 1` and the scan + /// then fails with [`TransactError::WholeGraphScanTooLarge`]. + pub async fn whole_graph( + &self, + g_id: GraphId, + g_sid: Option<&Sid>, + limit: Option, + tracker: Option<&Tracker>, + ) -> Result> { + let db_ref = match tracker { + Some(t) => self.ledger.as_graph_db_ref(g_id).with_tracker(t), + None => self.ledger.as_graph_db_ref(g_id), + }; + // `Eq` with an empty match is the whole-graph scan on both range + // paths: the V3 provider treats "nothing bound" as a full-index + // cursor and rejects every other `RangeTest`, and the genesis + // (overlay-only) path matches an empty `Eq` against every flake. + let opts = RangeOptions { + flake_limit: limit.map(|l| l.saturating_add(1)), + ..Default::default() + }; + let flakes = db_ref + .range_with_opts(IndexType::Spot, RangeTest::Eq, RangeMatch::new(), opts) + .await + .map_err(|e| TransactError::FlakeGeneration(format!("graph scan failed: {e}")))?; + if let Some(l) = limit { + if flakes.len() > l { + return Err(TransactError::WholeGraphScanTooLarge { limit: l }); + } + } + // Already lifecycle-resolved by the range read. + let index_t = self.ledger.snapshot.t; + Ok(flakes + .into_iter() + .map(|flake| stored(flake, g_sid, index_t)) + .collect()) + } + + /// Persisted-index rows matching `rm` in graph `g_id`, read through + /// `NoOverlay`: one leaf seek, no novelty translation. At genesis (no + /// index yet) there is no provider and the answer is empty. + async fn base(&self, g_id: GraphId, rm: RangeMatch) -> Result> { + Ok(range_with_overlay( + &self.ledger.snapshot, + g_id, + &NoOverlay, + IndexType::Spot, + RangeTest::Eq, + rm, + RangeOptions::new().with_to_t(self.to_t), + ) + .await?) + } + + /// Resolve base rows plus novelty ops to the current facts, stamping + /// the graph and the origin. + fn resolve(&self, flakes: Vec, g_sid: Option<&Sid>) -> Vec { + if flakes.is_empty() { + return Vec::new(); + } + let index_t = self.ledger.snapshot.t; + resolve_current_flakes(flakes, IndexType::Spot) + .into_iter() + .map(|flake| stored(flake, g_sid, index_t)) + .collect() + } +} + +/// Wrap a resolved flake: stamp the graph (index rows decode with `g: +/// None`, whatever graph they live in) and record where it came from. +/// Novelty holds only commits after the index, so a fact whose surviving op +/// is at or below the index `t` was read from the index. +fn stored(mut flake: Flake, g_sid: Option<&Sid>, index_t: i64) -> StoredFact { + flake.g = g_sid.cloned(); + let origin = if flake.t <= index_t { + Origin::Index + } else { + Origin::Novelty + }; + StoredFact { flake, origin } +} + +/// How each graph's novelty is read for a batch of slots. +struct NoveltyPlan { + /// Graphs whose novelty was walked once, keyed `(graph, s, p)`. + walked: HashMap<(GraphId, Sid, Sid), Vec>, + walked_graphs: HashSet, + /// Graphs with no novelty at all: nothing to read. + empty_graphs: HashSet, +} + +impl NoveltyPlan { + fn for_slots(ledger: &LedgerState, slots: &[Slot], to_t: i64) -> Self { + let novelty = ledger.novelty.as_ref(); + let mut per_graph: HashMap = HashMap::new(); + for slot in slots { + *per_graph.entry(slot.g_id).or_default() += 1; + } + let mut plan = NoveltyPlan { + walked: HashMap::new(), + walked_graphs: HashSet::new(), + empty_graphs: HashSet::new(), + }; + for (g_id, slot_count) in per_graph { + let flakes = novelty.overlay_flake_count(g_id).unwrap_or(usize::MAX); + if flakes == 0 { + plan.empty_graphs.insert(g_id); + continue; + } + let segments = novelty.segment_count(g_id).max(1); + if slot_count + .saturating_mul(segments) + .saturating_mul(NOVELTY_WALK_RATIO) + <= flakes + { + continue; // per-slot seeks + } + let wanted: HashSet<(&Sid, &Sid)> = slots + .iter() + .filter(|slot| slot.g_id == g_id) + .map(|slot| (&slot.s, &slot.p)) + .collect(); + let walked = &mut plan.walked; + novelty.for_each_overlay_flake( + g_id, + IndexType::Spot, + None, + None, + true, + to_t, + &mut |f| { + if wanted.contains(&(&f.s, &f.p)) { + walked + .entry((g_id, f.s.clone(), f.p.clone())) + .or_default() + .push(f.clone()); + } + }, + ); + plan.walked_graphs.insert(g_id); + } + plan + } + + /// Append the novelty ops of `slot` to `out`. + fn collect_slot(&self, ledger: &LedgerState, slot: &Slot, to_t: i64, out: &mut Vec) { + if self.empty_graphs.contains(&slot.g_id) { + return; + } + if self.walked_graphs.contains(&slot.g_id) { + if let Some(ops) = self + .walked + .get(&(slot.g_id, slot.s.clone(), slot.p.clone())) + { + out.extend(ops.iter().cloned()); + } + return; + } + let lo = Flake::min_for_subject_predicate(slot.s.clone(), slot.p.clone()); + let hi = Flake::max_for_subject_predicate(slot.s.clone(), slot.p.clone()); + ledger.novelty.for_each_overlay_flake( + slot.g_id, + IndexType::Spot, + Some(&lo), + Some(&hi), + false, + to_t, + &mut |f| out.push(f.clone()), + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use fluree_db_core::{FlakeMeta, FlakeValue, LedgerSnapshot}; + use fluree_db_novelty::Novelty; + + fn sid(name: &str) -> Sid { + Sid::new(100, name) + } + + fn string_dt() -> Sid { + Sid::new(fluree_vocab::namespaces::XSD, "string") + } + + fn lang_dt() -> Sid { + Sid::new(fluree_vocab::namespaces::RDF, "langString") + } + + fn flake(s: &str, p: &str, o: &str, t: i64, op: bool, m: Option) -> Flake { + let dt = if m.as_ref().is_some_and(|m| m.lang.is_some()) { + lang_dt() + } else { + string_dt() + }; + Flake::new( + sid(s), + sid(p), + FlakeValue::String(o.to_string()), + dt, + t, + op, + m, + ) + } + + fn in_graph(mut f: Flake, g: &Sid) -> Flake { + f.g = Some(g.clone()); + f + } + + /// A genesis ledger (nothing indexed) whose novelty holds `commits`, one + /// commit per inner vector at t = 1, 2, … + fn ledger(commits: Vec>, reverse_graph: &HashMap) -> LedgerState { + let mut novelty = Novelty::new(0); + for (i, flakes) in commits.into_iter().enumerate() { + novelty + .apply_commit(flakes, i as i64 + 1, reverse_graph) + .expect("apply commit"); + } + LedgerState::new(LedgerSnapshot::genesis("test:main"), novelty) + } + + fn slot(g_id: GraphId, g_sid: Option<&Sid>, s: &str, p: &str) -> Slot { + Slot { + g_id, + g_sid: g_sid.cloned(), + s: sid(s), + p: sid(p), + } + } + + fn objects(facts: &[StoredFact]) -> Vec<(String, Option)> { + facts + .iter() + .map(|f| match &f.flake().o { + FlakeValue::String(s) => (s.clone(), f.flake().m.clone()), + other => (format!("{other:?}"), f.flake().m.clone()), + }) + .collect() + } + + /// Lang-tagged and list values come back as stored — with their tag and + /// position — and a value retracted in a later commit is gone. + #[tokio::test] + async fn slot_facts_keep_lang_and_list_metadata_and_drop_retracted_values() { + let en = FlakeMeta::with_lang("en"); + let fr = FlakeMeta::with_lang("fr"); + let ledger = ledger( + vec![ + vec![ + flake("a", "label", "x", 1, true, Some(en.clone())), + flake("a", "label", "x", 1, true, Some(fr.clone())), + flake("a", "items", "v", 1, true, Some(FlakeMeta::with_index(0))), + flake("a", "items", "v", 1, true, Some(FlakeMeta::with_index(1))), + flake("a", "items", "w", 1, true, Some(FlakeMeta::with_index(2))), + flake("a", "other", "o", 1, true, None), + ], + vec![flake("a", "items", "w", 2, false, Some(FlakeMeta::with_index(2)))], + ], + &HashMap::new(), + ); + let facts = CurrentFacts::new(&ledger) + .of_slots(&[slot(0, None, "a", "label"), slot(0, None, "a", "items")]) + .await + .unwrap(); + let mut labels = objects(&facts[0]); + labels.sort_by(|a, b| a.1.cmp(&b.1)); + assert_eq!( + labels, + vec![("x".to_string(), Some(en)), ("x".to_string(), Some(fr))] + ); + assert_eq!( + objects(&facts[1]), + vec![ + ("v".to_string(), Some(FlakeMeta::with_index(0))), + ("v".to_string(), Some(FlakeMeta::with_index(1))), + ], + "list duplicates stay separate facts; the retracted entry is gone" + ); + assert!(facts + .iter() + .flatten() + .all(|f| f.origin() == Origin::Novelty && f.flake().g.is_none())); + } + + /// A slot in a named graph reads that graph only, and every fact is + /// stamped with the graph Sid the caller gives, which is what lets the + /// retraction cancel a same-transaction assertion in that graph. + #[tokio::test] + async fn named_graph_slots_read_their_graph_and_stamp_it() { + let g1 = sid("g1"); + let reverse: HashMap = [(g1.clone(), 3)].into_iter().collect(); + let ledger = ledger( + vec![vec![ + flake("a", "p", "default", 1, true, None), + in_graph(flake("a", "p", "named", 1, true, None), &g1), + ]], + &reverse, + ); + let facts = CurrentFacts::new(&ledger) + .of_slots(&[slot(3, Some(&g1), "a", "p"), slot(0, None, "a", "p")]) + .await + .unwrap(); + assert_eq!(objects(&facts[0]), vec![("named".to_string(), None)]); + assert_eq!(facts[0][0].flake().g.as_ref(), Some(&g1)); + assert_eq!(objects(&facts[1]), vec![("default".to_string(), None)]); + assert_eq!(facts[1][0].flake().g, None); + } + + /// The per-slot seek and the single filtered walk must give the same + /// answer: which one runs depends only on how many slots a graph gets + /// relative to its novelty size. + #[tokio::test] + async fn seek_and_walk_agree() { + let mut commit = Vec::new(); + for i in 0..40 { + commit.push(flake(&format!("s{i:02}"), "p", &format!("v{i}"), 1, true, None)); + commit.push(flake(&format!("s{i:02}"), "q", &format!("w{i}"), 1, true, None)); + } + let retract = vec![flake("s07", "p", "v7", 2, false, None)]; + let ledger = ledger(vec![commit, retract], &HashMap::new()); + let facts = CurrentFacts::new(&ledger); + + // One slot against 81 novelty flakes: seek. + let one = facts.of_slots(&[slot(0, None, "s07", "q")]).await.unwrap(); + // Every slot: walk. + let all_slots: Vec = (0..40) + .flat_map(|i| { + [ + slot(0, None, &format!("s{i:02}"), "p"), + slot(0, None, &format!("s{i:02}"), "q"), + ] + }) + .collect(); + let all = facts.of_slots(&all_slots).await.unwrap(); + assert_eq!(objects(&one[0]), vec![("w7".to_string(), None)]); + assert_eq!(objects(&all[15]), vec![("w7".to_string(), None)]); + assert!(all[14].is_empty(), "s07 p was retracted"); + assert_eq!( + all.iter().map(Vec::len).sum::(), + 79, + "80 values minus the retracted one" + ); + } + + #[tokio::test] + async fn subject_facts_cover_every_predicate() { + let ledger = ledger( + vec![vec![ + flake("a", "p", "1", 1, true, None), + flake("a", "q", "2", 1, true, None), + flake("b", "p", "3", 1, true, None), + ]], + &HashMap::new(), + ); + let facts = CurrentFacts::new(&ledger) + .of_subject(0, None, &sid("a")) + .await + .unwrap(); + assert_eq!( + objects(&facts), + vec![("1".to_string(), None), ("2".to_string(), None)] + ); + } + + /// `retract` keeps the stored identity — graph, datatype, tag, list + /// position — and changes only `t` and the op. + #[tokio::test] + async fn retract_keeps_the_stored_identity() { + let g1 = sid("g1"); + let reverse: HashMap = [(g1.clone(), 3)].into_iter().collect(); + let stored_flake = in_graph( + flake( + "a", + "label", + "x", + 1, + true, + FlakeMeta::from_parts(Some("en"), Some(4)), + ), + &g1, + ); + let ledger = ledger(vec![vec![stored_flake.clone()]], &reverse); + let mut facts = CurrentFacts::new(&ledger) + .of_slots(&[slot(3, Some(&g1), "a", "label")]) + .await + .unwrap(); + let retraction = facts.remove(0).remove(0).retract(9).into_flake(); + // `Flake`'s equality ignores `g`, `t` and `op`, so check them apart. + assert_eq!(retraction, stored_flake); + assert_eq!((retraction.t, retraction.op), (9, false)); + assert_eq!(retraction.g, stored_flake.g); + assert_eq!(retraction.m, stored_flake.m); + assert_eq!(retraction.dt, stored_flake.dt); + } +} diff --git a/fluree-db-transact/src/lib.rs b/fluree-db-transact/src/lib.rs index 9dfc3a492e..f62716c1a5 100644 --- a/fluree-db-transact/src/lib.rs +++ b/fluree-db-transact/src/lib.rs @@ -34,6 +34,7 @@ pub mod address; pub mod commit; pub mod commit_flakes; +pub mod current_facts; pub mod datatype_limit; pub mod error; pub mod flake_sink; @@ -62,6 +63,7 @@ pub use commit::{ build_commit, commit, CommitOpts, CommitOptsRequest, CommitReceipt, StagedCommit, }; pub use commit_flakes::generate_commit_flakes; +pub use current_facts::{CurrentFacts, Origin, Retraction, Slot, StoredFact}; pub use error::{Result, TransactError}; pub use flake_sink::FlakeSink; pub use generate::{apply_cancellation, FlakeGenerator}; From 6d793a38bb500d6b293fba84ca956626cfc4f7c9 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 01:27:27 -0400 Subject: [PATCH 04/36] fix(transact): upsert retracts the values it replaces as they are stored The upsert wave queried each (graph, subject, predicate) with execute_pattern and rebuilt every retraction from the ?o binding with m: None. A language-tagged value was retracted without its tag and a list entry without its position, so neither retraction matched anything: the old values stayed, and an identical refresh of such values committed two flakes every time (#1976). A named graph without a range provider also walked the graph's whole novelty once per slot. The wave now asks CurrentFacts for each slot's current facts and retracts them as stored. The #1549 absence pre-check and its counters are unchanged; pattern_queries now counts slot reads. binding_to_flake_object, query_novelty_for_graph and the per-group materializer are gone. The replacement unit is one function, upsert_replacement_slots: every current value of each (graph, subject, predicate) the payload names, all languages and list positions together. Blank subjects join the wave when the skolem scope is deterministic (the upsert payload scope), so an identical re-upsert of a document with blank nodes is now a no-op too. --- fluree-db-api/tests/grp_transact.rs | 2 + .../tests/it_upsert_stored_retractions.rs | 561 ++++++++++++++++++ fluree-db-transact/src/generate/flakes.rs | 10 + fluree-db-transact/src/stage.rs | 347 ++++------- 4 files changed, 675 insertions(+), 245 deletions(-) create mode 100644 fluree-db-api/tests/it_upsert_stored_retractions.rs diff --git a/fluree-db-api/tests/grp_transact.rs b/fluree-db-api/tests/grp_transact.rs index 82d608849b..94bf0129b2 100644 --- a/fluree-db-api/tests/grp_transact.rs +++ b/fluree-db-api/tests/grp_transact.rs @@ -51,3 +51,5 @@ mod it_txn_meta; mod it_update_wildcard_delete_indexed; #[path = "it_upsert_duplicate_ids_repro.rs"] mod it_upsert_duplicate_ids_repro; +#[path = "it_upsert_stored_retractions.rs"] +mod it_upsert_stored_retractions; diff --git a/fluree-db-api/tests/it_upsert_stored_retractions.rs b/fluree-db-api/tests/it_upsert_stored_retractions.rs new file mode 100644 index 0000000000..a4402bd8e4 --- /dev/null +++ b/fluree-db-api/tests/it_upsert_stored_retractions.rs @@ -0,0 +1,561 @@ +//! Upsert retracts the values it replaces exactly as they are stored. +//! +//! The upsert wave used to rebuild each retraction from a query binding, +//! which carries neither a language tag nor a list position. `"a"@en` +//! was retracted as a tagless `"a"`, list entry `"x"` at position 3 as a +//! plain `"x"`, and neither matched anything: the old values stayed, and an +//! identical refresh committed two flakes every time (#1976). The wave now +//! reads each `(graph, subject, predicate)` slot's current facts from +//! storage and retracts them as stored. +//! +//! Every case runs on three lanes and is checked before and after the next +//! index build: +//! - `novelty`: nothing indexed; +//! - `indexed`: the replaced values are all in the persisted index; +//! - `novelty-over-index`: the first seed commit is indexed and the rest +//! sit in novelty on top of it, so one slot's values span both. + +use crate::support; +use fluree_db_api::{Fluree, FlureeBuilder, LedgerState}; +use fluree_db_core::comparator::IndexType; +use fluree_db_core::{range_with_overlay, FlakeValue, RangeMatch, RangeOptions, RangeTest}; +use serde_json::{json, Value as JsonValue}; + +const EX: &str = "http://example.org/ns/"; +const G1: &str = "http://example.org/g1"; + +fn ctx() -> JsonValue { + json!({"ex": EX}) +} + +#[derive(Clone)] +enum Payload { + JsonLd(JsonValue), + Turtle(String), +} + +fn ttl(body: &str) -> Payload { + Payload::Turtle(format!("@prefix ex: <{EX}> .\n{body}")) +} + +fn jsonld(node: JsonValue) -> Payload { + let mut doc = json!({"@context": ctx()}); + match node { + JsonValue::Array(nodes) => doc["@graph"] = JsonValue::Array(nodes), + JsonValue::Object(map) => { + for (k, v) in map { + doc[k] = v; + } + } + other => panic!("not a node: {other}"), + } + Payload::JsonLd(doc) +} + +#[derive(Clone, Copy, Debug)] +enum Lane { + Novelty, + Indexed, + NoveltyOverIndex, +} + +const LANES: [Lane; 3] = [Lane::Novelty, Lane::Indexed, Lane::NoveltyOverIndex]; + +/// A file-backed ledger, so an index can be built and reloaded mid-test. +struct Harness { + _dir: tempfile::TempDir, + fluree: Fluree, + id: String, + ledger: Option, +} + +impl Harness { + async fn new(name: &str) -> Self { + let dir = tempfile::tempdir().expect("tempdir"); + let fluree = FlureeBuilder::file(dir.path().to_string_lossy().to_string()) + .build() + .expect("file fluree"); + let id = format!("it/upsert-stored-{name}:main"); + let ledger = fluree.create_ledger(&id).await.expect("create ledger"); + Self { + _dir: dir, + fluree, + id, + ledger: Some(ledger), + } + } + + fn ledger(&self) -> &LedgerState { + self.ledger.as_ref().expect("ledger") + } + + async fn insert(&mut self, payload: &Payload) { + let ledger = self.ledger.take().expect("ledger"); + let result = match payload { + Payload::JsonLd(doc) => self.fluree.insert(ledger, doc).await, + Payload::Turtle(text) => self.fluree.insert_turtle(ledger, text).await, + }; + self.ledger = Some(result.expect("seed insert").ledger); + } + + async fn upsert(&mut self, payload: &Payload) -> fluree_db_api::CommitReceipt { + let ledger = self.ledger.take().expect("ledger"); + let result = match payload { + Payload::JsonLd(doc) => self.fluree.upsert(ledger, doc).await, + // The builder lane reads TriG as well as Turtle. + Payload::Turtle(text) => { + self.fluree + .stage_owned(ledger) + .upsert_turtle(text) + .execute() + .await + } + }; + let result = result.expect("upsert"); + self.ledger = Some(result.ledger); + result.receipt + } + + /// Build and publish an index at the current head, then reload. + async fn reindex(&mut self) { + drop(self.ledger.take()); + support::rebuild_and_publish_index(&self.fluree, &self.id).await; + let ledger = self.fluree.ledger(&self.id).await.expect("reload"); + assert!( + ledger.snapshot.range_provider.is_some(), + "{}: the reload must see the index", + self.id + ); + self.ledger = Some(ledger); + } + + /// The current facts of `subject` in `graph` (None = default graph), as + /// `pred=value[@lang][#i]`, sorted. Read at the flake level so list + /// positions and language tags are visible. + async fn facts(&self, graph: Option<&str>, subject: &str) -> Vec { + let ledger = self.ledger(); + let g_id = match graph { + None => 0, + Some(iri) => match ledger.snapshot.graph_registry.graph_id_for_iri(iri) { + Some(g) => g, + None => return Vec::new(), + }, + }; + let Some(s) = ledger.snapshot.encode_iri(&format!("{EX}{subject}")) else { + return Vec::new(); + }; + let flakes = range_with_overlay( + &ledger.snapshot, + g_id, + ledger.novelty.as_ref(), + IndexType::Spot, + RangeTest::Eq, + RangeMatch::new().with_subject(s), + RangeOptions::new().with_to_t(ledger.t()), + ) + .await + .expect("range"); + let mut out: Vec = flakes + .iter() + .filter(|f| f.op) + .map(|f| { + let o = match &f.o { + FlakeValue::String(s) => s.clone(), + FlakeValue::Ref(sid) => format!("<{}>", sid.name), + other => format!("{other:?}"), + }; + let lang = + f.m.as_ref() + .and_then(|m| m.lang.as_deref()) + .map(|l| format!("@{l}")) + .unwrap_or_default(); + let i = + f.m.as_ref() + .and_then(|m| m.i) + .map(|i| format!("#{i}")) + .unwrap_or_default(); + format!("{}={o}{lang}{i}", f.p.name) + }) + .collect(); + out.sort(); + out + } +} + +/// One upsert-replace scenario. +struct Case { + name: &'static str, + /// Seed commits, in order. On the `novelty-over-index` lane the first is + /// indexed and the rest are committed on top. + seed: Vec, + upsert: Payload, + graph: Option<&'static str>, + /// Facts of `ex:s` after the upsert. + expect: Vec<&'static str>, + /// `(retract_count, assert_count)` of the upsert's commit; `(0, 0)` means + /// no commit at all. + counts: (usize, usize), +} + +async fn run(case: &Case) -> Vec { + let mut failures = Vec::new(); + for lane in LANES { + let label = format!("{} [{lane:?}]", case.name); + let slug = format!("{}-{lane:?}", case.name) + .to_lowercase() + .replace(' ', "-"); + let mut h = Harness::new(&slug).await; + match lane { + Lane::Novelty => { + for p in &case.seed { + h.insert(p).await; + } + } + Lane::Indexed => { + for p in &case.seed { + h.insert(p).await; + } + h.reindex().await; + } + Lane::NoveltyOverIndex => { + // Something must be indexed even for a one-commit seed. + h.insert(&jsonld(json!({"@id": "ex:unrelated", "ex:u": "u"}))) + .await; + let (first, rest) = case.seed.split_first().expect("seed"); + if case.seed.len() > 1 { + h.insert(first).await; + h.reindex().await; + } else { + h.reindex().await; + h.insert(first).await; + } + for p in rest { + h.insert(p).await; + } + } + } + let t_before = h.ledger().t(); + let receipt = h.upsert(&case.upsert).await; + let counts = if receipt.flake_count == 0 { + if receipt.t != t_before { + failures.push(format!("{label}: an empty upsert advanced t")); + } + (0, 0) + } else { + (receipt.retract_count, receipt.assert_count) + }; + if counts != case.counts { + failures.push(format!( + "{label}: (retracts, asserts) = {counts:?}, expected {:?}", + case.counts + )); + } + let expect: Vec = case.expect.iter().map(ToString::to_string).collect(); + let got = h.facts(case.graph, "s").await; + if got != expect { + failures.push(format!( + "{label}: after upsert {got:?}, expected {expect:?}" + )); + } + h.reindex().await; + let got = h.facts(case.graph, "s").await; + if got != expect { + failures.push(format!( + "{label}: after the next index build {got:?}, expected {expect:?}" + )); + } + } + failures +} + +async fn run_all(cases: &[Case]) { + let mut failures = Vec::new(); + for case in cases { + failures.extend(run(case).await); + } + assert!(failures.is_empty(), "\n{}", failures.join("\n")); +} + +fn lang(v: &str, tag: &str) -> JsonValue { + json!({"@value": v, "@language": tag}) +} + +#[tokio::test] +async fn upsert_replaces_every_language_of_a_slot() { + run_all(&[ + Case { + name: "jsonld-lang", + seed: vec![ + jsonld(json!({"@id": "ex:s", "ex:label": lang("a", "en"), "ex:other": "keep"})), + jsonld(json!({"@id": "ex:s", "ex:label": lang("a", "fr")})), + ], + upsert: jsonld(json!({"@id": "ex:s", "ex:label": lang("b", "en")})), + graph: None, + expect: vec!["label=b@en", "other=keep"], + counts: (2, 1), + }, + Case { + name: "turtle-lang", + seed: vec![ + ttl("ex:s ex:label \"a\"@en ; ex:other \"keep\" ."), + ttl("ex:s ex:label \"a\"@fr ."), + ], + upsert: ttl("ex:s ex:label \"b\"@en ."), + graph: None, + expect: vec!["label=b@en", "other=keep"], + counts: (2, 1), + }, + // A payload in one language replaces the others too: the unit is + // the whole (graph, subject, predicate) slot. + Case { + name: "partial-language", + seed: vec![jsonld(json!({ + "@id": "ex:s", + "ex:label": [lang("a", "en"), lang("a", "fr"), lang("a", "de")] + }))], + upsert: jsonld(json!({"@id": "ex:s", "ex:label": lang("b", "fr")})), + graph: None, + expect: vec!["label=b@fr"], + counts: (3, 1), + }, + ]) + .await; +} + +#[tokio::test] +async fn upsert_replaces_a_list_as_stored() { + run_all(&[ + // Turtle keeps list duplicates; the upsert retracts each position. + Case { + name: "duplicates", + seed: vec![ttl("ex:s ex:items ( \"a\" \"a\" \"b\" ) .")], + upsert: jsonld(json!({"@id": "ex:s", "ex:items": {"@list": ["c"]}})), + graph: None, + expect: vec!["items=c#0"], + counts: (3, 1), + }, + // Unchanged positions cancel: one retraction, one assertion. + Case { + name: "overlap", + seed: vec![jsonld( + json!({"@id": "ex:s", "ex:items": {"@list": ["a", "b"]}}), + )], + upsert: jsonld(json!({"@id": "ex:s", "ex:items": {"@list": ["a", "c"]}})), + graph: None, + expect: vec!["items=a#0", "items=c#1"], + counts: (1, 1), + }, + Case { + name: "plain-to-list", + seed: vec![jsonld(json!({"@id": "ex:s", "ex:items": ["a", "b"]}))], + upsert: jsonld(json!({"@id": "ex:s", "ex:items": {"@list": ["a"]}})), + graph: None, + expect: vec!["items=a#0"], + counts: (2, 1), + }, + Case { + name: "list-to-plain", + seed: vec![jsonld( + json!({"@id": "ex:s", "ex:items": {"@list": ["a", "b"]}}), + )], + upsert: jsonld(json!({"@id": "ex:s", "ex:items": "a"})), + graph: None, + expect: vec!["items=a"], + counts: (2, 1), + }, + Case { + name: "lang-list", + seed: vec![jsonld(json!({ + "@id": "ex:s", + "ex:items": {"@list": [lang("x", "en"), lang("y", "fr")]} + }))], + upsert: jsonld(json!({"@id": "ex:s", "ex:items": {"@list": ["z"]}})), + graph: None, + expect: vec!["items=z#0"], + counts: (2, 1), + }, + ]) + .await; +} + +#[tokio::test] +async fn an_identical_refresh_commits_nothing() { + run_all(&[ + Case { + name: "refresh-jsonld", + seed: vec![jsonld(json!({ + "@id": "ex:s", + "ex:label": [lang("a", "en"), lang("a", "fr")], + "ex:items": {"@list": ["x", "x", "y"]}, + "ex:note": "same" + }))], + upsert: jsonld(json!({ + "@id": "ex:s", + "ex:label": [lang("a", "en"), lang("a", "fr")], + "ex:items": {"@list": ["x", "x", "y"]}, + "ex:note": "same" + })), + graph: None, + expect: vec![ + "items=x#0", + "items=x#1", + "items=y#2", + "label=a@en", + "label=a@fr", + "note=same", + ], + counts: (0, 0), + }, + Case { + name: "refresh-turtle", + seed: vec![ttl("ex:s ex:label \"a\"@en, \"a\"@fr ; ex:note \"same\" .")], + upsert: ttl("ex:s ex:label \"a\"@en, \"a\"@fr ; ex:note \"same\" ."), + graph: None, + expect: vec!["label=a@en", "label=a@fr", "note=same"], + counts: (0, 0), + }, + ]) + .await; +} + +#[tokio::test] +async fn upsert_replaces_stored_values_in_a_named_graph() { + run_all(&[ + // Node-level `@graph` inside an envelope: a top-level `@graph` + // string would read as the txn-meta envelope. + Case { + name: "named-jsonld", + seed: vec![ + jsonld(json!([{ + "@id": "ex:s", "@graph": G1, + "ex:label": lang("a", "en"), + "ex:items": {"@list": ["x", "y"]} + }])), + jsonld(json!([{"@id": "ex:s", "@graph": G1, "ex:label": lang("a", "fr")}])), + ], + upsert: jsonld(json!([{ + "@id": "ex:s", "@graph": G1, + "ex:label": lang("b", "en"), + "ex:items": {"@list": ["y"]} + }])), + graph: Some(G1), + expect: vec!["items=y#0", "label=b@en"], + counts: (4, 2), + }, + Case { + name: "named-trig", + seed: vec![ + ttl(&format!("GRAPH <{G1}> {{ ex:s ex:label \"a\"@en . }}")), + ttl(&format!("GRAPH <{G1}> {{ ex:s ex:label \"a\"@fr . }}")), + ], + upsert: ttl(&format!("GRAPH <{G1}> {{ ex:s ex:label \"b\"@en . }}")), + graph: Some(G1), + expect: vec!["label=b@en"], + counts: (2, 1), + }, + ]) + .await; +} + +/// An identical re-upsert of a document holding blank nodes commits +/// nothing: the payload-scoped skolem id maps each blank node to the node +/// the first upsert stored, and the wave replaces its values like any other +/// subject's. Before, blank subjects were left out of the wave, so their +/// triples were re-asserted on every refresh. +#[tokio::test] +async fn an_identical_refresh_with_blank_nodes_commits_nothing() { + for (name, payload) in [ + ( + "bnode-turtle", + ttl("ex:s ex:p [ ex:q \"x\" ; ex:r [ ex:q \"y\" ] ] ."), + ), + ( + "bnode-jsonld", + jsonld(json!({"@id": "ex:s", "ex:p": {"ex:q": "x", "ex:r": {"ex:q": "y"}}})), + ), + ] { + let mut h = Harness::new(name).await; + let first = h.upsert(&payload).await; + assert_eq!(first.assert_count, 4, "{name}: first upsert"); + let t = h.ledger().t(); + let again = h.upsert(&payload).await; + assert_eq!( + (again.flake_count, h.ledger().t()), + (0, t), + "{name}: identical refresh in novelty" + ); + h.reindex().await; + let t = h.ledger().t(); + let again = h.upsert(&payload).await; + assert_eq!( + (again.flake_count, h.ledger().t()), + (0, t), + "{name}: identical refresh over the index" + ); + } +} + +/// Replacing an annotated language-tagged value retracts the edge as +/// stored, which lets the annotation cascade find and retract the edge's +/// `f:reifies*` bundle. With the tag dropped the retraction named no stored +/// edge, so both the old value and its annotation survived. +#[tokio::test] +async fn upsert_over_an_annotated_lang_value_cascades_the_annotation() { + for indexed in [false, true] { + let mut h = Harness::new(&format!("annotated-{indexed}")).await; + h.insert(&jsonld(json!({ + "@id": "ex:s", + "ex:label": { + "@value": "a", "@language": "en", + "@annotation": {"ex:source": "wiki"} + } + }))) + .await; + if indexed { + h.reindex().await; + } + let receipt = h + .upsert(&jsonld(json!({"@id": "ex:s", "ex:label": lang("b", "en")}))) + .await; + assert_eq!( + h.facts(None, "s").await, + vec!["label=b@en"], + "indexed={indexed}" + ); + // The edge, its bundle and the anonymous annotation's body. + assert!( + receipt.retract_count > 1, + "indexed={indexed}: the annotation cascade must retract the bundle \ + (retracts={})", + receipt.retract_count + ); + let annotations = + support::decode_annotations_for_subject(h.ledger(), 0, &format!("{EX}s")).await; + assert!( + annotations.is_empty(), + "indexed={indexed}: the retracted edge keeps no annotation: {annotations:?}" + ); + } +} + +/// U5: a big integer typed with an XSD subtype loses its declared datatype +/// in the index (`NUM_BIG` stores no datatype), so after an index build an +/// identical re-upsert stages a retraction of the decoded `xsd:integer` +/// value that shares one storage key with the assertion of the declared +/// one, and the retraction wins: the value is deleted. The retraction +/// resolver reads what the index can tell it, which does not include the +/// declared subtype; the fix belongs to the index encoding. +#[tokio::test] +#[ignore = "U5: the index drops the declared subtype of NUM_BIG values (filing held, AJ-4)"] +async fn an_identical_reupsert_keeps_an_indexed_big_integer_subtype() { + let mut h = Harness::new("u5").await; + let doc = jsonld(json!({ + "@context": {"ex": EX, "xsd": "http://www.w3.org/2001/XMLSchema#"}, + "@id": "ex:s", + "ex:n": {"@value": "18446744073709551615", "@type": "xsd:unsignedLong"} + })); + h.upsert(&doc).await; + h.reindex().await; + h.upsert(&doc).await; + assert_eq!(h.facts(None, "s").await.len(), 1, "the value survives"); +} diff --git a/fluree-db-transact/src/generate/flakes.rs b/fluree-db-transact/src/generate/flakes.rs index fc75cdbe80..4e6bd19df3 100644 --- a/fluree-db-transact/src/generate/flakes.rs +++ b/fluree-db-transact/src/generate/flakes.rs @@ -491,6 +491,16 @@ impl<'a> FlakeGenerator<'a> { } } + /// The Sid an upsert's blank node `label` skolemizes to. + /// + /// Upsert has no WHERE, so its templates are instantiated for exactly one + /// solution — index 0 — and this is the Sid that solution's assertions + /// carry. The upsert wave uses it to find the stored values of a blank + /// subject whose skolem scope is deterministic. + pub(crate) fn upsert_blank_subject(&mut self, label: &str) -> Sid { + self.skolemize_blank_node(label, 0) + } + /// Skolemize a blank node to a Sid. /// /// Creates a unique Sid for a blank node label within this transaction and diff --git a/fluree-db-transact/src/stage.rs b/fluree-db-transact/src/stage.rs index 0f0273f427..26ebaf9aed 100644 --- a/fluree-db-transact/src/stage.rs +++ b/fluree-db-transact/src/stage.rs @@ -8,6 +8,7 @@ //! When the `shacl` feature is enabled, [`validate_view_with_shacl`] validates a //! staged view against SHACL shapes. +use crate::current_facts::{CurrentFacts, Retraction, Slot}; use crate::error::{Result, TransactError}; use crate::generate::{infer_datatype, FlakeAccumulator, FlakeGenerator}; use crate::ir::InlineValues; @@ -31,8 +32,7 @@ use fluree_db_policy::{ }; use fluree_db_query::parse::{lower_unresolved_patterns, UnresolvedPattern}; use fluree_db_query::{ - Batch, Binding, Pattern, QueryPolicyEnforcer, QueryPolicyExecutor, Ref, Term, TriplePattern, - VarId, VarRegistry, + Batch, Binding, Pattern, QueryPolicyEnforcer, QueryPolicyExecutor, VarId, VarRegistry, }; use fluree_db_sparql::ast::{ QueryBody as SparqlQueryBody, SelectClause, SelectQuery, SolutionModifiers, SparqlAst, @@ -892,18 +892,20 @@ pub async fn stage_with_graph_delta( // WHERE that yields no rows therefore correctly inserts nothing — we do // NOT fall back to a synthetic empty solution here. - // Upsert second wave: retractions derived from direct ledger lookups - // (not WHERE). These flakes already carry correct `m` from the - // underlying asserted flakes, so no hydration is needed. + // Upsert second wave: retract the current values of every slot the + // payload names. `CurrentFacts` reads them from storage, so each + // retraction is a stored fact with its `m` (language tag, list + // position) intact. if txn.txn_type == TxnType::Upsert { tracing::debug!("generating upsert deletions"); let upsert_retractions = - generate_upsert_deletions(&ledger, &txn, new_t, &graph_sids).await?; + generate_upsert_deletions(&ledger, &txn, new_t, &graph_sids, &mut generator) + .await?; tracing::debug!( upsert_retraction_count = upsert_retractions.len(), "upsert deletions generated" ); - acc.push_retractions(upsert_retractions); + acc.push_retractions(upsert_retractions.into_iter().map(Retraction::into_flake)); } // Graph-sync wave: push every currently-asserted flake of the target @@ -3255,58 +3257,6 @@ pub fn generate_txn_id() -> String { format!("{now:x}") } -/// Convert a Binding to a (FlakeValue, datatype Sid) pair for flake generation -/// -/// Returns `None` for non-materializable bindings (Unbound, Poisoned, Grouped, Iri). -/// This is used when generating retraction flakes from query results. -/// -/// When a `Materializer` is provided, encoded bindings (`EncodedLit`, `EncodedSid`) -/// are decoded via the binary index store before conversion; a value that cannot be -/// decoded is an error, since skipping it would leave the old value unretracted. -/// Without a materializer, encoded bindings return `None` (this can cause upsert to -/// silently skip retractions for values that live in the binary index — see issue #88). -fn binding_to_flake_object( - binding: &Binding, - materializer: Option<&mut fluree_db_query::Materializer>, -) -> Result> { - Ok(match binding { - Binding::Sid { sid, .. } => Some((FlakeValue::Ref(sid.clone()), Sid::new(1, "id"))), - Binding::IriMatch { primary_sid, .. } => { - Some((FlakeValue::Ref(primary_sid.clone()), Sid::new(1, "id"))) - } - Binding::Lit { val, dtc, .. } => Some((val.clone(), dtc.datatype().clone())), - Binding::EncodedLit { .. } | Binding::EncodedSid { .. } | Binding::EncodedPid { .. } => { - match materializer { - Some(mat) => binding_to_flake_object(&mat.to_term(binding)?, None)?, - None => None, - } - } - // Non-materializable bindings - Binding::Unbound | Binding::Poisoned => None, - Binding::Grouped(_) => { - debug_assert!( - false, - "Grouped binding encountered in flake generation (unexpected)" - ); - None - } - Binding::Path { .. } | Binding::Rel(_) | Binding::List(_) | Binding::Map(_) => { - debug_assert!( - false, - "Path/List binding encountered in flake generation (unexpected)" - ); - None - } - Binding::Iri(_) => { - debug_assert!( - false, - "Raw IRI binding cannot be materialized to flake (no SID)" - ); - None - } - }) -} - /// Convert a TemplateTerm to a Binding for VALUES clause fn template_term_to_binding(term: &TemplateTerm) -> Result { match term { @@ -3335,36 +3285,27 @@ fn inline_values_to_pattern(values: &InlineValues) -> Result { Ok(Pattern::Values { vars, rows: rows? }) } -/// Generate deletions for Upsert transactions -/// -/// For each (subject, predicate, graph) tuple with concrete SIDs in the insert templates, -/// query existing values and generate retractions for them. This implements the -/// "replace mode" semantics of Upsert. +/// The slots an upsert replaces, grouped by `(subject, graph IRI)`. /// -/// Subjects absent from both the persisted subject dictionary and novelty are -/// skipped without any index query: they cannot have existing values. This -/// matters because a bound-subject scan for a subject the dictionaries can't -/// resolve degrades to a full PSOT predicate-partition walk with per-row IRI -/// decoding (`unresolved_bound_subject_iri` in `BinaryScanOperator`) — for bulk -/// upserts of new entities that turned staging into minutes of work producing -/// zero retractions. +/// **The replacement unit.** An upsert replaces every current value of each +/// `(graph, subject, predicate)` its payload names — all language tags and +/// all list positions together — with the payload's values. This is the +/// one place that decides that unit (`docs/transactions/upsert.md`). /// -/// Named graph support: retractions are created in the same graph as the insert -/// templates to ensure proper cancellation with assertions. -async fn generate_upsert_deletions( - ledger: &LedgerState, +/// Subjects are the payload's IRIs, plus its blank nodes when the skolem +/// scope is deterministic (`txn.opts.skolem_txn_id`: the upsert payload +/// scope, or a caller-supplied id). Such a blank node skolemizes to the same +/// Sid every time the same payload is upserted, so its stored values are +/// replaced like any other subject's and an identical refresh is a no-op. A +/// per-transaction scope mints fresh Sids that cannot have stored values, +/// so those are left out. Upsert has no WHERE, so its templates are +/// instantiated for exactly one solution, and `upsert_blank_subject` mints +/// the Sid that solution's assertions carry. +fn upsert_replacement_slots( txn: &Txn, - new_t: i64, - graph_sids: &HashMap, -) -> Result> { - use fluree_db_binary_index::BinaryGraphView; - use fluree_db_core::{Flake, IndexType}; - use fluree_db_query::materializer::JoinKeyMode; - use fluree_db_query::{BinaryRangeProvider, Materializer}; - - // Group deduplicated predicates by (subject, graph IRI) so subject - // existence is resolved once per subject rather than once per (subject, - // predicate). + generator: &mut FlakeGenerator<'_>, +) -> HashMap<(Sid, Option>), Vec> { + let deterministic_blank_scope = txn.opts.skolem_txn_id.is_some(); let mut subject_groups: HashMap<(Sid, Option>), Vec> = HashMap::new(); for template in &txn.insert_templates { let graph = match &template.graph { @@ -3374,34 +3315,63 @@ async fn generate_upsert_deletions( // stored values to replace. TemplateGraph::Var(_) => continue, }; - if let (TemplateTerm::Sid(s), TemplateTerm::Sid(p)) = - (&template.subject, &template.predicate) - { - subject_groups - .entry((s.clone(), graph)) - .or_default() - .push(p.clone()); - } - // Variables and blank nodes are skipped - we can't query for them + let TemplateTerm::Sid(p) = &template.predicate else { + continue; + }; + let subject = match &template.subject { + TemplateTerm::Sid(s) => s.clone(), + TemplateTerm::BlankNode(label) if deterministic_blank_scope => { + generator.upsert_blank_subject(label) + } + // Variables, and blank nodes minted fresh by this transaction. + _ => continue, + }; + subject_groups + .entry((subject, graph)) + .or_default() + .push(p.clone()); } for predicates in subject_groups.values_mut() { predicates.sort_unstable(); predicates.dedup(); } + subject_groups +} + +/// The upsert wave: retract every current value of each slot the payload +/// names (see [`upsert_replacement_slots`]), read from storage by +/// [`CurrentFacts`] so each retraction is a stored fact — language tag, +/// list position and all. The payload's assertions cancel the retractions +/// of values it restates, so an identical refresh commits nothing. +/// +/// Subjects absent from both the persisted subject dictionary and novelty +/// are skipped without any read: they cannot have values. Bulk upserts of +/// new entities are exactly that shape, and the check is two dictionary +/// probes per subject plus at most one novelty walk per graph (#1549). +/// +/// Retractions carry the graph Sid of the payload's graph, so they cancel +/// the payload's assertions in that graph. +async fn generate_upsert_deletions( + ledger: &LedgerState, + txn: &Txn, + new_t: i64, + graph_sids: &HashMap, + generator: &mut FlakeGenerator<'_>, +) -> Result> { + use fluree_db_core::IndexType; + use fluree_db_query::BinaryRangeProvider; + let subject_groups = upsert_replacement_slots(txn, generator); if subject_groups.is_empty() { return Ok(Vec::new()); } - // Extract the binary index store and DictNovelty (if present) so we can - // materialize EncodedLit/EncodedSid bindings returned by the binary scan path. - let brp_ref = ledger + let binary_store = ledger .snapshot .range_provider .as_ref() - .and_then(|rp| rp.as_any().downcast_ref::()); - let binary_store = brp_ref.map(|brp| Arc::clone(brp.store())); - let dict_novelty = brp_ref.map(|brp| Arc::clone(brp.dict_novelty())); + .and_then(|rp| rp.as_any().downcast_ref::()) + .map(|brp| Arc::clone(brp.store())); // Ledger graph id per graph IRI. None in the value position means the // graph is not yet in the ledger registry (new graph in this txn), so @@ -3428,17 +3398,18 @@ async fn generate_upsert_deletions( // system RAM when indexing lags — which is why it runs at most once per // graph, and only on demand. It is authoritative in Sid space, needing // no dictionary translation. - // Genesis, or nothing indexed yet: novelty is the only place a subject can - // exist, so the presence check below is authoritative on its own. + // + // Genesis, or nothing indexed yet: novelty is the only place a subject + // can exist, so the presence check below is authoritative on its own. // // The `t == 0` conjunct mirrors `fluree_db_core::range`, which treats a // missing range provider as an empty index only at genesis and errors // otherwise ("binary-only db has no range_provider attached"). A binary - // store that fails to load is non-fatal in the ledger manager, which leaves - // an indexed ledger (`t > 0`) with no provider attached; subjects there DO - // have base rows we cannot see, so absence must stay undecidable and the - // per-predicate query must run — that path surfaces the load failure - // instead of silently skipping every retraction. + // store that fails to load is non-fatal in the ledger manager, which + // leaves an indexed ledger (`t > 0`) with no provider attached; subjects + // there DO have base rows we cannot see, so absence must stay + // undecidable and the read must run — that path surfaces the load + // failure instead of silently skipping every retraction. let base_index_absent = ledger.snapshot.range_provider.is_none() && ledger.snapshot.t == 0; let can_decide_absence = binary_store.is_some() || base_index_absent; @@ -3452,11 +3423,11 @@ async fn generate_upsert_deletions( } let mut novelty_present: HashMap> = HashMap::new(); - // Persisted presence: the subject reverse dictionary is authoritative under - // canonical namespace encoding (see `fluree_db_core::ns_encoding`) — a miss - // on both the (ns_code, suffix) key and the full-IRI key means the subject - // has no rows in the base index. Lookup errors fall back to "present" so the - // per-predicate query surfaces the real failure. + // Persisted presence: the subject reverse dictionary is authoritative + // under canonical namespace encoding (see `fluree_db_core::ns_encoding`) + // — a miss on both the (ns_code, suffix) key and the full-IRI key means + // the subject has no rows in the base index. Lookup errors fall back to + // "present" so the read surfaces the real failure. let subject_in_base = |subject: &Sid| -> bool { let Some(store) = binary_store.as_deref() else { // Only reachable under `base_index_absent` (the caller gates on @@ -3484,22 +3455,17 @@ async fn generate_upsert_deletions( // Reporting "present" here instead defeats the skip for every IRI shape // that mints a namespace per subject — `MostGranular` splits // `urn:…::r:` at the last `:` — sending each one down a - // per-(subject, predicate) degraded scan. Novelty presence is still - // checked by the caller, so a subject that exists only in unindexed - // commits is never wrongly skipped. + // per-(subject, predicate) read. Novelty presence is still checked by + // the caller, so a subject that exists only in unindexed commits is + // never wrongly skipped. match ledger.snapshot.decode_sid(subject) { Some(iri) => !matches!(store.find_subject_id(&iri), Ok(None)), None => false, } }; - let mut retractions = Vec::new(); + let mut slots: Vec = Vec::new(); let mut skipped_subjects = 0usize; - let mut pattern_queries = 0usize; - - // Query existing values for each (subject, predicate, graph) tuple - let mut query_vars = VarRegistry::new(); - let o_var = query_vars.get_or_insert("?o"); for ((subject, graph_id), predicates) in &subject_groups { let ledger_g_id: Option = ledger_g_for_txn_g.get(graph_id).copied().flatten(); @@ -3551,137 +3517,28 @@ async fn generate_upsert_deletions( } } - // Create a materializer for this graph context if a binary store exists. - // BinaryGraphView::with_novelty handles watermark routing internally, - // so novelty-only string/subject IDs resolve correctly. - let mut materializer = binary_store.as_ref().map(|store| { - let view = BinaryGraphView::with_novelty( - Arc::clone(store), - effective_g_id, - dict_novelty.clone(), - ); - Materializer::new(view, JoinKeyMode::SingleLedger) - }); - - for predicate in predicates { - pattern_queries += 1; - // Query: ?o - let pattern = TriplePattern::new( - Ref::Sid(subject.clone()), - Ref::Sid(predicate.clone()), - Term::Var(o_var), - ); - - let batches = if graph_id.is_some() { - if ledger.snapshot.range_provider.is_some() { - fluree_db_query::execute_pattern( - ledger.as_graph_db_ref(effective_g_id), - &query_vars, - pattern, - ) - .await? - } else { - // No binary store available (genesis / not indexed): scan novelty directly. - query_novelty_for_graph(ledger, subject, predicate, effective_g_id, o_var) - } - } else { - // Default graph: use standard query path through range_provider - fluree_db_query::execute_pattern(ledger.as_graph_db_ref(0), &query_vars, pattern) - .await? - }; - - for batch in &batches { - for row in 0..batch.len() { - let flake_obj = match batch.get(row, o_var) { - Some(b) => binding_to_flake_object(b, materializer.as_mut())?, - None => None, - }; - if let Some((o, dt)) = flake_obj { - let flake = match graph_sid.clone() { - Some(g) => Flake::new_in_graph( - g, - subject.clone(), - predicate.clone(), - o, - dt, - new_t, - false, // retraction - None, - ), - None => Flake::new( - subject.clone(), - predicate.clone(), - o, - dt, - new_t, - false, // retraction - None, - ), - }; - retractions.push(flake); - } - } - } - } + slots.extend(predicates.iter().map(|p| Slot { + g_id: effective_g_id, + g_sid: graph_sid.clone(), + s: subject.clone(), + p: p.clone(), + })); } + // `pattern_queries` keeps its name: it counts slot reads. tracing::debug!( subject_count = subject_groups.len(), skipped_subjects, - pattern_queries, + pattern_queries = slots.len(), "upsert deletion subject pre-check" ); - Ok(retractions) -} - -/// Query novelty directly for a specific named graph -/// -/// This function scans the novelty overlay for flakes matching the given -/// subject, predicate, and graph context. It's used for named graph upserts -/// because the db.range_provider is scoped to the default graph (g_id=0). -fn query_novelty_for_graph( - ledger: &LedgerState, - subject: &Sid, - predicate: &Sid, - target_g_id: u16, - o_var: VarId, -) -> Vec { - use fluree_db_core::IndexType; - - // Collect matching flakes from novelty for the target graph - let mut matching_values = Vec::new(); - ledger.novelty.for_each_overlay_flake( - target_g_id, - IndexType::Spot, - None, - None, - true, - ledger.t(), - &mut |flake| { - // Check if flake matches (subject, predicate) and is an assertion - if &flake.s == subject && &flake.p == predicate && flake.op { - matching_values.push((flake.o.clone(), flake.dt.clone())); - } - }, - ); - - // Convert to batch format - if matching_values.is_empty() { - return Vec::new(); - } - - // Create a simple batch with just the object values - let schema: Arc<[VarId]> = Arc::new([o_var]); - let mut o_col = Vec::with_capacity(matching_values.len()); - for (o, dt) in &matching_values { - o_col.push(Binding::from_object(o.clone(), dt.clone())); - } - - match Batch::new(schema, vec![o_col]) { - Ok(batch) => vec![batch], - Err(_) => Vec::new(), - } + let facts = CurrentFacts::new(ledger).of_slots(&slots).await?; + Ok(facts + .into_iter() + .flatten() + .map(|fact| fact.retract(new_t)) + .collect()) } /// Per-graph SHACL policy — how a specific graph's violations should be From 06983c373bb7cf57bef11238eff6e8dbfa456d2a Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 01:33:01 -0400 Subject: [PATCH 05/36] refactor(transact): graph sync and CLEAR/COPY/MOVE retract through CurrentFacts The whole-graph scan moves from scan_graph_flakes to CurrentFacts::whole_graph, which returns StoredFacts with the graph stamped. Graph sync, CLEAR/DROP and the COPY/MOVE destination and source clears now build their retractions with StoredFact::retract instead of flipping op and t on scanned flakes. Behavior is unchanged: the scan, its limit, the stamping and the policy model are the same. --- fluree-db-transact/src/stage.rs | 143 +++++++++++--------------------- 1 file changed, 49 insertions(+), 94 deletions(-) diff --git a/fluree-db-transact/src/stage.rs b/fluree-db-transact/src/stage.rs index 26ebaf9aed..7a26280dd8 100644 --- a/fluree-db-transact/src/stage.rs +++ b/fluree-db-transact/src/stage.rs @@ -8,7 +8,7 @@ //! When the `shacl` feature is enabled, [`validate_view_with_shacl`] validates a //! staged view against SHACL shapes. -use crate::current_facts::{CurrentFacts, Retraction, Slot}; +use crate::current_facts::{CurrentFacts, Retraction, Slot, StoredFact}; use crate::error::{Result, TransactError}; use crate::generate::{infer_datatype, FlakeAccumulator, FlakeGenerator}; use crate::ir::InlineValues; @@ -19,8 +19,6 @@ use crate::ir::{ use crate::namespace::NamespaceRegistry; use fluree_db_core::comparator::IndexType; use fluree_db_core::graph_registry::{FIRST_USER_GRAPH_ID, TXN_META_GRAPH_ID}; -use fluree_db_core::query_bounds::RangeTest; -use fluree_db_core::range::RangeMatch; use fluree_db_core::tracking::schedule::TXN_BASELINE_MICRO_FUEL; use fluree_db_core::OverlayProvider; use fluree_db_core::Tracker; @@ -908,13 +906,11 @@ pub async fn stage_with_graph_delta( acc.push_retractions(upsert_retractions.into_iter().map(Retraction::into_flake)); } - // Graph-sync wave: push every currently-asserted flake of the target - // graph as a retraction (see [`Txn::sync_graph`]). The accumulator - // nets retract+assert of the same fact to nothing, so what survives - // `finalize()` is exactly `current − payload` retractions plus - // `payload − current` assertions — the delta. Scanned flakes carry - // correct `m` from storage, so (like the upsert wave) no hydration - // is needed. + // Graph-sync wave: retract every current fact of the target graph + // (see [`Txn::sync_graph`]). The accumulator nets retract+assert of + // the same fact to nothing, so what survives `finalize()` is exactly + // `current − payload` retractions plus `payload − current` + // assertions — the delta. // // Policy model follows CLEAR (roadmap O4): the scan is not // view-policy filtered — sync is an authoritative whole-graph @@ -926,28 +922,24 @@ pub async fn stage_with_graph_delta( // graph's flakes at staging time; backpressure is the pre-check // above plus `NoveltyWouldExceed` sizing at commit (which sees only // the surviving delta). Chunked staging for whole-graph ops is the - // same known follow-up flagged on `scan_graph_flakes`. + // known follow-up flagged on `whole_graph_facts`. if let Some((sync_g_id, sync_graph_sid)) = &sync_scan { - // The scan attributes every flake to the graph Sid, matching the + // Every fact comes back attributed to the graph Sid, matching the // payload's assertions — both sides must agree on `flake.g` for // the accumulator's unchanged-fact cancellation to fire. - let mut sync_retractions = scan_graph_flakes( + let current = whole_graph_facts( &ledger, *sync_g_id, sync_graph_sid.as_ref(), options.tracker, ) .await?; - for f in &mut sync_retractions { - f.op = false; - f.t = new_t; - } tracing::debug!( graph_id = sync_g_id, - scanned = sync_retractions.len(), + scanned = current.len(), "graph-sync retractions generated" ); - acc.push_retractions(sync_retractions); + acc.push_retractions(current.into_iter().map(|f| f.retract(new_t).into_flake())); } let retraction_count = stream_stats.retraction_count; @@ -1309,22 +1301,9 @@ fn whole_graph_scan_limit() -> Option { } } -/// Scan every currently-asserted flake in graph `g_id` (merged snapshot + -/// novelty view as of the ledger's current `t`), attributed to `g_sid`. -/// -/// Every flake comes back with `g = g_sid` (`None` for the default graph). -/// The range provider materializes index-resident rows with `g: None` -/// regardless of graph — only novelty-resident flakes carry it — and every -/// caller here routes by `flake.g` (`resolve_flake_graph_id`, where `None` -/// is the default graph). Without the stamp, retracting an indexed named -/// graph silently retracted phantoms from the default graph instead. -/// -/// Scale note: a whole-graph operation (`CLEAR ALL`, a large COPY/MOVE) -/// materializes every scanned flake into a `Vec` and re-stages it, and -/// backpressure (`at_max_novelty`) is only checked at commit entry — so one -/// graph-management op can roughly double novelty in a single commit. That is -/// exactly the op class most likely to touch the whole store; chunked staging -/// for whole-graph ops is a known follow-up if this cliff is hit in practice. +/// Every current fact of graph `g_id`, attributed to `g_sid`, under the +/// [`whole_graph_scan_limit`] backstop. The graph-management verbs +/// (CLEAR/DROP/COPY/MOVE/ADD) and graph sync read through this. /// /// O4 (by design): this scan is NOT view-policy filtered — unlike the /// DELETE-WHERE path, which reads through a `QueryPolicyEnforcer`. So the set a @@ -1336,41 +1315,22 @@ fn whole_graph_scan_limit() -> Option { /// not a privilege escalation; it only means that under `default_allow` + a /// view restriction, a `CLEAR` can retract flakes an equivalent DELETE-WHERE /// (which only sees viewable rows) would not. -async fn scan_graph_flakes( +/// +/// Scale note: a whole-graph operation (`CLEAR ALL`, a large COPY/MOVE) +/// materializes every scanned flake into a `Vec` and re-stages it, and +/// backpressure (`at_max_novelty`) is only checked at commit entry — so one +/// graph-management op can roughly double novelty in a single commit. That is +/// exactly the op class most likely to touch the whole store; chunked staging +/// for whole-graph ops is a known follow-up if this cliff is hit in practice. +async fn whole_graph_facts( ledger: &LedgerState, g_id: GraphId, g_sid: Option<&Sid>, tracker: Option<&Tracker>, -) -> Result> { - let db_ref = match tracker { - Some(t) => ledger.as_graph_db_ref(g_id).with_tracker(t), - None => ledger.as_graph_db_ref(g_id), - }; - // `Eq` with an empty match is the whole-graph scan on both range paths: - // the V3 provider treats "nothing bound" as a full-index cursor and - // rejects every other `RangeTest`, and the genesis (overlay-only) path - // matches an empty `Eq` against every flake. `Ge` only ever worked on - // the genesis path, where non-`Eq` tests pass through unfiltered. - // `flake_limit` stops the provider's drain loop mid-scan, so the - // backstop bounds what is materialized, not just what is returned. - let limit = whole_graph_scan_limit(); - let opts = fluree_db_core::RangeOptions { - flake_limit: limit.map(|l| l.saturating_add(1)), - ..Default::default() - }; - let mut flakes = db_ref - .range_with_opts(IndexType::Spot, RangeTest::Eq, RangeMatch::new(), opts) +) -> Result> { + CurrentFacts::new(ledger) + .whole_graph(g_id, g_sid, whole_graph_scan_limit(), tracker) .await - .map_err(|e| TransactError::FlakeGeneration(format!("graph scan failed: {e}")))?; - if let Some(l) = limit { - if flakes.len() > l { - return Err(TransactError::WholeGraphScanTooLarge { limit: l }); - } - } - for f in &mut flakes { - f.g = g_sid.cloned(); - } - Ok(flakes) } /// Resolve the ledger `GraphId` and graph `Sid` for a named graph IRI, if it @@ -1490,13 +1450,9 @@ async fn stage_graph_mgmt( if let Some(sid) = &sid { graph_sids.insert(g_id, sid.clone()); } - for mut f in - scan_graph_flakes(&ledger, g_id, sid.as_ref(), options.tracker).await? - { - f.op = false; - f.t = new_t; - flakes.push(f); - } + let current = + whole_graph_facts(&ledger, g_id, sid.as_ref(), options.tracker).await?; + flakes.extend(current.into_iter().map(|f| f.retract(new_t).into_flake())); } } @@ -1600,19 +1556,22 @@ async fn stage_graph_mgmt( graph_sids.insert(dest_g_id, sid.clone()); } - let src_flakes = match src_g_id { + let src_facts = match src_g_id { Some(g) => { - scan_graph_flakes(&ledger, g, src_sid.as_ref(), options.tracker).await? + whole_graph_facts(&ledger, g, src_sid.as_ref(), options.tracker).await? } None => Vec::new(), }; + let src_flakes: Vec<&Flake> = src_facts.iter().map(StoredFact::flake).collect(); - let dest_flakes = - scan_graph_flakes(&ledger, dest_g_id, dest_sid.as_ref(), options.tracker) + let dest_facts = + whole_graph_facts(&ledger, dest_g_id, dest_sid.as_ref(), options.tracker) .await?; - let dest_contents: HashSet = - dest_flakes.iter().map(flake_content).collect(); + let dest_contents: HashSet = dest_facts + .iter() + .map(|f| flake_content(f.flake())) + .collect(); // O5: re-homing rewrites the `f:reifiesGraph` anchor per src/ // dest graph (see the assert loop below), so an @@ -1638,7 +1597,7 @@ async fn stage_graph_mgmt( // produce a `Duplicate`; same signature (same edge) dedups // cleanly against `dest_contents` and is fine. let edge_signatures = - |bundle: &[Flake]| -> HashMap> { + |bundle: &mut dyn Iterator| -> HashMap> { let mut sigs: HashMap> = HashMap::new(); for f in bundle { if fluree_db_core::is_reserved_reifies_predicate(&f.p) @@ -1651,9 +1610,10 @@ async fn stage_graph_mgmt( } sigs }; - let dest_sigs = edge_signatures(&dest_flakes); + let dest_sigs = + edge_signatures(&mut dest_facts.iter().map(StoredFact::flake)); if !dest_sigs.is_empty() { - let src_sigs = edge_signatures(&src_flakes); + let src_sigs = edge_signatures(&mut src_flakes.iter().copied()); for (subject, src_sig) in &src_sigs { if let Some(dest_sig) = dest_sigs.get(subject) { if src_sig != dest_sig { @@ -1700,7 +1660,7 @@ async fn stage_graph_mgmt( }; let mut rehomed: Vec = Vec::with_capacity(src_flakes.len() + synthesized); - for f in &src_flakes { + for &f in &src_flakes { if fluree_db_core::is_reifies_graph(&f.p) { if let Some(dest) = &dest_sid { let mut a = f.clone(); @@ -1721,7 +1681,7 @@ async fn stage_graph_mgmt( } if src_is_default { if let Some(dest) = &dest_sid { - for f in &src_flakes { + for &f in &src_flakes { if fluree_db_core::is_reifies_subject(&f.p) { rehomed.push(Flake::new_in_graph( dest.clone(), @@ -1758,12 +1718,9 @@ async fn stage_graph_mgmt( // is the intended behavior, so this no longer silently loses data // on a typo. if *clear_dest { - for f in &dest_flakes { - if !rehomed_contents.contains(&flake_content(f)) { - let mut r = f.clone(); - r.op = false; - r.t = new_t; - flakes.push(r); + for fact in dest_facts { + if !rehomed_contents.contains(&flake_content(fact.flake())) { + flakes.push(fact.retract(new_t).into_flake()); } } } @@ -1781,13 +1738,11 @@ async fn stage_graph_mgmt( // destination assertions, so there is no cancellation. if *clear_src { if let Some(src_g) = src_g_id { - for mut f in src_flakes { - if let Some(g_sid) = &f.g { + for fact in src_facts { + if let Some(g_sid) = &fact.flake().g { graph_sids.entry(src_g).or_insert_with(|| g_sid.clone()); } - f.op = false; - f.t = new_t; - flakes.push(f); + flakes.push(fact.retract(new_t).into_flake()); } } } From fa5476bb51c35e31904fa0969dc38a716b669703 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 01:58:35 -0400 Subject: [PATCH 06/36] fix(transact): a DELETE retracts only facts that are stored MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A DELETE template instantiated into a retraction of a fact the ledger does not hold used to be committed anyway. DELETE DATA of an absent triple committed a new t with one retraction, which docs/transactions/retractions.md says is a no-op. Worse, DELETE {x} INSERT {x} WHERE {} with x absent committed nothing and never wrote x: the phantom retraction cancelled the insert in the accumulator, against SPARQL 1.1 Update's (DS - D) ∪ I. Both twins (SPARQL and JSON-LD) behaved the same. Every DELETE instantiation is now a RetractIntent, and the accumulator accepts only Retraction, which CurrentFacts produces: - An intent whose template a WHERE triple witnesses (delete_witness: same subject, predicate and object variable, the object bound nowhere else, the same graph) is the decode of a fact the WHERE just matched. On a ledger that holds no @list position (ListFree) it retracts as decoded with no read, which keeps bulk filtered deletes at zero lookups. On a list-bearing ledger it reads its slot for the list position, under the rule hydration had: the first stored list entry with the same term. - Every other intent (template constants, DELETE DATA, values bound by another pattern) is matched against the stored facts of its slot: exact datatype (#1990's rule, one function), case-insensitive language tag, the named list position or else the first list entry, else the plain value. An intent that names no stored fact stages nothing. hydrate_list_index_meta_for_retractions is gone, and so is the unused generate::cancellation module, which pushed arbitrary flakes as retractions. The delete_witnessed routing stamp reports which lane ran. The it_fast_stats_1391 fixture relied on a DELETE writing a no-op retraction; it now hand-builds that commit, the shape legacy and replayed commits still carry. --- fluree-db-api/Cargo.toml | 7 + fluree-db-api/tests/grp_transact.rs | 2 + fluree-db-api/tests/it_delete_stored_facts.rs | 412 +++++++++++++++++ .../tests/it_delete_witness_routing.rs | 116 +++++ .../tests/it_fast_stats_1391_regression.rs | 66 ++- fluree-db-transact/src/current_facts.rs | 236 +++++++++- fluree-db-transact/src/delete_witness.rs | 424 ++++++++++++++++++ .../src/generate/accumulator.rs | 58 ++- .../src/generate/cancellation.rs | 251 ----------- fluree-db-transact/src/generate/flakes.rs | 64 ++- fluree-db-transact/src/generate/mod.rs | 5 +- fluree-db-transact/src/lib.rs | 8 +- fluree-db-transact/src/stage.rs | 233 ++++------ 13 files changed, 1416 insertions(+), 466 deletions(-) create mode 100644 fluree-db-api/tests/it_delete_stored_facts.rs create mode 100644 fluree-db-api/tests/it_delete_witness_routing.rs create mode 100644 fluree-db-transact/src/delete_witness.rs delete mode 100644 fluree-db-transact/src/generate/cancellation.rs diff --git a/fluree-db-api/Cargo.toml b/fluree-db-api/Cargo.toml index 956cb30da8..6624802727 100644 --- a/fluree-db-api/Cargo.toml +++ b/fluree-db-api/Cargo.toml @@ -421,6 +421,13 @@ path = "tests/it_minmax_fast_path_fired.rs" name = "it_upsert_skip_fires" path = "tests/it_upsert_skip_fires.rs" +# Standalone (not in grp_transact): asserts on the `delete_witnessed` routing +# stamp via the thread-local span capture, so it must be the only test in its +# process. Same convention as `it_upsert_skip_fires`. +[[test]] +name = "it_delete_witness_routing" +path = "tests/it_delete_witness_routing.rs" + # Standalone (NOT in grp_ledger): sets FLUREE_MAX_GRAPH_SCAN_FLAKES to exercise # the whole-graph-scan cap, then clears it. The limit is read from the process # env per scan and has no programmatic override, so a grouped binary — whose diff --git a/fluree-db-api/tests/grp_transact.rs b/fluree-db-api/tests/grp_transact.rs index 94bf0129b2..d058a9daf2 100644 --- a/fluree-db-api/tests/grp_transact.rs +++ b/fluree-db-api/tests/grp_transact.rs @@ -7,6 +7,8 @@ mod it_cached_handle_cow_cancel; mod it_cached_handle_cow_recovery; #[path = "it_concurrent_update_reconcile.rs"] mod it_concurrent_update_reconcile; +#[path = "it_delete_stored_facts.rs"] +mod it_delete_stored_facts; #[path = "it_enforce_unique_upsert_indexed.rs"] mod it_enforce_unique_upsert_indexed; #[path = "it_jsonld_empty_list.rs"] diff --git a/fluree-db-api/tests/it_delete_stored_facts.rs b/fluree-db-api/tests/it_delete_stored_facts.rs new file mode 100644 index 0000000000..27005aa95d --- /dev/null +++ b/fluree-db-api/tests/it_delete_stored_facts.rs @@ -0,0 +1,412 @@ +//! A DELETE retracts only facts that are stored. +//! +//! A retraction of a fact that is not stored used to be committed anyway +//! (a "phantom"), with two effects: +//! - `DELETE DATA` of an absent triple committed a new `t` holding one +//! retraction, though `docs/transactions/retractions.md` says it is a +//! no-op; +//! - `DELETE {x} INSERT {x}` with `x` absent committed nothing and never +//! wrote `x`: the phantom retraction cancelled the insert, against SPARQL +//! 1.1 Update's `(DS − D) ∪ I`. +//! +//! Every DELETE intent that a WHERE triple does not witness is now matched +//! against the stored facts of its slot, and one that names none stages +//! nothing. Each case has SPARQL and JSON-LD twins (Cypher where its +//! lowering produces the shape), on a novelty-only ledger and an indexed +//! one. + +use crate::support; +use fluree_db_api::{Fluree, FlureeBuilder, LedgerState, TransactResult}; +use serde_json::{json, Value as JsonValue}; + +const EX: &str = "http://example.org/ns/"; +const G1: &str = "http://example.org/g1"; + +fn ctx() -> JsonValue { + json!({"ex": EX, "xsd": "http://www.w3.org/2001/XMLSchema#"}) +} + +async fn sparql_update(fluree: &Fluree, ledger: LedgerState, body: &str) -> TransactResult { + let text = + format!("PREFIX ex: <{EX}>\nPREFIX xsd: \n{body}"); + let parsed = fluree_db_sparql::parse_sparql(&text); + assert!(!parsed.has_errors(), "{text}: {:?}", parsed.diagnostics); + let mut ns = fluree_db_transact::NamespaceRegistry::from_db(&ledger.snapshot); + let txn = fluree_db_transact::lower_sparql_update_ast( + &parsed.ast.expect("ast"), + &mut ns, + fluree_db_transact::TxnOpts::default(), + ) + .expect("lower"); + fluree + .stage_owned(ledger) + .txn(txn) + .execute() + .await + .expect("SPARQL update") +} + +async fn jsonld_update(fluree: &Fluree, ledger: LedgerState, mut txn: JsonValue) -> TransactResult { + txn["@context"] = ctx(); + fluree.update(ledger, &txn).await.expect("JSON-LD update") +} + +/// `SELECT ?o` over `?s ?p ?o` for `ex:s`, as `p=value[@lang]`, sorted. +async fn facts(fluree: &Fluree, ledger: &LedgerState) -> Vec { + let q = format!( + "SELECT ?p ?v ?l WHERE {{ <{EX}s> ?p ?o BIND(STR(?o) AS ?v) BIND(LANG(?o) AS ?l) }}" + ); + let result = support::query_sparql(fluree, ledger, &q) + .await + .expect("query"); + let json = result.to_jsonld(&ledger.snapshot).expect("jsonld"); + let cell = |c: &JsonValue| { + c.as_str() + .or_else(|| c.get("@id").and_then(JsonValue::as_str)) + .or_else(|| c.get("@value").and_then(JsonValue::as_str)) + .map(str::to_string) + .unwrap_or_else(|| c.to_string()) + }; + let mut rows: Vec = json + .as_array() + .expect("rows") + .iter() + .map(|r| { + let r = r.as_array().expect("row"); + let p = cell(&r[0]); + let p = p.rsplit(['/', '#', ':']).next().unwrap_or(&p).to_string(); + let lang = cell(&r[2]); + if lang.is_empty() || lang == "null" { + format!("{p}={}", cell(&r[1])) + } else { + format!("{p}={}@{lang}", cell(&r[1])) + } + }) + .collect(); + rows.sort(); + rows +} + +/// A file-backed ledger holding `seed`, optionally indexed. +async fn seeded( + name: &str, + seed: JsonValue, + indexed: bool, +) -> (tempfile::TempDir, Fluree, LedgerState) { + let dir = tempfile::tempdir().expect("tempdir"); + let fluree = FlureeBuilder::file(dir.path().to_string_lossy().to_string()) + .build() + .expect("file fluree"); + let id = format!("it/delete-stored-{name}-{indexed}:main"); + let ledger = fluree.create_ledger(&id).await.expect("create"); + let mut doc = seed; + doc["@context"] = ctx(); + let ledger = fluree.insert(ledger, &doc).await.expect("seed").ledger; + if !indexed { + return (dir, fluree, ledger); + } + drop(ledger); + support::rebuild_and_publish_index(&fluree, &id).await; + let ledger = fluree.ledger(&id).await.expect("reload"); + assert!(ledger.snapshot.range_provider.is_some()); + (dir, fluree, ledger) +} + +fn no_commit(r: &TransactResult, t_before: i64, what: &str) { + assert_eq!( + (r.receipt.flake_count, r.ledger.t()), + (0, t_before), + "{what}: expected no commit" + ); +} + +#[tokio::test] +async fn deleting_an_absent_fact_commits_nothing() { + for indexed in [false, true] { + let seed = json!({"@id": "ex:s", "ex:p": "present"}); + + let (_d, fluree, ledger) = seeded("absent-sparql", seed.clone(), indexed).await; + let t = ledger.t(); + let r = sparql_update(&fluree, ledger, "DELETE DATA { ex:s ex:p \"absent\" }").await; + no_commit(&r, t, &format!("SPARQL DELETE DATA (indexed={indexed})")); + assert_eq!(facts(&fluree, &r.ledger).await, ["p=present"]); + + let (_d, fluree, ledger) = seeded("absent-jsonld", seed.clone(), indexed).await; + let t = ledger.t(); + let r = jsonld_update( + &fluree, + ledger, + json!({"delete": {"@id": "ex:s", "ex:p": "absent"}}), + ) + .await; + no_commit(&r, t, &format!("JSON-LD delete (indexed={indexed})")); + + // A retraction of a present fact next to an absent one retracts + // only the present one. + let (_d, fluree, ledger) = seeded("absent-mixed", seed.clone(), indexed).await; + let r = sparql_update( + &fluree, + ledger, + "DELETE DATA { ex:s ex:p \"absent\" . ex:s ex:p \"present\" }", + ) + .await; + assert_eq!( + (r.receipt.retract_count, r.receipt.assert_count), + (1, 0), + "indexed={indexed}" + ); + assert!(facts(&fluree, &r.ledger).await.is_empty()); + + // Cypher `REMOVE n:Label` retracts `rdf:type Label`, a constant. + let (_d, fluree, ledger) = seeded("absent-cypher", seed.clone(), indexed).await; + let ledger = fluree + .transact_cypher(ledger, "CREATE (n:Person {name: 'n'})") + .await + .expect("cypher CREATE") + .ledger; + let t = ledger.t(); + let r = fluree + .transact_cypher(ledger, "MATCH (n:Person {name: 'n'}) REMOVE n:Missing") + .await + .expect("cypher REMOVE"); + no_commit( + &r, + t, + &format!("Cypher REMOVE of an absent label (indexed={indexed})"), + ); + } +} + +#[tokio::test] +async fn delete_insert_of_an_absent_fact_inserts_it() { + for indexed in [false, true] { + let seed = json!({"@id": "ex:s", "ex:other": "o"}); + + let (_d, fluree, ledger) = seeded("di-sparql", seed.clone(), indexed).await; + let r = sparql_update( + &fluree, + ledger, + "DELETE { ex:s ex:p \"x\" } INSERT { ex:s ex:p \"x\" } WHERE {}", + ) + .await; + assert_eq!( + facts(&fluree, &r.ledger).await, + ["other=o", "p=x"], + "SPARQL, x absent (indexed={indexed})" + ); + // x present now: the same update is a no-op. + let t = r.ledger.t(); + let r = sparql_update( + &fluree, + r.ledger, + "DELETE { ex:s ex:p \"x\" } INSERT { ex:s ex:p \"x\" } WHERE {}", + ) + .await; + no_commit(&r, t, &format!("SPARQL, x present (indexed={indexed})")); + assert_eq!(facts(&fluree, &r.ledger).await, ["other=o", "p=x"]); + + let (_d, fluree, ledger) = seeded("di-jsonld", seed.clone(), indexed).await; + let txn = json!({ + "delete": {"@id": "ex:s", "ex:p": "x"}, + "insert": {"@id": "ex:s", "ex:p": "x"} + }); + let r = jsonld_update(&fluree, ledger, txn.clone()).await; + assert_eq!( + facts(&fluree, &r.ledger).await, + ["other=o", "p=x"], + "JSON-LD, x absent (indexed={indexed})" + ); + let t = r.ledger.t(); + let r = jsonld_update(&fluree, r.ledger, txn).await; + no_commit(&r, t, &format!("JSON-LD, x present (indexed={indexed})")); + } +} + +#[tokio::test] +async fn delete_data_names_terms_exactly() { + for indexed in [false, true] { + // A language tag compares case-insensitively; the stored tag is the + // one retracted. + let (_d, fluree, ledger) = seeded( + "lang", + json!({"@id": "ex:s", "ex:label": {"@value": "a", "@language": "en"}}), + indexed, + ) + .await; + let r = sparql_update(&fluree, ledger, "DELETE DATA { ex:s ex:label \"a\"@EN }").await; + assert!( + facts(&fluree, &r.ledger).await.is_empty(), + "indexed={indexed}" + ); + + // The datatype is part of the term: an xsd:int is not deleted by an + // xsd:integer of the same value. + let (_d, fluree, ledger) = seeded( + "dt", + json!({"@id": "ex:s", "ex:n": {"@value": "1", "@type": "xsd:int"}}), + indexed, + ) + .await; + let t = ledger.t(); + let r = sparql_update(&fluree, ledger, "DELETE DATA { ex:s ex:n 1 }").await; + no_commit( + &r, + t, + &format!("xsd:integer vs stored xsd:int (indexed={indexed})"), + ); + assert_eq!(facts(&fluree, &r.ledger).await, ["n=1"]); + // The same term deletes it. (Spelled in JSON-LD: SPARQL UPDATE lowers + // `"1"^^xsd:int` to a string value, a separate gap that deletes + // nothing before or after this change.) + let r = jsonld_update( + &fluree, + r.ledger, + json!({"delete": {"@id": "ex:s", "ex:n": {"@value": "1", "@type": "xsd:int"}}}), + ) + .await; + assert!( + facts(&fluree, &r.ledger).await.is_empty(), + "indexed={indexed}" + ); + } +} + +/// SPARQL `DELETE DATA` of typed literals whose datatype SPARQL UPDATE does +/// not coerce today (`xsd:int`, `xsd:long`, `xsd:dateTime`): the lowering +/// keeps the lexical form as a string, so the intent names no stored term. +/// Before the retraction resolver that committed phantom retractions that +/// deleted nothing; with it alone nothing is committed and the values still +/// stay. #1988 lowers these literals through the same coercion JSON-LD uses, +/// after which the intent names the stored term and the resolver deletes it. +#[tokio::test] +#[ignore = "needs #1988: SPARQL UPDATE keeps typed literal values"] +async fn delete_data_of_typed_literals_deletes_them() { + for indexed in [false, true] { + let (_d, fluree, ledger) = seeded( + "typed", + json!({ + "@id": "ex:s", + "ex:n": {"@value": "1", "@type": "xsd:int"}, + "ex:l": {"@value": "5", "@type": "xsd:long"}, + "ex:d": {"@value": "2020-01-01T00:00:00Z", "@type": "xsd:dateTime"} + }), + indexed, + ) + .await; + let r = sparql_update( + &fluree, + ledger, + "DELETE DATA { ex:s ex:n \"1\"^^xsd:int . ex:s ex:l \"5\"^^xsd:long . \ + ex:s ex:d \"2020-01-01T00:00:00Z\"^^xsd:dateTime }", + ) + .await; + assert_eq!(r.receipt.retract_count, 3, "indexed={indexed}"); + assert!( + facts(&fluree, &r.ledger).await.is_empty(), + "indexed={indexed}" + ); + } +} + +/// The SPARQL and JSON-LD twins of the upsert-over-languages case: a +/// same-shape DELETE/INSERT WHERE replaces every language, because each +/// WHERE row is the decode of the stored fact, tag included. +#[tokio::test] +async fn delete_insert_where_replaces_every_language() { + for indexed in [false, true] { + let seed = json!({ + "@id": "ex:s", + "ex:label": [{"@value": "a", "@language": "en"}, {"@value": "a", "@language": "fr"}] + }); + let (_d, fluree, ledger) = seeded("langs-sparql", seed.clone(), indexed).await; + let r = sparql_update( + &fluree, + ledger, + "DELETE { ex:s ex:label ?o } INSERT { ex:s ex:label \"b\"@en } WHERE { ex:s ex:label ?o }", + ) + .await; + assert_eq!( + facts(&fluree, &r.ledger).await, + ["label=b@en"], + "indexed={indexed}" + ); + + let (_d, fluree, ledger) = seeded("langs-jsonld", seed, indexed).await; + let r = jsonld_update( + &fluree, + ledger, + json!({ + "where": {"@id": "ex:s", "ex:label": "?o"}, + "delete": {"@id": "ex:s", "ex:label": "?o"}, + "insert": {"@id": "ex:s", "ex:label": {"@value": "b", "@language": "en"}} + }), + ) + .await; + assert_eq!( + facts(&fluree, &r.ledger).await, + ["label=b@en"], + "indexed={indexed}" + ); + } +} + +/// A big integer typed with an XSD subtype decodes from the index with a +/// lossy datatype (`NUM_BIG` stores none). A witnessed DELETE WHERE row +/// carries that decode and still deletes the value through the shared +/// storage key, as before. On a list-bearing ledger the row also goes +/// through the resolver for list positions; this pins that the exact +/// datatype rule does not drop it there. +async fn numbig_rows(fluree: &Fluree, ledger: &LedgerState) -> usize { + let q = format!("PREFIX ex: <{EX}>\nSELECT ?o WHERE {{ GRAPH <{G1}> {{ ?s ex:n ?o }} }}"); + let r = support::query_sparql(fluree, ledger, &q).await.expect("q"); + r.to_jsonld(&ledger.snapshot) + .expect("jsonld") + .as_array() + .map_or(0, Vec::len) +} + +#[tokio::test] +async fn witnessed_delete_of_an_indexed_big_integer_in_a_named_graph() { + for with_list in [false, true] { + let mut graph = vec![json!({ + "@id": "ex:s", "@graph": G1, + "ex:n": {"@value": "18446744073709551615", "@type": "xsd:unsignedLong"} + })]; + if with_list { + graph.push(json!({"@id": "ex:other", "ex:items": {"@list": ["a", "b"]}})); + } + for surface in ["sparql", "jsonld"] { + let (_d, fluree, ledger) = seeded( + &format!("numbig-{surface}-{with_list}"), + json!({"@graph": graph.clone()}), + true, + ) + .await; + assert_eq!(ledger.snapshot.has_list_meta, Some(with_list)); + assert_eq!(numbig_rows(&fluree, &ledger).await, 1, "precondition"); + let r = if surface == "sparql" { + sparql_update( + &fluree, + ledger, + &format!("DELETE WHERE {{ GRAPH <{G1}> {{ ?s ex:n ?o }} }}"), + ) + .await + } else { + jsonld_update( + &fluree, + ledger, + json!({ + "where": [["graph", G1, {"@id": "?s", "ex:n": "?o"}]], + "delete": [["graph", G1, {"@id": "?s", "ex:n": "?o"}]] + }), + ) + .await + }; + assert_eq!( + numbig_rows(&fluree, &r.ledger).await, + 0, + "{surface}, with_list={with_list}" + ); + } + } +} diff --git a/fluree-db-api/tests/it_delete_witness_routing.rs b/fluree-db-api/tests/it_delete_witness_routing.rs new file mode 100644 index 0000000000..e03fc0de54 --- /dev/null +++ b/fluree-db-api/tests/it_delete_witness_routing.rs @@ -0,0 +1,116 @@ +//! The witnessed-DELETE lane must actually fire, and stand down when the +//! ledger can hold a list position. +//! +//! A DELETE whose template a WHERE triple witnesses (`DELETE { ?s ex:p ?o } +//! WHERE { ?s ex:p ?o }`) retracts its rows as decoded, with no stored-fact +//! read, when neither the index nor novelty holds an `@list` position; on a +//! list-bearing ledger the rows read their slots for list positions. The +//! answers agree either way, so only the `delete_witnessed` routing stamp +//! can tell which lane ran. Both directions are asserted on the same ledger: +//! `MustFire` while it is list-free, `MustNotFire` after one `@list` lands. +//! +//! Its own `[[test]]` binary, like `it_upsert_skip_fires`: tracing's +//! callsite-interest cache is process-global, so a sibling test in a bundled +//! binary can leave the stamp's callsite disabled. + +#[path = "support/span_capture.rs"] +mod span_capture; + +use fluree_db_api::FlureeBuilder; +use serde_json::{json, Value as JsonValue}; + +fn ctx() -> JsonValue { + json!({"ex": "http://example.org/ns/"}) +} + +fn witnessed_delete(subject: &str) -> JsonValue { + json!({ + "@context": ctx(), + "where": {"@id": subject, "ex:tag": "?o"}, + "delete": {"@id": subject, "ex:tag": "?o"} + }) +} + +#[tokio::test(flavor = "current_thread")] +async fn delete_witnessed_lane_fires_only_on_a_list_free_ledger() { + let (store, _guard) = span_capture::init_test_tracing(); + let site = fluree_db_transact::DELETE_WITNESSED_SITE; + let outcomes = |from: usize| -> Vec { + store.find_events("fast-path outcome")[from..] + .iter() + .filter(|e| e.fields.get("site").map(String::as_str) == Some(site)) + .filter_map(|e| e.fields.get("outcome").cloned()) + .collect() + }; + + let fluree = FlureeBuilder::memory().build_memory(); + let ledger = fluree + .create_ledger("it/delete-witness-routing:main") + .await + .expect("create"); + let ledger = fluree + .insert( + ledger, + &json!({ + "@context": ctx(), + "@graph": [ + {"@id": "ex:a", "ex:tag": "x"}, + {"@id": "ex:b", "ex:tag": "y"}, + {"@id": "ex:c", "ex:tag": "z"} + ] + }), + ) + .await + .expect("seed") + .ledger; + assert!(!ledger.novelty.has_list_meta); + + // MustFire: list-free. + let before = store.find_events("fast-path outcome").len(); + let r = fluree + .update(ledger, &witnessed_delete("ex:a")) + .await + .expect("witnessed delete"); + assert_eq!(r.receipt.retract_count, 1); + assert_eq!( + outcomes(before), + ["proceed"], + "a witnessed delete on a list-free ledger must take the zero-read lane" + ); + + // A DELETE DATA-shaped delete has no witness and stamps nothing. + let before = store.find_events("fast-path outcome").len(); + let r = fluree + .update( + r.ledger, + &json!({"@context": ctx(), "delete": {"@id": "ex:b", "ex:tag": "y"}}), + ) + .await + .expect("constant delete"); + assert_eq!(r.receipt.retract_count, 1); + assert!(outcomes(before).is_empty(), "no witness, no stamp"); + + // One @list on the same ledger turns the lane off. + let ledger = fluree + .insert( + r.ledger, + &json!({"@context": ctx(), "@id": "ex:l", "ex:items": {"@list": ["a", "b"]}}), + ) + .await + .expect("list") + .ledger; + assert!(ledger.novelty.has_list_meta); + + // MustNotFire: list-bearing. + let before = store.find_events("fast-path outcome").len(); + let r = fluree + .update(ledger, &witnessed_delete("ex:c")) + .await + .expect("witnessed delete, list-bearing"); + assert_eq!(r.receipt.retract_count, 1, "the delete still retracts"); + assert_eq!( + outcomes(before), + ["fallback:gate_declined"], + "a list-bearing ledger must read list positions" + ); +} diff --git a/fluree-db-api/tests/it_fast_stats_1391_regression.rs b/fluree-db-api/tests/it_fast_stats_1391_regression.rs index b798050f15..0a7403ad0c 100644 --- a/fluree-db-api/tests/it_fast_stats_1391_regression.rs +++ b/fluree-db-api/tests/it_fast_stats_1391_regression.rs @@ -110,27 +110,69 @@ fn novelty_doc() -> Value { }) } -/// The third commit, covering the *under*-count half of #1391. Novelty accepts -/// every retraction — `apply_commit`'s dedup gate short-circuits on -/// `flake.op &&` and never examines one — so a ground DELETE that matches -/// nothing still lands as a retraction flake and, under a blind delta log, -/// subtracts one from a count that never included it. -/// -/// * `ex:w2 ex:name "never-existed"` matches nothing: must charge zero; -/// * `ex:w1 ex:size 1` is a real base fact: must charge `-1`, taking -/// `ex:size`'s ledger-wide count to zero so the property drops out of -/// `apoc.meta.data` and `db.propertyKeys()` entirely. +/// The third commit, covering the *under*-count half of #1391: `ex:w1 +/// ex:size 1` is a real base fact and must charge `-1`, taking `ex:size`'s +/// ledger-wide count to zero so the property drops out of `apoc.meta.data` +/// and `db.propertyKeys()` entirely. fn deletion_doc() -> Value { json!({ "@context": {"ex": "http://example.org/ns/"}, "where": {}, "delete": [ - {"@id": "ex:w2", "ex:name": "never-existed"}, {"@id": "ex:w1", "ex:size": 1} ] }) } +/// The fourth commit: a retraction of `ex:w2 ex:name "never-existed"`, which +/// matches nothing and must charge zero. Novelty accepts every retraction — +/// `apply_commit`'s dedup gate short-circuits on `flake.op &&` and never +/// examines one — so under a blind delta log it subtracts one from a count +/// that never included it. +/// +/// A transaction no longer writes such a retraction (a DELETE of an absent +/// fact stages nothing), but commits written before that, and commits +/// replayed from elsewhere, still carry them. So the commit is built by hand +/// and staged the way a replayed commit is. +async fn commit_noop_retraction( + fluree: &Fluree, + ledger: fluree_db_api::LedgerState, + index_config: &IndexConfig, +) { + use fluree_db_core::{Flake, FlakeValue, Sid}; + let s = ledger + .snapshot + .encode_iri("http://example.org/ns/w2") + .expect("ex:w2"); + let p = ledger + .snapshot + .encode_iri("http://example.org/ns/name") + .expect("ex:name"); + let xsd_string = Sid::new(fluree_vocab::namespaces::XSD, "string"); + let t = ledger.t() + 1; + let noop = Flake::new( + s, + p, + FlakeValue::String("never-existed".into()), + xsd_string, + t, + false, + None, + ); + let ns = fluree_db_transact::NamespaceRegistry::from_db(&ledger.snapshot); + let view = fluree_db_transact::stage_flakes( + ledger, + vec![noop], + fluree_db_transact::StageOptions::new(), + ) + .await + .expect("stage the no-op retraction"); + fluree + .commit_staged(view, ns, index_config, CommitOpts::default()) + .await + .expect("commit the no-op retraction"); +} + /// `(label, property)` pairs the schema shims report, each paired with the /// SPARQL that counts the same facts by scanning. The hand-pinned number is /// the current-state truth; both the stats answer and the scan must equal it. @@ -275,6 +317,8 @@ async fn setup() -> (tempfile::TempDir, Fluree) { ) .await .expect("deletion update"); + let ledger = fluree.ledger(ALIAS).await.expect("reload after deletion"); + commit_noop_retraction(&fluree, ledger, &index_config).await; let ledger = fluree.ledger(ALIAS).await.expect("reload after deletion"); let retractions = ledger diff --git a/fluree-db-transact/src/current_facts.rs b/fluree-db-transact/src/current_facts.rs index ba4c304f72..11d695eaec 100644 --- a/fluree-db-transact/src/current_facts.rs +++ b/fluree-db-transact/src/current_facts.rs @@ -114,6 +114,133 @@ impl Retraction { pub fn into_flake(self) -> Flake { self.0 } + + /// Any flake, for unit tests of the retraction consumers. + #[cfg(test)] + pub(crate) fn for_test(flake: Flake) -> Self { + Retraction(flake) + } +} + +/// What one DELETE template instantiation asks to retract: the instantiated +/// `(g, s, p, o, dt, m)` at the transaction's `t`. Not a retraction — +/// [`CurrentFacts::resolve_intents`] turns it into one only if it names a +/// stored fact, or it is a WHERE row proven to be the decode of one. +#[derive(Clone, Debug)] +pub(crate) struct RetractIntent { + flake: Flake, + /// The template is witnessed by a WHERE triple (`delete_witness`), so + /// the row is the decode of a fact that WHERE matched at the same `t`. + witnessed: bool, +} + +impl RetractIntent { + pub(crate) fn new(flake: Flake, witnessed: bool) -> Self { + debug_assert!(!flake.op, "a retraction intent is op = false"); + Self { flake, witnessed } + } + + #[cfg(test)] + pub(crate) fn flake(&self) -> &Flake { + &self.flake + } +} + +/// Proof that no `@list` position is stored anywhere in the ledger: the +/// persisted index recorded none (`Some(false)` is an exact observation by +/// the indexer; `None`, a legacy root or a bulk import, proves nothing) and +/// novelty holds none. Only [`ListFree::check`] makes one. +pub(crate) struct ListFree(()); + +impl ListFree { + pub(crate) fn check(ledger: &LedgerState) -> Option { + (ledger.snapshot.has_list_meta == Some(false) && !ledger.novelty.has_list_meta) + .then_some(ListFree(())) + } +} + +/// What resolving a batch of DELETE intents did, for tracing. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct ResolveStats { + /// Witnessed rows retracted as decoded, with no read. + pub(crate) witnessed: usize, + /// Witnessed rows on a list-bearing ledger, given the stored list + /// position of the first matching entry. + pub(crate) enriched: usize, + /// Intents matched to a stored fact. + pub(crate) matched: usize, + /// Intents that named no stored fact and staged nothing. + pub(crate) phantom_intents: usize, + /// `(graph, subject, predicate)` slots read. + pub(crate) slot_reads: usize, +} + +impl ResolveStats { + pub(crate) fn add(&mut self, other: ResolveStats) { + self.witnessed += other.witnessed; + self.enriched += other.enriched; + self.matched += other.matched; + self.phantom_intents += other.phantom_intents; + self.slot_reads += other.slot_reads; + } +} + +/// Datatype identity for a DELETE term: exact. A DELETE names a term, and +/// RDF term identity includes the datatype, so `DELETE DATA { … 1 }` does +/// not remove `"1"^^xsd:int`. The one place this rule lives. +fn dt_matches(stored: &Sid, intent: &Sid) -> bool { + stored == intent +} + +/// Language tags compare case-insensitively (RDF 1.1): the retraction then +/// carries the stored tag verbatim, which is what a pre-normalization tag +/// in the index needs. +fn lang_matches(stored: Option<&str>, intent: Option<&str>) -> bool { + match (stored, intent) { + (None, None) => true, + (Some(a), Some(b)) => a.eq_ignore_ascii_case(b), + _ => false, + } +} + +fn lang_of(f: &Flake) -> Option<&str> { + f.m.as_ref().and_then(|m| m.lang.as_deref()) +} + +fn list_index_of(f: &Flake) -> Option { + f.m.as_ref().and_then(|m| m.i) +} + +/// The same RDF term: value, datatype and language tag. +fn same_term(stored: &Flake, intent: &Flake) -> bool { + stored.o == intent.o + && dt_matches(&stored.dt, &intent.dt) + && lang_matches(lang_of(stored), lang_of(intent)) +} + +/// The stored fact an intent names, if any. An intent with a list position +/// names that position. One without names the first list entry holding the +/// term, in index order, else the plain value: the contract hydration had, +/// under which a value asserted at N list positions loses exactly one entry +/// per distinct DELETE row. +fn match_stored<'f>(facts: &'f [StoredFact], intent: &Flake) -> Option<&'f StoredFact> { + let term = |f: &&StoredFact| same_term(&f.flake, intent); + match list_index_of(intent) { + Some(i) => facts + .iter() + .filter(|f| list_index_of(&f.flake) == Some(i)) + .find(term), + None => facts + .iter() + .filter(|f| list_index_of(&f.flake).is_some()) + .find(term) + .or_else(|| { + facts + .iter() + .filter(|f| list_index_of(&f.flake).is_none()) + .find(term) + }), + } } /// A `(graph, subject, predicate)` slot to read. @@ -228,6 +355,88 @@ impl<'l> CurrentFacts<'l> { .collect()) } + /// Resolve DELETE intents to retractions. + /// + /// - A witnessed row on a list-free ledger (`list_free`) is retracted as + /// decoded, with no read: the WHERE just matched that stored fact. + /// - A witnessed row on a list-bearing ledger reads its slot for the one + /// thing a WHERE binding does not carry, the list position: it takes + /// the position of the first stored list entry holding the same term, + /// and otherwise stays as decoded. + /// - Every other intent — a template constant, `DELETE DATA`, a value + /// bound by another pattern — is matched against the stored facts of + /// its slot ([`match_stored`]). An intent that names no stored fact + /// stages nothing. + /// + /// `graph_of` gives the ledger graph id of an intent's graph Sid. + pub(crate) async fn resolve_intents( + &self, + intents: Vec, + list_free: Option<&ListFree>, + graph_of: impl Fn(&Flake) -> Result, + ) -> Result<(Vec, ResolveStats)> { + let mut stats = ResolveStats::default(); + let mut out = Vec::with_capacity(intents.len()); + let mut pending = Vec::new(); + for intent in intents { + if intent.witnessed && list_free.is_some() { + stats.witnessed += 1; + out.push(Retraction(intent.flake)); + } else { + pending.push(intent); + } + } + if pending.is_empty() { + return Ok((out, stats)); + } + + let mut slot_of: HashMap<(GraphId, Sid, Sid), usize> = HashMap::new(); + let mut slots: Vec = Vec::new(); + let mut pending_slots = Vec::with_capacity(pending.len()); + for intent in &pending { + let f = &intent.flake; + let g_id = graph_of(f)?; + let idx = *slot_of + .entry((g_id, f.s.clone(), f.p.clone())) + .or_insert_with(|| { + slots.push(Slot { + g_id, + g_sid: f.g.clone(), + s: f.s.clone(), + p: f.p.clone(), + }); + slots.len() - 1 + }); + pending_slots.push(idx); + } + stats.slot_reads = slots.len(); + let facts = self.of_slots(&slots).await?; + + for (intent, idx) in pending.into_iter().zip(pending_slots) { + let slot_facts = &facts[idx]; + let t = intent.flake.t; + if intent.witnessed { + let mut flake = intent.flake; + if list_index_of(&flake).is_none() { + if let Some(entry) = slot_facts + .iter() + .find(|f| list_index_of(&f.flake).is_some() && same_term(&f.flake, &flake)) + { + flake.m = entry.flake.m.clone(); + stats.enriched += 1; + } + } + out.push(Retraction(flake)); + } else if let Some(stored) = match_stored(slot_facts, &intent.flake) { + stats.matched += 1; + out.push(stored.clone().retract(t)); + } else { + stats.phantom_intents += 1; + } + } + Ok((out, stats)) + } + /// Persisted-index rows matching `rm` in graph `g_id`, read through /// `NoOverlay`: one leaf seek, no novelty translation. At genesis (no /// index yet) there is no provider and the answer is empty. @@ -449,7 +658,14 @@ mod tests { flake("a", "items", "w", 1, true, Some(FlakeMeta::with_index(2))), flake("a", "other", "o", 1, true, None), ], - vec![flake("a", "items", "w", 2, false, Some(FlakeMeta::with_index(2)))], + vec![flake( + "a", + "items", + "w", + 2, + false, + Some(FlakeMeta::with_index(2)), + )], ], &HashMap::new(), ); @@ -508,8 +724,22 @@ mod tests { async fn seek_and_walk_agree() { let mut commit = Vec::new(); for i in 0..40 { - commit.push(flake(&format!("s{i:02}"), "p", &format!("v{i}"), 1, true, None)); - commit.push(flake(&format!("s{i:02}"), "q", &format!("w{i}"), 1, true, None)); + commit.push(flake( + &format!("s{i:02}"), + "p", + &format!("v{i}"), + 1, + true, + None, + )); + commit.push(flake( + &format!("s{i:02}"), + "q", + &format!("w{i}"), + 1, + true, + None, + )); } let retract = vec![flake("s07", "p", "v7", 2, false, None)]; let ledger = ledger(vec![commit, retract], &HashMap::new()); diff --git a/fluree-db-transact/src/delete_witness.rs b/fluree-db-transact/src/delete_witness.rs new file mode 100644 index 0000000000..1e898acc04 --- /dev/null +++ b/fluree-db-transact/src/delete_witness.rs @@ -0,0 +1,424 @@ +//! Which DELETE templates a WHERE triple witnesses. +//! +//! A DELETE template instantiated from a WHERE row usually names the very +//! fact a WHERE triple just matched: `DELETE { ?s ex:p ?o } WHERE { ?s ex:p +//! ?o }`. Such a row is the decode of a stored fact read at the same `t`, +//! so it may be retracted without reading the fact again — the zero-lookup +//! path bulk deletes depend on. Everything this analysis cannot prove goes +//! through the resolver, which retracts only facts it finds stored. +//! +//! A template `T` is **witnessed** by a WHERE triple `W` when: +//! - (a) `W` sits at the top level of the WHERE, or directly inside a +//! top-level `GRAPH`: never under OPTIONAL, UNION, MINUS, EXISTS, a +//! subquery, SERVICE or a property path; +//! - (b) `T`'s subject and predicate are `W`'s — the same variable or the +//! same Sid; +//! - (c) `T`'s object is a variable `v` and `W`'s object is the same `v`, +//! and `T` carries no datatype, language or list constraint of its own. +//! A constant object is never witnessed: matching is looser than term +//! identity (a bare number matches any numeric datatype); +//! - (d) nothing else in the WHERE mentions `v` — no other pattern, +//! VALUES, BIND or UNWIND — except a FILTER that reads `v` and no other +//! variable. A join across two patterns unifies literals looser than +//! term identity, so the row could carry the other fact's datatype or +//! language tag; a filter equating `v` with another variable could be +//! folded into exactly such a join; +//! - (e) `T` writes to the graph `W` read: the default-graph template with +//! the ledger's own default graph as the WHERE default; `GRAPH ` +//! templates with a WHERE default of exactly `g` (WITH, or a single +//! USING / JSON-LD `graph`/`from`) or with `W` inside `GRAPH `, `g` +//! a graph the ledger has; `GRAPH ?g` templates with `W` inside +//! `GRAPH ?g`. A `GRAPH ` whose name the WHERE dataset aliases to +//! another graph is never a witness: it reads that graph, while a +//! template naming the same IRI writes to the IRI's own graph. +//! +//! Subject and predicate are always nodes, and joins compare nodes +//! exactly, so (b) needs no condition like (d). + +use crate::ir::{TemplateGraph, TemplateTerm, TripleTemplate}; +use fluree_db_query::ir::GraphName; +use fluree_db_query::{Pattern, Ref, Term, TriplePattern, VarId}; + +/// The WHERE clause's default graph. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum WhereDefault<'a> { + /// No USING, WITH or JSON-LD `graph`/`from`: the ledger's default graph. + Ledger, + /// Exactly one graph, named by this IRI. + Graph(&'a str), + /// Anything else: several graphs, or none (USING NAMED alone). + Other, +} + +/// What the rule needs to know about the WHERE's dataset. +pub(crate) struct WitnessContext<'a> { + pub(crate) default: WhereDefault<'a>, + /// Whether the WHERE dataset resolves this graph name to a different + /// graph (a dataset alias). + pub(crate) aliased: &'a dyn Fn(&str) -> bool, + /// Whether an IRI names a graph the ledger has. + pub(crate) registered: &'a dyn Fn(&str) -> bool, +} + +/// Where a candidate witness reads. +#[derive(Clone, Copy)] +enum ReadGraph<'p> { + Default, + Named(&'p GraphName), +} + +/// A candidate witness and its position: top-level index, and index inside +/// a top-level `GRAPH` when it sits in one. +struct Candidate<'p> { + triple: &'p TriplePattern, + graph: ReadGraph<'p>, + at: (usize, Option), +} + +/// For each template, whether a WHERE triple witnesses it. +pub(crate) fn witnessed_templates( + patterns: &[Pattern], + templates: &[TripleTemplate], + cx: &WitnessContext<'_>, +) -> Vec { + let candidates = candidates(patterns); + templates + .iter() + .map(|t| candidates.iter().any(|w| witnesses(w, t, patterns, cx))) + .collect() +} + +fn candidates(patterns: &[Pattern]) -> Vec> { + let mut out = Vec::new(); + for (i, p) in patterns.iter().enumerate() { + match p { + Pattern::Triple(triple) => out.push(Candidate { + triple, + graph: ReadGraph::Default, + at: (i, None), + }), + Pattern::Graph { name, patterns } => { + for (j, inner) in patterns.iter().enumerate() { + if let Pattern::Triple(triple) = inner { + out.push(Candidate { + triple, + graph: ReadGraph::Named(name), + at: (i, Some(j)), + }); + } + } + } + _ => {} + } + } + out +} + +fn witnesses( + w: &Candidate<'_>, + t: &TripleTemplate, + patterns: &[Pattern], + cx: &WitnessContext<'_>, +) -> bool { + // (c): a variable object, the same variable, no constraint of T's own. + let (TemplateTerm::Var(v), Term::Var(wv)) = (&t.object, &w.triple.o) else { + return false; + }; + if v != wv || t.dtc.is_some() || t.list_index.is_some() { + return false; + } + // (b) + if !same_node(&t.subject, &w.triple.s) || !same_node(&t.predicate, &w.triple.p) { + return false; + } + // (e) + let same_graph = match (w.graph, &t.graph) { + (ReadGraph::Default, TemplateGraph::Default) => cx.default == WhereDefault::Ledger, + (ReadGraph::Default, TemplateGraph::Iri(g)) => { + cx.default == WhereDefault::Graph(g) && (cx.registered)(g) + } + (ReadGraph::Named(GraphName::Iri(read)), TemplateGraph::Iri(g)) => { + read == g && !(cx.aliased)(g) && (cx.registered)(g) + } + (ReadGraph::Named(GraphName::Var(read)), TemplateGraph::Var(g)) => read == g, + _ => false, + }; + // (d) + same_graph && !mentioned_elsewhere(*v, patterns, w.at) +} + +fn same_node(t: &TemplateTerm, r: &Ref) -> bool { + match (t, r) { + (TemplateTerm::Var(a), Ref::Var(b)) => a == b, + (TemplateTerm::Sid(a), Ref::Sid(b)) => a == b, + _ => false, + } +} + +/// Whether anything but the witness at `skip` mentions `v`, other than a +/// top-level FILTER reading `v` alone (see (d)). +fn mentioned_elsewhere(v: VarId, patterns: &[Pattern], skip: (usize, Option)) -> bool { + patterns.iter().enumerate().any(|(i, p)| match p { + Pattern::Triple(_) if skip == (i, None) => false, + Pattern::Graph { name, patterns } => { + matches!(name, GraphName::Var(g) if *g == v) + || patterns + .iter() + .enumerate() + .any(|(j, inner)| skip != (i, Some(j)) && mentions(inner, v)) + } + other => mentions(other, v), + }) +} + +fn mentions(p: &Pattern, v: VarId) -> bool { + match p { + Pattern::Filter(expr) => { + let vars = expr.referenced_vars(); + vars.contains(&v) && vars.iter().any(|x| *x != v) + } + other => other.referenced_vars().contains(&v), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use fluree_db_core::{DatatypeConstraint, FlakeValue, Sid}; + use fluree_db_query::{Expression, Function}; + use std::sync::Arc; + + const S: VarId = VarId(0); + const P: VarId = VarId(1); + const O: VarId = VarId(2); + const X: VarId = VarId(3); + const G: VarId = VarId(4); + const GRAPH_IRI: &str = "http://example.org/g1"; + + fn pred() -> Sid { + Sid::new(100, "p") + } + + fn triple(s: Ref, p: Ref, o: Term) -> Pattern { + Pattern::Triple(TriplePattern::new(s, p, o)) + } + + /// `?s ex:p ?o` + fn where_sp_o() -> Pattern { + triple(Ref::Var(S), Ref::Sid(pred()), Term::Var(O)) + } + + /// DELETE template `?s ex:p ?o`. + fn t_sp_o() -> TripleTemplate { + TripleTemplate::new( + TemplateTerm::Var(S), + TemplateTerm::Sid(pred()), + TemplateTerm::Var(O), + ) + } + + fn registered(iri: &str) -> bool { + iri == GRAPH_IRI + } + + fn not_aliased(_: &str) -> bool { + false + } + + fn cx(default: WhereDefault<'_>) -> WitnessContext<'_> { + WitnessContext { + default, + aliased: ¬_aliased, + registered: ®istered, + } + } + + fn check(patterns: &[Pattern], t: TripleTemplate, cx: &WitnessContext<'_>) -> bool { + witnessed_templates(patterns, &[t], cx)[0] + } + + #[test] + fn delete_where_and_same_shape_updates_are_witnessed() { + let ledger = cx(WhereDefault::Ledger); + assert!(check(&[where_sp_o()], t_sp_o(), &ledger)); + // DELETE WHERE { ?s ?p ?o } + let spo = triple(Ref::Var(S), Ref::Var(P), Term::Var(O)); + let t = TripleTemplate::new( + TemplateTerm::Var(S), + TemplateTerm::Var(P), + TemplateTerm::Var(O), + ); + assert!(check(&[spo], t, &ledger)); + // A join on the subject keeps it witnessed: nodes compare exactly. + let other = triple( + Ref::Var(S), + Ref::Sid(Sid::new(100, "q")), + Term::Value(FlakeValue::Long(1)), + ); + assert!(check(&[other, where_sp_o()], t_sp_o(), &ledger)); + // VALUES on the subject only. + let values = Pattern::Values { + vars: vec![S], + rows: vec![vec![fluree_db_query::Binding::sid(Sid::new(100, "a"))]], + }; + assert!(check(&[values, where_sp_o()], t_sp_o(), &ledger)); + // A FILTER that reads only the object. + let filter = Pattern::Filter(Expression::Call { + func: Function::Gt, + args: vec![Expression::Var(O), Expression::Const(FlakeValue::Long(5))], + }); + assert!(check(&[where_sp_o(), filter], t_sp_o(), &ledger)); + } + + #[test] + fn constants_and_constraints_are_not_witnessed() { + let ledger = cx(WhereDefault::Ledger); + // DELETE DATA: no WHERE at all. + assert!(!check(&[], t_sp_o(), &ledger)); + // A constant object. + let t = TripleTemplate::new( + TemplateTerm::Var(S), + TemplateTerm::Sid(pred()), + TemplateTerm::Value(FlakeValue::Long(1)), + ); + let w = triple( + Ref::Var(S), + Ref::Sid(pred()), + Term::Value(FlakeValue::Long(1)), + ); + assert!(!check(&[w], t, &ledger)); + // A template with its own language tag or list position. + let tagged = t_sp_o().with_dtc(DatatypeConstraint::LangTag(Arc::from("fr"))); + assert!(!check(&[where_sp_o()], tagged, &ledger)); + let listed = t_sp_o().with_list_index(0); + assert!(!check(&[where_sp_o()], listed, &ledger)); + // A different predicate. + let t = TripleTemplate::new( + TemplateTerm::Var(S), + TemplateTerm::Sid(Sid::new(100, "q")), + TemplateTerm::Var(O), + ); + assert!(!check(&[where_sp_o()], t, &ledger)); + } + + #[test] + fn an_object_bound_twice_is_not_witnessed() { + let ledger = cx(WhereDefault::Ledger); + // Cross-pattern: the object is joined with another predicate's value. + let other = triple(Ref::Var(X), Ref::Sid(Sid::new(100, "q")), Term::Var(O)); + assert!(!check(&[where_sp_o(), other], t_sp_o(), &ledger)); + // Repeated: the same pattern twice. + assert!(!check(&[where_sp_o(), where_sp_o()], t_sp_o(), &ledger)); + // VALUES / BIND on the object. + let values = Pattern::Values { + vars: vec![O], + rows: vec![vec![fluree_db_query::Binding::lit( + FlakeValue::Long(1), + Sid::new(2, "integer"), + )]], + }; + assert!(!check(&[values, where_sp_o()], t_sp_o(), &ledger)); + let bind = Pattern::Bind { + var: O, + expr: Expression::Var(X), + }; + assert!(!check(&[where_sp_o(), bind], t_sp_o(), &ledger)); + // A FILTER equating the object with another variable could fold into + // a join. + let filter = Pattern::Filter(Expression::Call { + func: Function::Eq, + args: vec![Expression::Var(O), Expression::Var(X)], + }); + let x = triple(Ref::Var(X), Ref::Sid(Sid::new(100, "q")), Term::Var(X)); + assert!(!check(&[where_sp_o(), x, filter], t_sp_o(), &ledger)); + } + + #[test] + fn optional_union_and_minus_do_not_witness() { + let ledger = cx(WhereDefault::Ledger); + let optional = Pattern::Optional(vec![where_sp_o()]); + assert!(!check(&[optional], t_sp_o(), &ledger)); + let union = Pattern::Union(vec![vec![where_sp_o()], vec![]]); + assert!(!check(&[union], t_sp_o(), &ledger)); + // A top-level witness plus an OPTIONAL that mentions the object. + let optional = Pattern::Optional(vec![triple( + Ref::Var(S), + Ref::Sid(Sid::new(100, "q")), + Term::Var(O), + )]); + assert!(!check(&[where_sp_o(), optional], t_sp_o(), &ledger)); + let minus = Pattern::Minus(vec![triple( + Ref::Var(S), + Ref::Sid(Sid::new(100, "q")), + Term::Var(O), + )]); + assert!(!check(&[where_sp_o(), minus], t_sp_o(), &ledger)); + } + + #[test] + fn graph_contexts_must_agree() { + let in_graph = t_sp_o().in_graph(GRAPH_IRI); + // WITH / a single USING . + assert!(check( + &[where_sp_o()], + in_graph.clone(), + &cx(WhereDefault::Graph(GRAPH_IRI)) + )); + // ... but the ledger default read by a named-graph template is not. + assert!(!check( + &[where_sp_o()], + in_graph.clone(), + &cx(WhereDefault::Ledger) + )); + // USING with a default-graph template. + assert!(!check( + &[where_sp_o()], + t_sp_o(), + &cx(WhereDefault::Graph(GRAPH_IRI)) + )); + // Several USING graphs. + assert!(!check(&[where_sp_o()], t_sp_o(), &cx(WhereDefault::Other))); + // A graph the ledger does not have. + let unknown = t_sp_o().in_graph("http://example.org/nope"); + assert!(!check( + &[where_sp_o()], + unknown, + &cx(WhereDefault::Graph("http://example.org/nope")) + )); + + // GRAPH { … } + let graph_block = |name: GraphName| Pattern::Graph { + name, + patterns: vec![where_sp_o()], + }; + let ledger = cx(WhereDefault::Ledger); + assert!(check( + &[graph_block(GraphName::Iri(Arc::from(GRAPH_IRI)))], + in_graph.clone(), + &ledger + )); + assert!(!check( + &[graph_block(GraphName::Iri(Arc::from(GRAPH_IRI)))], + t_sp_o(), + &ledger + )); + // A name the dataset aliases to another graph reads that graph. + let alias_of_g1 = |iri: &str| iri == GRAPH_IRI; + let aliased = WitnessContext { + aliased: &alias_of_g1, + ..cx(WhereDefault::Ledger) + }; + assert!(!check( + &[graph_block(GraphName::Iri(Arc::from(GRAPH_IRI)))], + in_graph, + &aliased + )); + // GRAPH ?g { … } with a GRAPH ?g template. + assert!(check( + &[graph_block(GraphName::Var(G))], + t_sp_o().with_graph_var(G), + &ledger + )); + assert!(!check(&[graph_block(GraphName::Var(G))], t_sp_o(), &ledger)); + } +} diff --git a/fluree-db-transact/src/generate/accumulator.rs b/fluree-db-transact/src/generate/accumulator.rs index c4ee12a3a2..446f389987 100644 --- a/fluree-db-transact/src/generate/accumulator.rs +++ b/fluree-db-transact/src/generate/accumulator.rs @@ -25,6 +25,7 @@ //! retractions survive (RDF set semantics: 4 retracts + 1 assert collapses //! to 1 surviving retract, not 0). +use crate::current_facts::Retraction; use fluree_db_core::{Flake, IndexType, Sid}; use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet}; @@ -114,10 +115,16 @@ impl FlakeAccumulator { self.input_count } - /// Push retractions from any source. Duplicates collapse to a single - /// representative survivor (RDF set semantics). - pub fn push_retractions>(&mut self, flakes: I) { + /// Push retractions. Duplicates collapse to a single representative + /// survivor (RDF set semantics). + /// + /// Only a [`Retraction`] is accepted — a retraction of a stored fact (see + /// [`crate::current_facts`]). A retraction of a fact that is not stored + /// would cancel a same-transaction assertion of that fact, and the + /// assertion would never be written. + pub fn push_retractions>(&mut self, retractions: I) { let hint = self.capacity_hint; + let flakes = retractions.into_iter().map(Retraction::into_flake); match &mut self.inner { AccInner::PureRetract(graphs) => { for f in flakes { @@ -249,6 +256,10 @@ fn push_into_mixed(set_assertions: bool, map: &mut FxHashMap #[cfg(test)] mod tests { use super::*; + + fn retractions(flakes: Vec) -> Vec { + flakes.into_iter().map(Retraction::for_test).collect() + } use fluree_db_core::{FlakeValue, Sid}; fn flake(s: u16, p: u16, o: i64, t: i64, op: bool) -> Flake { @@ -268,11 +279,11 @@ mod tests { #[test] fn pure_delete_dedups_duplicates() { let mut acc = FlakeAccumulator::pure_delete(4); - acc.push_retractions(vec![ + acc.push_retractions(retractions(vec![ flake(1, 1, 100, 5, false), flake(1, 1, 100, 5, false), flake(1, 1, 100, 5, false), - ]); + ])); let out = acc.finalize(); assert_eq!(out.len(), 1); assert!(!out[0].op); @@ -281,11 +292,11 @@ mod tests { #[test] fn pure_delete_preserves_distinct_facts() { let mut acc = FlakeAccumulator::pure_delete(4); - acc.push_retractions(vec![ + acc.push_retractions(retractions(vec![ flake(1, 1, 100, 5, false), flake(2, 1, 100, 5, false), flake(3, 1, 100, 5, false), - ]); + ])); let out = acc.finalize(); assert_eq!(out.len(), 3); // SPOT order: by subject namespace_code first @@ -297,7 +308,10 @@ mod tests { #[test] fn pure_delete_input_count_tracks_total_pushes() { let mut acc = FlakeAccumulator::pure_delete(2); - acc.push_retractions(vec![flake(1, 1, 100, 5, false), flake(1, 1, 100, 5, false)]); + acc.push_retractions(retractions(vec![ + flake(1, 1, 100, 5, false), + flake(1, 1, 100, 5, false), + ])); assert_eq!(acc.input_count(), 2); assert_eq!(acc.finalize().len(), 1); } @@ -320,7 +334,7 @@ mod tests { #[test] fn mixed_cancels_one_to_one_pair() { let mut acc = FlakeAccumulator::mixed(2); - acc.push_retractions(vec![flake(1, 1, 100, 5, false)]); + acc.push_retractions(retractions(vec![flake(1, 1, 100, 5, false)])); acc.push_assertions(vec![flake(1, 1, 100, 5, true)]); assert!(acc.finalize().is_empty()); } @@ -328,7 +342,7 @@ mod tests { #[test] fn mixed_keeps_unmatched() { let mut acc = FlakeAccumulator::mixed(2); - acc.push_retractions(vec![flake(1, 1, 100, 5, false)]); + acc.push_retractions(retractions(vec![flake(1, 1, 100, 5, false)])); acc.push_assertions(vec![flake(1, 1, 200, 5, true)]); // different object let out = acc.finalize(); assert_eq!(out.len(), 2); @@ -337,12 +351,12 @@ mod tests { #[test] fn mixed_4_retracts_1_assert_yields_1_retract() { let mut acc = FlakeAccumulator::mixed(2); - acc.push_retractions(vec![ + acc.push_retractions(retractions(vec![ flake(1, 1, 100, 5, false), flake(1, 1, 100, 6, false), flake(1, 1, 100, 7, false), flake(1, 1, 100, 8, false), - ]); + ])); acc.push_assertions(vec![flake(1, 1, 100, 9, true)]); let out = acc.finalize(); assert_eq!(out.len(), 1); @@ -357,7 +371,7 @@ mod tests { flake(1, 1, 100, 6, true), flake(1, 1, 100, 7, true), ]); - acc.push_retractions(vec![flake(1, 1, 100, 8, false)]); + acc.push_retractions(retractions(vec![flake(1, 1, 100, 8, false)])); let out = acc.finalize(); assert_eq!(out.len(), 1); assert!(out[0].op, "the survivor must be an assertion"); @@ -371,12 +385,12 @@ mod tests { // semantics the fact is unchanged and nothing survives. let mut counting = FlakeAccumulator::mixed(2); counting.push_assertions(vec![flake(1, 1, 100, 5, true), flake(1, 1, 100, 5, true)]); - counting.push_retractions(vec![flake(1, 1, 100, 5, false)]); + counting.push_retractions(retractions(vec![flake(1, 1, 100, 5, false)])); assert_eq!(counting.finalize().len(), 1); let mut set = FlakeAccumulator::mixed_set_assertions(2); set.push_assertions(vec![flake(1, 1, 100, 5, true), flake(1, 1, 100, 5, true)]); - set.push_retractions(vec![flake(1, 1, 100, 5, false)]); + set.push_retractions(retractions(vec![flake(1, 1, 100, 5, false)])); assert!( set.finalize().is_empty(), "unchanged fact must net to nothing" @@ -386,7 +400,7 @@ mod tests { // is still retracted. let mut set = FlakeAccumulator::mixed_set_assertions(2); set.push_assertions(vec![flake(1, 1, 101, 5, true), flake(1, 1, 101, 5, true)]); - set.push_retractions(vec![flake(1, 1, 100, 5, false)]); + set.push_retractions(retractions(vec![flake(1, 1, 100, 5, false)])); let out = set.finalize(); assert_eq!(out.len(), 2); assert_eq!(out.iter().filter(|f| f.op).count(), 1); @@ -423,8 +437,8 @@ mod tests { #[test] fn mixed_multi_feed_retractions_merge_correctly() { let mut acc = FlakeAccumulator::mixed(2); - acc.push_retractions(vec![flake(1, 1, 100, 5, false)]); - acc.push_retractions(vec![flake(1, 1, 100, 6, false)]); // duplicate fact + acc.push_retractions(retractions(vec![flake(1, 1, 100, 5, false)])); + acc.push_retractions(retractions(vec![flake(1, 1, 100, 6, false)])); // duplicate fact acc.push_assertions(vec![flake(1, 1, 100, 7, true)]); // 2 retracts vs 1 assert → cancel = 1 → 1 retract survives. let out = acc.finalize(); @@ -467,10 +481,10 @@ mod tests { // Same fact retracted in the default graph AND a named graph must yield // two survivors (one per graph), not collapse to one. let mut acc = FlakeAccumulator::pure_delete(2); - acc.push_retractions(vec![ + acc.push_retractions(retractions(vec![ flake(1, 1, 100, 5, false), // default graph (g = None) flake_in_graph(7, 1, 1, 100, 5, false), // named graph - ]); + ])); let out = acc.finalize(); assert_eq!(out.len(), 2, "default and named retractions are distinct"); assert!(out.iter().any(|f| f.g.is_none())); @@ -497,7 +511,7 @@ mod tests { // default graph target different graphs and must NOT cancel. let mut acc = FlakeAccumulator::mixed(2); acc.push_assertions(vec![flake_in_graph(7, 1, 1, 100, 5, true)]); - acc.push_retractions(vec![flake(1, 1, 100, 6, false)]); + acc.push_retractions(retractions(vec![flake(1, 1, 100, 6, false)])); let out = acc.finalize(); assert_eq!(out.len(), 2, "cross-graph assert/retract must not cancel"); assert!(out.iter().any(|f| f.op && f.g.is_some())); @@ -510,7 +524,7 @@ mod tests { // cancel 1:1 (unchanged set semantics, now graph-scoped). let mut acc = FlakeAccumulator::mixed(2); acc.push_assertions(vec![flake_in_graph(7, 1, 1, 100, 5, true)]); - acc.push_retractions(vec![flake_in_graph(7, 1, 1, 100, 6, false)]); + acc.push_retractions(retractions(vec![flake_in_graph(7, 1, 1, 100, 6, false)])); assert!(acc.finalize().is_empty()); } } diff --git a/fluree-db-transact/src/generate/cancellation.rs b/fluree-db-transact/src/generate/cancellation.rs deleted file mode 100644 index caaadff35b..0000000000 --- a/fluree-db-transact/src/generate/cancellation.rs +++ /dev/null @@ -1,251 +0,0 @@ -//! Assertion/retraction cancellation -//! -//! When the same fact appears both as an assertion and a retraction -//! in a transaction, they cancel each other out. This module implements -//! that cancellation logic. -//! -//! Uses `Flake`'s existing `Eq`/`Hash` implementation which ignores -//! `t` and `op` but includes metadata `m`. - -use crate::generate::accumulator::FlakeAccumulator; -use fluree_db_core::Flake; - -/// Apply cancellation to a set of flakes -/// -/// Removes matching assertion/retraction pairs where the flakes are -/// equal (same subject, predicate, object, datatype, metadata) but -/// differ only in operation. -/// -/// Uses counter-based tracking so that duplicate flakes (same fact at -/// different `t` values) are handled correctly. For example, if a fact -/// was asserted 4 times (at t=1,2,3,4) and retracted once, 3 assertions -/// survive — not 0 or 1. -/// -/// Returns flakes in deterministic sorted order (by SPOT index) for -/// reproducible hashing and tests. -pub fn apply_cancellation(flakes: Vec) -> Vec { - if flakes.is_empty() { - return flakes; - } - let mut acc = FlakeAccumulator::mixed(flakes.len()); - for f in flakes { - if f.op { - acc.push_assertions(std::iter::once(f)); - } else { - acc.push_retractions(std::iter::once(f)); - } - } - acc.finalize() -} - -/// Dedup a Vec of retractions (cheap path for pure-DELETE transactions). -/// -/// Thin wrapper over [`FlakeAccumulator::pure_delete`] kept for backward -/// compatibility with callers that already hold an owned `Vec`. New -/// code should construct a [`FlakeAccumulator`] directly so retractions can -/// stream from multiple sources without the intermediate `Vec`. -pub fn dedup_retractions(retractions: Vec) -> Vec { - let mut acc = FlakeAccumulator::pure_delete(retractions.len()); - acc.push_retractions(retractions); - acc.finalize() -} - -#[cfg(test)] -mod tests { - use super::*; - use fluree_db_core::{FlakeValue, Sid}; - - fn make_flake(s: u16, p: u16, o: i64, t: i64, op: bool) -> Flake { - Flake::new( - Sid::new(s, format!("s{s}")), - Sid::new(p, format!("p{p}")), - FlakeValue::Long(o), - Sid::new(2, "long"), - t, - op, - None, - ) - } - - #[test] - fn test_cancellation_removes_pairs() { - let assertion = make_flake(1, 2, 100, 1, true); - let retraction = make_flake(1, 2, 100, 1, false); - - let result = apply_cancellation(vec![assertion, retraction]); - - assert!(result.is_empty()); - } - - #[test] - fn test_cancellation_keeps_unmatched() { - let assertion = make_flake(1, 2, 100, 1, true); - let retraction = make_flake(1, 2, 200, 1, false); // Different object - - let result = apply_cancellation(vec![assertion, retraction]); - - assert_eq!(result.len(), 2); - } - - #[test] - fn test_cancellation_order_independent() { - let flakes1 = vec![ - make_flake(1, 2, 100, 1, true), - make_flake(1, 2, 100, 1, false), - ]; - let flakes2 = vec![ - make_flake(1, 2, 100, 1, false), - make_flake(1, 2, 100, 1, true), - ]; - - let result1 = apply_cancellation(flakes1); - let result2 = apply_cancellation(flakes2); - - assert!(result1.is_empty()); - assert!(result2.is_empty()); - } - - #[test] - fn test_cancellation_deterministic_output() { - let flakes = vec![ - make_flake(3, 1, 100, 1, true), - make_flake(1, 1, 100, 1, true), - make_flake(2, 1, 100, 1, true), - ]; - - let result = apply_cancellation(flakes); - - // Should be sorted by SPOT (subject first) - assert_eq!(result.len(), 3); - assert_eq!(result[0].s.namespace_code, 1); - assert_eq!(result[1].s.namespace_code, 2); - assert_eq!(result[2].s.namespace_code, 3); - } - - #[test] - fn test_cancellation_multiple_pairs() { - let flakes = vec![ - make_flake(1, 1, 100, 1, true), - make_flake(1, 1, 100, 1, false), // Cancels first - make_flake(2, 1, 200, 1, true), - make_flake(2, 1, 200, 1, false), // Cancels third - make_flake(3, 1, 300, 1, true), // Remains - ]; - - let result = apply_cancellation(flakes); - - assert_eq!(result.len(), 1); - assert_eq!(result[0].s.namespace_code, 3); - } - - #[test] - fn test_cancellation_different_t_still_cancels() { - // Flake's Eq ignores t, so these should cancel - let assertion = make_flake(1, 2, 100, 5, true); - let retraction = make_flake(1, 2, 100, 10, false); - - let result = apply_cancellation(vec![assertion, retraction]); - - assert!(result.is_empty()); - } - - /// RDF set semantics: duplicate retractions of the same fact within one - /// transaction collapse to a single retraction. - /// - /// Scenario: entity has 4 copies of "open" (asserted at t=1,2,3,4 due to - /// prior bug). Upsert generates 4 retractions + 1 assertion for "open". - /// After cancellation + set-dedup: 1 retraction survives. - #[test] - fn test_cancellation_collapses_duplicate_retractions() { - let flakes = vec![ - // 4 retractions for same (s,p,o) at different t values - make_flake(1, 2, 100, 1, false), - make_flake(1, 2, 100, 2, false), - make_flake(1, 2, 100, 3, false), - make_flake(1, 2, 100, 4, false), - // 1 assertion (upsert re-asserts "open") - make_flake(1, 2, 100, 5, true), - ]; - - let result = apply_cancellation(flakes); - - // 1 assertion cancels 1 retraction, remaining 3 collapse to 1 - assert_eq!(result.len(), 1, "should have 1 surviving retraction"); - assert!( - result.iter().all(|f| !f.op), - "survivor should be a retraction" - ); - } - - /// RDF set semantics: duplicate assertions collapse to one. - #[test] - fn test_cancellation_collapses_duplicate_assertions() { - let flakes = vec![ - // 3 assertions for same (s,p,o) at different t values - make_flake(1, 2, 100, 1, true), - make_flake(1, 2, 100, 2, true), - make_flake(1, 2, 100, 3, true), - // 1 retraction - make_flake(1, 2, 100, 5, false), - ]; - - let result = apply_cancellation(flakes); - - // 1 retraction cancels 1 assertion, remaining 2 collapse to 1 - assert_eq!(result.len(), 1, "should have 1 surviving assertion"); - assert!( - result.iter().all(|f| f.op), - "survivor should be an assertion" - ); - } - - /// Mixed scenario: some facts have duplicates, some don't. - /// Set semantics collapses duplicate survivors to one per fact. - #[test] - fn test_cancellation_mixed_duplicates_and_unique() { - let flakes = vec![ - // Fact A: 3 retractions, 1 assertion → 2 retractions collapse to 1 - make_flake(1, 1, 100, 1, false), - make_flake(1, 1, 100, 2, false), - make_flake(1, 1, 100, 3, false), - make_flake(1, 1, 100, 5, true), - // Fact B: 1 retraction, 1 assertion → cancel out - make_flake(1, 1, 200, 1, false), - make_flake(1, 1, 200, 5, true), - // Fact C: 1 assertion only → survives - make_flake(1, 1, 300, 5, true), - ]; - - let result = apply_cancellation(flakes); - - // 1 retraction (fact A) + 1 assertion (fact C) = 2 - assert_eq!(result.len(), 2); - let retraction_count = result.iter().filter(|f| !f.op).count(); - let assertion_count = result.iter().filter(|f| f.op).count(); - assert_eq!(retraction_count, 1, "fact A: 1 retraction survives"); - assert_eq!(assertion_count, 1, "fact C: 1 assertion survives"); - } - - /// Regression test for the nested-object duplication bug: when a JSON-LD - /// transaction contains the same entity nested in multiple parents (e.g., - /// a Member nested in 14 different Channel objects), parsing produces N - /// identical assertion flakes per property. Set semantics must collapse - /// these to 1 flake per unique fact. - #[test] - fn test_set_semantics_dedup_pure_assertions() { - // Simulates member with 4 properties, each duplicated 14 times - let mut flakes = Vec::new(); - for prop in 1..=4u16 { - for _ in 0..14 { - flakes.push(make_flake(1, prop, 42, 1, true)); - } - } - assert_eq!(flakes.len(), 56); - - let result = apply_cancellation(flakes); - - // Should collapse to 4 unique assertions (one per property) - assert_eq!(result.len(), 4, "56 duplicate assertions → 4 unique facts"); - assert!(result.iter().all(|f| f.op), "all should be assertions"); - } -} diff --git a/fluree-db-transact/src/generate/flakes.rs b/fluree-db-transact/src/generate/flakes.rs index 4e6bd19df3..2af9b0118f 100644 --- a/fluree-db-transact/src/generate/flakes.rs +++ b/fluree-db-transact/src/generate/flakes.rs @@ -3,6 +3,7 @@ //! This module provides `FlakeGenerator` for materializing triple templates //! with variable bindings into concrete flakes. +use crate::current_facts::RetractIntent; use crate::error::{Result, TransactError}; use crate::ir::{TemplateGraph, TemplateTerm, TripleTemplate}; use crate::namespace::NamespaceRegistry; @@ -137,13 +138,29 @@ impl<'a> FlakeGenerator<'a> { self.generate_flakes(templates, bindings, true) } - /// Generate retraction flakes from delete templates - pub fn generate_retractions( + /// Instantiate DELETE templates into retraction intents, one per template + /// per solution row. `witnessed[i]` says whether a WHERE triple witnesses + /// `templates[i]` (see `delete_witness`). An intent is what the + /// transaction asks to retract, not yet a retraction: the resolver turns + /// it into one only if it names a stored fact. + pub(crate) fn generate_retract_intents( &mut self, templates: &[TripleTemplate], + witnessed: &[bool], bindings: &Batch, - ) -> Result> { - self.generate_flakes(templates, bindings, false) + ) -> Result> { + debug_assert_eq!(templates.len(), witnessed.len()); + let mut intents = Vec::new(); + for row_idx in 0..Self::solution_rows(bindings) { + for (template, &witnessed) in templates.iter().zip(witnessed) { + if let Some(flake) = + self.materialize_template(template, bindings, row_idx, false)? + { + intents.push(RetractIntent::new(flake, witnessed)); + } + } + } + Ok(intents) } /// Generate flakes from templates with given operation flag @@ -154,28 +171,29 @@ impl<'a> FlakeGenerator<'a> { op: bool, ) -> Result> { let mut flakes = Vec::new(); - - // Row count semantics: - // - INSERT without WHERE produces an "empty bindings" batch (0 vars, 0 rows). We still need - // to materialize templates once, so treat it as a single empty row. - // - UPDATE/UPSERT where WHERE matches nothing produces an empty batch with a non-empty - // schema (vars present but 0 rows). In that case, there are **zero solution rows** and - // templates must produce **zero flakes** (no-op). - let row_count = if bindings.is_empty() { - usize::from(bindings.schema().is_empty()) - } else { - bindings.len() - }; - - for row_idx in 0..row_count { + for row_idx in 0..Self::solution_rows(bindings) { for template in templates { flakes.extend(self.materialize_template(template, bindings, row_idx, op)?); } } - Ok(flakes) } + /// Solution rows in a WHERE batch. + /// + /// - INSERT without WHERE produces an "empty bindings" batch (0 vars, 0 + /// rows). Templates still materialize once, so it is one empty row. + /// - UPDATE/UPSERT where WHERE matches nothing produces an empty batch + /// with a non-empty schema (vars present but 0 rows): **zero solution + /// rows**, so templates produce **zero flakes** (no-op). + fn solution_rows(bindings: &Batch) -> usize { + if bindings.is_empty() { + usize::from(bindings.schema().is_empty()) + } else { + bindings.len() + } + } + /// Materialize a single template with bindings into a flake fn materialize_template( &mut self, @@ -676,10 +694,12 @@ mod tests { )]; let batch = make_empty_batch(); - let flakes = generator.generate_retractions(&templates, &batch).unwrap(); + let intents = generator + .generate_retract_intents(&templates, &[false], &batch) + .unwrap(); - assert_eq!(flakes.len(), 1); - assert!(!flakes[0].op); + assert_eq!(intents.len(), 1); + assert!(!intents[0].flake().op); } #[test] diff --git a/fluree-db-transact/src/generate/mod.rs b/fluree-db-transact/src/generate/mod.rs index 29bf4e81ae..1dd86d6d21 100644 --- a/fluree-db-transact/src/generate/mod.rs +++ b/fluree-db-transact/src/generate/mod.rs @@ -2,14 +2,13 @@ //! //! This module provides functionality for materializing triple templates //! into concrete flakes, handling variable substitution, blank node -//! skolemization, and assertion/retraction cancellation. +//! skolemization, and assertion/retraction cancellation +//! ([`FlakeAccumulator`]). pub mod accumulator; -pub mod cancellation; pub mod flakes; pub use accumulator::FlakeAccumulator; -pub use cancellation::{apply_cancellation, dedup_retractions}; #[cfg(feature = "import")] pub(crate) use flakes::DT_ID; pub use flakes::{infer_datatype, FlakeGenerator}; diff --git a/fluree-db-transact/src/lib.rs b/fluree-db-transact/src/lib.rs index f62716c1a5..3de24e67b9 100644 --- a/fluree-db-transact/src/lib.rs +++ b/fluree-db-transact/src/lib.rs @@ -36,6 +36,7 @@ pub mod commit; pub mod commit_flakes; pub mod current_facts; pub mod datatype_limit; +mod delete_witness; pub mod error; pub mod flake_sink; pub mod generate; @@ -66,7 +67,7 @@ pub use commit_flakes::generate_commit_flakes; pub use current_facts::{CurrentFacts, Origin, Retraction, Slot, StoredFact}; pub use error::{Result, TransactError}; pub use flake_sink::FlakeSink; -pub use generate::{apply_cancellation, FlakeGenerator}; +pub use generate::FlakeGenerator; pub use ir::{ GraphMgmtOp, GraphSel, GraphTarget, InlineValues, TemplateGraph, TemplateTerm, TripleTemplate, Txn, TxnOpts, TxnType, @@ -84,7 +85,10 @@ pub use parse::{ RawTerm, RawTrigMeta, RawTriple, TrigPhase1Result, UnwrappedTrig, }; pub use raw_txn_upload::PendingRawTxnUpload; -pub use stage::{generate_txn_id, stage, stage_flakes, stage_with_graph_delta, StageOptions}; +pub use stage::{ + generate_txn_id, stage, stage_flakes, stage_with_graph_delta, StageOptions, + DELETE_WITNESSED_SITE, +}; pub use staged_dicts::{ attach_binary_provider, attach_staged_dicts, detach_binary_provider, staged_dicts, StagedDicts, }; diff --git a/fluree-db-transact/src/stage.rs b/fluree-db-transact/src/stage.rs index 7a26280dd8..37d5076294 100644 --- a/fluree-db-transact/src/stage.rs +++ b/fluree-db-transact/src/stage.rs @@ -8,7 +8,8 @@ //! When the `shacl` feature is enabled, [`validate_view_with_shacl`] validates a //! staged view against SHACL shapes. -use crate::current_facts::{CurrentFacts, Retraction, Slot, StoredFact}; +use crate::current_facts::{CurrentFacts, ListFree, ResolveStats, Retraction, Slot, StoredFact}; +use crate::delete_witness::{witnessed_templates, WhereDefault, WitnessContext}; use crate::error::{Result, TransactError}; use crate::generate::{infer_datatype, FlakeAccumulator, FlakeGenerator}; use crate::ir::InlineValues; @@ -17,7 +18,6 @@ use crate::ir::{ Txn, TxnType, }; use crate::namespace::NamespaceRegistry; -use fluree_db_core::comparator::IndexType; use fluree_db_core::graph_registry::{FIRST_USER_GRAPH_ID, TXN_META_GRAPH_ID}; use fluree_db_core::tracking::schedule::TXN_BASELINE_MICRO_FUEL; use fluree_db_core::OverlayProvider; @@ -28,6 +28,7 @@ use fluree_db_policy::{ is_schema_flake, lookup_subject_classes, PolicyContext, PolicyDecision, PolicyError, WriteFlakeInfo, WriteVerb, }; +use fluree_db_query::fast_path_outcome::{stamp_fast_path, FastPathFallback, FastPathOutcome}; use fluree_db_query::parse::{lower_unresolved_patterns, UnresolvedPattern}; use fluree_db_query::{ Batch, Binding, Pattern, QueryPolicyEnforcer, QueryPolicyExecutor, VarId, VarRegistry, @@ -44,6 +45,12 @@ use tracing::Instrument; #[cfg(feature = "shacl")] use fluree_db_shacl::{ShaclCache, ShaclEngine, ValidationReport}; +/// Routing stamp site for a DELETE whose templates a WHERE triple +/// witnesses: `proceed` when the witnessed rows retract with no read (the +/// ledger holds no `@list` position), `fallback:gate_declined` when they read +/// their slots for list positions. +pub const DELETE_WITNESSED_SITE: &str = "delete_witnessed"; + /// Build a reverse lookup from graph Sid → GraphId. /// /// Given `graph_sids` (ledger GraphId → Sid), returns the @@ -903,7 +910,7 @@ pub async fn stage_with_graph_delta( upsert_retraction_count = upsert_retractions.len(), "upsert deletions generated" ); - acc.push_retractions(upsert_retractions.into_iter().map(Retraction::into_flake)); + acc.push_retractions(upsert_retractions); } // Graph-sync wave: retract every current fact of the target graph @@ -939,7 +946,7 @@ pub async fn stage_with_graph_delta( scanned = current.len(), "graph-sync retractions generated" ); - acc.push_retractions(current.into_iter().map(|f| f.retract(new_t).into_flake())); + acc.push_retractions(current.into_iter().map(|f| f.retract(new_t))); } let retraction_count = stream_stats.retraction_count; @@ -2159,133 +2166,6 @@ pub async fn stage_flakes( .await } -async fn hydrate_list_index_meta_for_retractions( - ledger: &LedgerState, - retractions: &mut [Flake], - reverse_graph: &HashMap, -) -> Result<()> { - use std::collections::BTreeMap; - - // Nothing to copy when neither the indexed base nor novelty holds a - // single `@list` position. `Some(false)` is an exact observation by the - // indexer; `None` (legacy root, bulk import) must fall through. - if ledger.snapshot.has_list_meta == Some(false) && !ledger.novelty.has_list_meta { - return Ok(()); - } - - // Group candidates by (graph, subject, predicate). - let mut groups: HashMap<(GraphId, Sid, Sid), Vec> = HashMap::new(); - for (idx, flake) in retractions.iter().enumerate() { - // Only retractions lacking a list position are candidates. A - // language-tagged binding already carries `m = { lang, i: None }` - // and still needs its position filled in. - if flake.op || flake.m.as_ref().is_some_and(|m| m.i.is_some()) { - continue; - } - let g_id = resolve_flake_graph_id(flake, reverse_graph)?; - groups - .entry((g_id, flake.s.clone(), flake.p.clone())) - .or_default() - .push(idx); - } - if groups.is_empty() { - return Ok(()); - } - - let to_t = ledger.t(); - - // Novelty side: ONE SPOT walk per touched graph, keeping every op on a - // requested (subject, predicate) pair. `range_with_overlay` per group - // would instead translate (or walk) the graph's entire overlay once per - // group — O(groups × novelty), observed as a multi-minute-to-never - // filtered DELETE once both are in the tens of thousands. - let mut wanted: HashMap> = HashMap::new(); - for (g_id, s, p) in groups.keys() { - wanted.entry(*g_id).or_default().insert((s, p)); - } - let mut overlay_by_key: HashMap<(GraphId, Sid, Sid), Vec> = HashMap::new(); - for (g_id, pairs) in &wanted { - ledger.novelty.for_each_overlay_flake( - *g_id, - IndexType::Spot, - None, - None, - true, - to_t, - &mut |f| { - if f.t <= to_t && pairs.contains(&(&f.s, &f.p)) { - overlay_by_key - .entry((*g_id, f.s.clone(), f.p.clone())) - .or_default() - .push(f.clone()); - } - }, - ); - } - - for ((g_id, s, p), members) in groups { - // Base side: subject + predicate bound against the persisted index - // only (`NoOverlay`) — a leaf seek, no overlay translation. - let rm = fluree_db_core::RangeMatch::new() - .with_subject(s.clone()) - .with_predicate(p.clone()); - let mut found = fluree_db_core::range_with_overlay( - &ledger.snapshot, - g_id, - &fluree_db_core::NoOverlay, - IndexType::Spot, - fluree_db_core::RangeTest::Eq, - rm, - fluree_db_core::RangeOptions::new().with_to_t(to_t), - ) - .await?; - if let Some(ops) = overlay_by_key.remove(&(g_id, s, p)) { - found.extend(ops); - } - // Same lifecycle rule `range_with_overlay` applies to its merged - // result: newest op per fact key wins, retractions drop out. - let found = fluree_db_core::range::resolve_current_flakes(found, IndexType::Spot); - - // Index asserted list-carrying metas per object value, in index - // order. Every matching retraction copies the FIRST dt-compatible - // meta: identical duplicates then collapse in the accumulator, so a - // value asserted at N list positions loses exactly one entry per - // distinct WHERE binding (pinned by the `object-probe-list-retract` - // case in `it_join_batched_overlay.rs`). - let mut metas: BTreeMap> = - BTreeMap::new(); - for f in found { - if f.op { - if let Some(m) = f.m.filter(|m| m.i.is_some()) { - metas.entry(f.o).or_default().push((f.dt, m)); - } - } - } - if metas.is_empty() { - continue; - } - - for idx in members { - let flake = &mut retractions[idx]; - let Some(candidates) = metas.get(&flake.o) else { - continue; - }; - // Same lexical value under different language tags are distinct - // facts: the candidate must match the retraction's tag (absent - // on both for plain literals) as well as its datatype. - let lang = flake.m.as_ref().and_then(|m| m.lang.as_deref()); - if let Some((_, m)) = candidates - .iter() - .find(|(dt, m)| &flake.dt == dt && m.lang.as_deref() == lang) - { - flake.m = Some(m.clone()); - } - } - } - - Ok(()) -} - /// Per-(graph, subject) write-time state computed once per transaction for /// policy enforcement: the class sets each policy flavor targets against and /// the subject's lifecycle verb. @@ -2624,19 +2504,20 @@ struct WhereStreamStats { } /// Stream the WHERE result into `acc`, projecting → materializing encoded -/// bindings in place → generating retractions → hydrating list-index meta → -/// pushing into the accumulator, one batch at a time. Assertions are -/// generated and pushed on the same batch when not in pure-delete mode. +/// bindings in place → instantiating DELETE templates as retraction intents +/// → resolving them to retractions of stored facts → pushing into the +/// accumulator, one batch at a time. Assertions are generated and pushed on +/// the same batch when not in pure-delete mode. /// /// `template_vars` is the union of variables referenced by INSERT/DELETE /// templates; WHERE-only helper columns are dropped before materialization /// to keep per-batch memory tied to template width, not WHERE width. /// -/// **Hydration must run before push.** `Flake` identity includes `m`, so a -/// raw retraction with `m = None` would collapse with its peers in the -/// accumulator before hydrate had a chance to fill in the list-index — we'd -/// end up retracting only one of N list entries. Hydrating per batch keeps -/// `m` correct on every retraction before it reaches the dedup layer. +/// **Resolution runs before push.** An intent that names no stored fact +/// stages nothing, so it cannot cancel a same-transaction assertion of that +/// fact; and `Flake` identity includes `m`, so each retraction must carry +/// its stored list position before the accumulator dedups, or N list +/// entries with the same `(s, p, o, dt)` would collapse to one. #[allow(clippy::too_many_arguments)] async fn stream_where_into_accumulator( ledger: &LedgerState, @@ -2908,6 +2789,42 @@ async fn stream_where_into_accumulator( runtime_dataset.with_named_graph_alias(name, make_graph_ref(g_id)) }; } + // Which DELETE templates a WHERE triple witnesses, and whether the + // ledger can hold a list position at all: together they decide which + // retractions need no read (see `CurrentFacts::resolve_intents`). + let where_default = if where_default_is_empty { + WhereDefault::Other + } else { + match desired_where_default_graph_iris.as_slice() { + [] => WhereDefault::Ledger, + [iri] => WhereDefault::Graph(iri), + _ => WhereDefault::Other, + } + }; + let registered = |iri: &str| resolve_graph_id(iri).is_some(); + let aliased = |iri: &str| graph_aliases.contains_key(iri); + let witnessed = witnessed_templates( + &query_patterns, + &txn.delete_templates, + &WitnessContext { + default: where_default, + aliased: &aliased, + registered: ®istered, + }, + ); + let list_free = ListFree::check(ledger); + if witnessed.iter().any(|w| *w) { + stamp_fast_path( + DELETE_WITNESSED_SITE, + if list_free.is_some() { + FastPathOutcome::Proceed + } else { + FastPathOutcome::Fallback(FastPathFallback::GateDeclined) + }, + ); + } + let current_facts = CurrentFacts::new(ledger); + let mut resolve_stats = ResolveStats::default(); generator.set_graph_aliases(graph_aliases); // Open the streaming WHERE cursor. For empty patterns it emits one @@ -2947,27 +2864,28 @@ async fn stream_where_into_accumulator( template_count = txn.delete_templates.len(), retraction_count = tracing::field::Empty, ); - let retractions = { + let intents = { let _g = delete_span.enter(); - let mut r = generator.generate_retractions(&txn.delete_templates, &batch)?; + let intents = + generator.generate_retract_intents(&txn.delete_templates, &witnessed, &batch)?; route_var_graphs( ledger, generator.written_graphs(), fixed_graph_iris, reverse_graph, )?; - - // Hydrate BEFORE push. `Flake::eq` includes `m`, so raw retractions - // with `m = None` must have their list-index filled in from the - // asserted flake before they reach the accumulator — otherwise - // N list entries with the same `(s,p,o,dt)` would collapse to one - // retraction survivor and only one list entry would actually be - // retracted from the index. - hydrate_list_index_meta_for_retractions(ledger, &mut r, reverse_graph).await?; - - delete_span.record("retraction_count", r.len() as u64); - r + intents }; + // Resolve BEFORE push (see the doc comment above). + let routing: &HashMap = reverse_graph; + let (retractions, stats) = current_facts + .resolve_intents(intents, list_free.as_ref(), |f| { + resolve_flake_graph_id(f, routing) + }) + .instrument(delete_span.clone()) + .await?; + delete_span.record("retraction_count", retractions.len() as u64); + resolve_stats.add(stats); retraction_count += retractions.len(); acc.push_retractions(retractions); @@ -2996,6 +2914,17 @@ async fn stream_where_into_accumulator( } cursor.close(); + if !txn.delete_templates.is_empty() { + tracing::debug!( + witnessed = resolve_stats.witnessed, + enriched = resolve_stats.enriched, + matched = resolve_stats.matched, + phantom_intents = resolve_stats.phantom_intents, + slot_reads = resolve_stats.slot_reads, + "delete intent resolution" + ); + } + Ok(WhereStreamStats { total_binding_rows, retraction_count, From 9d0a6ee047e89aadd35741b8938045079b6c71e0 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 02:00:17 -0400 Subject: [PATCH 07/36] fix(transact): the annotation cascade retracts stored bundles in their graph The cascade that follows a retracted edge to its annotations rebuilt each f:reifies* bundle from the decoded EdgeKey, and read annotation subjects with range_with_overlay(&novelty). Its body-cleanup pass (a by-id annotation delete in LPG mode) did not stamp the graph on index-resident rows, so the body's retraction landed in the default graph and the body survived in its named graph. And nothing retracted the bundle's f:reifiesGraph anchor, which the by-id delete does not name, so it stayed behind describing nothing. All three passes now read annotation subjects with CurrentFacts::of_subject, graph stamped, and retract bundles and bodies as stored. A complete by-id bundle retract also retracts every other stored slot of that bundle, unless the transaction re-points the reifier. --- fluree-db-api/tests/grp_misc.rs | 2 + .../tests/it_annotation_delete_named_graph.rs | 138 +++++++++++ fluree-db-transact/src/stage.rs | 214 ++++++++---------- 3 files changed, 230 insertions(+), 124 deletions(-) create mode 100644 fluree-db-api/tests/it_annotation_delete_named_graph.rs diff --git a/fluree-db-api/tests/grp_misc.rs b/fluree-db-api/tests/grp_misc.rs index 4e97469360..ba3b9244e3 100644 --- a/fluree-db-api/tests/grp_misc.rs +++ b/fluree-db-api/tests/grp_misc.rs @@ -1,6 +1,8 @@ #[path = "support/mod.rs"] mod support; +#[path = "it_annotation_delete_named_graph.rs"] +mod it_annotation_delete_named_graph; #[path = "it_annotation_filter_pushdown.rs"] mod it_annotation_filter_pushdown; #[path = "it_compile_breakdown.rs"] diff --git a/fluree-db-api/tests/it_annotation_delete_named_graph.rs b/fluree-db-api/tests/it_annotation_delete_named_graph.rs new file mode 100644 index 0000000000..98473b4cd9 --- /dev/null +++ b/fluree-db-api/tests/it_annotation_delete_named_graph.rs @@ -0,0 +1,138 @@ +//! Deleting a named-graph annotation by id retracts its whole bundle and +//! its body in that graph. +//! +//! The by-id delete names the bundle's subject, predicate and object slots. +//! The cascade then cleaned up the annotation's body (LPG mode) through a +//! range read whose index-resident rows carry no graph, so the body's +//! retraction landed in the default graph and the body survived in its own +//! graph. And nothing retracted the bundle's `f:reifiesGraph` anchor, which +//! stayed behind describing nothing. Both now come from the stored facts of +//! the annotation, graph stamped. + +use crate::support; +use fluree_db_api::{FlureeBuilder, LedgerState}; +use fluree_db_core::comparator::IndexType; +use fluree_db_core::{range_with_overlay, FlakeValue, RangeMatch, RangeOptions, RangeTest}; +use serde_json::json; + +const EX: &str = "http://example.org/"; +const G1: &str = "http://example.org/g1"; + +/// Every current fact of `subject` in `graph` (None = default), as +/// `pred=value`, sorted — `f:reifies*` included. +async fn facts(ledger: &LedgerState, graph: Option<&str>, subject: &str) -> Vec { + let g_id = match graph { + None => 0, + Some(iri) => ledger + .snapshot + .graph_registry + .graph_id_for_iri(iri) + .expect("graph"), + }; + let s = ledger + .snapshot + .encode_iri(&format!("{EX}{subject}")) + .expect("subject"); + let flakes = range_with_overlay( + &ledger.snapshot, + g_id, + ledger.novelty.as_ref(), + IndexType::Spot, + RangeTest::Eq, + RangeMatch::new().with_subject(s), + RangeOptions::new().with_to_t(ledger.t()), + ) + .await + .expect("range"); + let mut out: Vec = flakes + .iter() + .filter(|f| f.op) + .map(|f| match &f.o { + FlakeValue::String(v) => format!("{}={v}", f.p.name), + FlakeValue::Ref(r) => format!("{}=<{}>", f.p.name, r.name), + other => format!("{}={other:?}", f.p.name), + }) + .collect(); + out.sort(); + out +} + +#[tokio::test] +async fn by_id_delete_of_a_named_graph_annotation_retracts_bundle_and_body_in_that_graph() { + for indexed in [false, true] { + let dir = tempfile::tempdir().expect("tempdir"); + let fluree = FlureeBuilder::file(dir.path().to_string_lossy().to_string()) + .build() + .expect("fluree"); + let id = format!("it/ann-delete-g1-{indexed}:main"); + let ledger = fluree.create_ledger(&id).await.unwrap(); + let mut ledger = fluree + .insert( + ledger, + &json!({ + "@context": {"ex": EX}, + "@graph": [{ + "@id": "ex:alice", "@graph": G1, + "ex:worksFor": { + "@id": "ex:acme", + "@annotation": {"@id": "ex:empA", "ex:role": "Engineer"} + } + }] + }), + ) + .await + .unwrap() + .ledger; + if indexed { + drop(ledger); + support::rebuild_and_publish_index(&fluree, &id).await; + ledger = fluree.ledger(&id).await.unwrap(); + } + let before = facts(&ledger, Some(G1), "empA").await; + assert!( + before.iter().any(|f| f.starts_with("reifiesGraph=")) + && before.iter().any(|f| f == "role=Engineer"), + "indexed={indexed}: precondition {before:?}" + ); + + let r = fluree + .update( + ledger, + &json!({ + "@context": {"ex": EX}, + "graph": G1, + "delete": { + "@id": "ex:alice", + "ex:worksFor": {"@id": "ex:acme", "@annotation": {"@id": "ex:empA"}} + }, + "opts": {"lpgEdgeLifecycle": true} + }), + ) + .await + .expect("by-id annotation delete"); + let mut ledger = r.ledger; + for phase in ["after the delete", "after the next index build"] { + assert_eq!( + facts(&ledger, Some(G1), "empA").await, + Vec::::new(), + "indexed={indexed}, {phase}: the bundle (anchor included) and the body are gone" + ); + assert_eq!( + facts(&ledger, Some(G1), "alice").await, + ["worksFor="], + "indexed={indexed}, {phase}: the base edge stays" + ); + if phase == "after the delete" { + drop(ledger); + support::rebuild_and_publish_index(&fluree, &id).await; + ledger = fluree.ledger(&id).await.unwrap(); + } + } + // Nothing of the annotation's was ever in the default graph, so + // nothing may be retracted there. + assert_eq!( + r.receipt.retract_count, 5, + "indexed={indexed}: three named slots, the anchor, and the body" + ); + } +} diff --git a/fluree-db-transact/src/stage.rs b/fluree-db-transact/src/stage.rs index 37d5076294..cb2e897caf 100644 --- a/fluree-db-transact/src/stage.rs +++ b/fluree-db-transact/src/stage.rs @@ -51,33 +51,23 @@ use fluree_db_shacl::{ShaclCache, ShaclEngine, ValidationReport}; /// their slots for list positions. pub const DELETE_WITNESSED_SITE: &str = "delete_witnessed"; -/// Build a reverse lookup from graph Sid → GraphId. -/// -/// Given `graph_sids` (ledger GraphId → Sid), returns the -/// inverse mapping. Used by SHACL/policy to determine which graph a flake -/// belongs to based on its `Flake.g` field. /// Generate cascade `f:reifies*` retraction flakes for any base edges /// being retracted in `flakes`. /// /// For each retract flake whose predicate is *not* a system-controlled -/// `f:reifies*` predicate, build the corresponding `EdgeKey` and look -/// up its currently-asserted annotations against the merged -/// snapshot+novelty view. For each annotation, emit the inverse -/// `f:reifies*` bundle so the durable encoding doesn't keep pointing -/// at a retracted edge. -/// -/// **Read path:** uses scan-based lookup through -/// `range_with_overlay`, which reads novelty + indexed base storage, -/// so annotations that have rolled into base storage post-reindex -/// are still found and cascaded. +/// `f:reifies*` predicate, build the corresponding `EdgeKey` and look up the +/// annotations that reify it. Each one's bundle is retracted **as stored**, +/// read by [`CurrentFacts::of_subject`] with the edge's graph stamped: the +/// retraction names exactly the bundle facts the ledger holds, whichever +/// builder wrote them, so a named-graph bundle is retracted in its graph with +/// its `f:reifiesGraph` anchor, and a default-graph bundle stays in the +/// default graph. /// -/// **Graph context:** the inverse bundle is emitted via -/// `EdgeKey::to_reifies_facts_jsonld_compatible`, which carries the -/// edge's `g` through to each retract flake. Named-graph assertions -/// are retracted in the same named graph; default-graph assertions -/// stay in the default graph. Without this graph-aware emission, -/// named-graph annotations would be orphaned by mismatched-graph -/// retracts. +/// **Read path:** the annotation candidates come from one POST point lookup +/// per retracted edge (`range_with_overlay`, novelty + indexed base), so +/// annotations that have rolled into base storage post-reindex are still +/// found and cascaded; each candidate's facts then take one bounded subject +/// read. /// /// **Performance:** a ledger that has never observed a `f:reifies*` /// flake pays zero — the dual-gate fast-path below returns immediately @@ -142,6 +132,19 @@ async fn cascade_attachment_retracts( fluree_vocab::db::REIFIES_SUBJECT, ); let to_t = ledger.t(); + let current = CurrentFacts::new(ledger); + let is_bundle = |f: &StoredFact| is_reserved_reifies_predicate(&f.flake().p); + let retract_all = |facts: Vec| { + facts + .into_iter() + .map(|f| f.retract(new_t).into_flake()) + .collect::>() + }; + // `EdgeKey::from_reifies_facts` reads flakes. + let decode = |bundle: &[StoredFact]| { + let flakes: Vec = bundle.iter().map(|f| f.flake().clone()).collect(); + EdgeKey::from_reifies_facts(&flakes) + }; // Annotation subjects already cascaded in pass 1 (base-edge // retract). Pass 2 (orphan-cleanup) skips these so the @@ -210,34 +213,19 @@ async fn cascade_attachment_retracts( continue; } - // SPOT scan for ALL of the candidate's flakes (system - // bundle + body metadata). Splitting after the scan - // lets us reuse the same flake set for bundle-validation - // and (for anonymous annotations) metadata cleanup. - let all_ann_flakes = fluree_db_core::range_with_overlay( - &ledger.snapshot, - g_id, - ledger.novelty.as_ref(), - IndexType::Spot, - RangeTest::Eq, - RangeMatch::new().with_subject(ann_sid.clone()), - RangeOptions::new().with_to_t(to_t), - ) - .await?; - // `from_reifies_facts` reconciles the bundle's `f:reifiesGraph` - // value against the flake-level `g`, so an indexed named-graph - // bundle decoded as `GraphMismatch` and the `Err(_) => continue` - // below swallowed it: deleting a base edge left the claim that - // reifies it live, pointing at a triple that no longer exists. - let mut all_ann_flakes = all_ann_flakes; - stamp_graph(&mut all_ann_flakes, flake.g.as_ref()); - let (bundle, metadata): (Vec, Vec) = all_ann_flakes + // All of the candidate's facts (system bundle + body + // metadata), stamped with the edge's graph: index rows decode + // without one, and `from_reifies_facts` reconciles the + // bundle's `f:reifiesGraph` against the flake-level `g`. + let (bundle, metadata): (Vec, Vec) = current + .of_subject(g_id, flake.g.as_ref(), &ann_sid) + .await? .into_iter() - .partition(|f| is_reserved_reifies_predicate(&f.p)); + .partition(is_bundle); if bundle.is_empty() { continue; } - let cand_edge = match EdgeKey::from_reifies_facts(&bundle) { + let cand_edge = match decode(&bundle) { Ok(k) => k, Err(_) => continue, }; @@ -245,11 +233,8 @@ async fn cascade_attachment_retracts( continue; } - // Build the f:reifies* retraction bundle. JSON-LD- - // compatible shape (no f:reifiesDatatype) so the - // inverse retract is byte-symmetric with the original - // assertion. - cascade.extend(edge_key.to_reifies_facts_jsonld_compatible(&ann_sid, new_t, false)); + // Retract the bundle as stored. + cascade.extend(retract_all(bundle)); // Annotation-body metadata cleanup. Two modes: // @@ -266,16 +251,7 @@ async fn cascade_attachment_retracts( let cleanup_metadata = lpg_edge_lifecycle || ann_sid.namespace_code == fluree_vocab::namespaces::BLANK_NODE; if cleanup_metadata { - for asserted in metadata { - // Mirror the asserted shape with `op = false` - // and the new transaction's `t`. Preserves - // graph (`g`), datatype, and metadata so the - // retract matches the assertion's identity. - let mut retract = asserted.clone(); - retract.t = new_t; - retract.op = false; - cascade.push(retract); - } + cascade.extend(retract_all(metadata)); } // Track this annotation as already-cascaded so the // orphan-cleanup pass below doesn't double-retract. @@ -336,35 +312,24 @@ async fn cascade_attachment_retracts( continue; } - // Scan all currently-asserted flakes for this subject. - let all_flakes = fluree_db_core::range_with_overlay( - &ledger.snapshot, - g_id, - ledger.novelty.as_ref(), - IndexType::Spot, - RangeTest::Eq, - RangeMatch::new().with_subject(ann_sid.clone()), - RangeOptions::new().with_to_t(to_t), - ) - .await?; - // Same seam as the base-edge pass above: stamp before decoding. - // The group's own retracts are this transaction's flakes for this - // subject, so they carry the graph the scan dropped. - let mut all_flakes = all_flakes; - stamp_graph( - &mut all_flakes, - retract_set.first().and_then(|f| f.g.as_ref()), - ); - let (bundle, current_metadata): (Vec, Vec) = all_flakes + // Every current fact of the subject. The group's own retracts + // are this transaction's flakes for this subject, so they carry + // the graph to stamp. + let (bundle, current_metadata): (Vec, Vec) = current + .of_subject( + g_id, + retract_set.first().and_then(|f| f.g.as_ref()), + &ann_sid, + ) + .await? .into_iter() - .partition(|f| is_reserved_reifies_predicate(&f.p)); + .partition(is_bundle); if bundle.is_empty() { continue; // not an annotation subject } - let edge_key = match EdgeKey::from_reifies_facts(&bundle) { - Ok(k) => k, - Err(_) => continue, // malformed; skip - }; + if decode(&bundle).is_err() { + continue; // malformed; skip + } // Compute the post-transaction metadata set: // (current metadata - retracts in this txn) ∪ asserts in this @@ -401,7 +366,7 @@ async fn cascade_attachment_retracts( // dedupe for free). let post_metadata: HashSet = current_metadata .iter() - .map(identity) + .map(|f| identity(f.flake())) .filter(|id| !retracted_ids.contains(id)) .chain(asserted_ids) .collect(); @@ -410,23 +375,22 @@ async fn cascade_attachment_retracts( } // Annotation has no surviving metadata after the txn → the - // user is disposing of it. Retract the bundle in the - // JSON-LD-compatible shape so it cancels the original - // assertion. - cascade.extend(edge_key.to_reifies_facts_jsonld_compatible(&ann_sid, new_t, false)); + // user is disposing of it. Retract the bundle as stored. + cascade.extend(retract_all(bundle)); } // --------------------------------------------------------------- - // Pass 3: body cleanup for user-explicit bundle retracts. + // Pass 3: completing a user-explicit bundle retract. // // The by-id retract pre-pass // (`fluree_db_transact::parse::edge_annotations::lower_delete_annotation_blocks`) - // synthesizes only `f:reifies*` retracts — the bundle, no body. - // Without this pass, the annotation's body metadata persists - // after the bundle is gone. That matches the design contract - // for explicit-IRI annotations in default RDF mode - // (user-named resources stay queryable as ordinary RDF) but - // breaks two cases: + // names the bundle's subject, predicate and object slots — not + // `f:reifiesGraph`, `f:reifiesLang` or the body. Retracting the + // required slots disposes of the attachment, so every other slot + // of the stored bundle is retracted with them: an `f:reifiesGraph` + // anchor left behind would describe nothing. + // + // The body is cleaned up in two cases: // // 1. **Anonymous (BLANK_NODE) annotations:** body becomes // orphaned bnode-keyed flakes that the wildcard-hide filter @@ -465,10 +429,8 @@ async fn cascade_attachment_retracts( } // Verify all three required slots are retracted at the same - // t. The pre-pass emits exactly those three (plus - // `f:reifiesGraph` for named-graph annotations); a partial - // user-issued retract isn't a "complete bundle retract" - // signal. + // t; a partial user-issued retract isn't a "complete bundle + // retract" signal. let has_required = [ fluree_vocab::db::REIFIES_SUBJECT, fluree_vocab::db::REIFIES_PREDICATE, @@ -487,31 +449,30 @@ async fn cascade_attachment_retracts( let is_anonymous = ann_sid.namespace_code == fluree_vocab::namespaces::BLANK_NODE; let cleanup_body = is_anonymous || lpg_edge_lifecycle; - if !cleanup_body { + // A re-pointed reifier keeps a bundle; the transaction writes it. + let complete_bundle = !repointed.contains(&ann_sid); + if !cleanup_body && !complete_bundle { continue; } - // Scan currently-asserted flakes for this annotation - // subject. Filter out the bundle (Pass 1's domain) and - // emit retracts mirroring each body assertion. - let all_flakes = fluree_db_core::range_with_overlay( - &ledger.snapshot, - g_id, - ledger.novelty.as_ref(), - IndexType::Spot, - RangeTest::Eq, - RangeMatch::new().with_subject(ann_sid.clone()), - RangeOptions::new().with_to_t(to_t), - ) - .await?; - for asserted in all_flakes { - if is_reserved_reifies_predicate(&asserted.p) { - continue; - } - let mut retract = asserted.clone(); - retract.t = new_t; - retract.op = false; - cascade.push(retract); + // Current facts of the annotation subject, stamped with the + // graph of the bundle being retracted. The slots this + // transaction already retracts come back too; the caller dedups + // cascade output against them. + let (bundle, body): (Vec, Vec) = current + .of_subject( + g_id, + bundle_retracts.first().and_then(|f| f.g.as_ref()), + &ann_sid, + ) + .await? + .into_iter() + .partition(is_bundle); + if complete_bundle { + cascade.extend(retract_all(bundle)); + } + if cleanup_body { + cascade.extend(retract_all(body)); } } @@ -522,6 +483,11 @@ async fn cascade_attachment_retracts( .await } +/// Build a reverse lookup from graph Sid → GraphId. +/// +/// Given `graph_sids` (ledger GraphId → Sid), returns the +/// inverse mapping. Used by SHACL/policy to determine which graph a flake +/// belongs to based on its `Flake.g` field. fn build_reverse_graph_lookup(graph_sids: &HashMap) -> HashMap { graph_sids .iter() From 7305a4a716819eb1038bb74089d3fb7ef0440113 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 02:02:15 -0400 Subject: [PATCH 08/36] fix(api): conflict retractions read index-resident named-graph values A take-source merge or revert, and a take-branch rebase, retract the losing side's current values on each conflicting (subject, predicate, graph). current_asserted_for_key read them with a range read and kept the rows whose graph equalled the key's, but index-resident rows come back without their graph, so in a named graph every indexed value was dropped: the take-source merge left the target's value next to the source's, and merge_preview showed no target value. Novelty-resident values and the default graph were unaffected. current_asserted_for_key now reads the slot through CurrentFacts, which stamps the graph, and the retractions come from StoredFact::retract. --- fluree-db-api/src/merge_preview.rs | 12 +- fluree-db-api/src/rebase.rs | 57 +++-- fluree-db-api/tests/grp_ledger.rs | 2 + .../it_named_graph_conflict_retractions.rs | 195 ++++++++++++++++++ 4 files changed, 229 insertions(+), 37 deletions(-) create mode 100644 fluree-db-api/tests/it_named_graph_conflict_retractions.rs diff --git a/fluree-db-api/src/merge_preview.rs b/fluree-db-api/src/merge_preview.rs index f1abb19c3f..f8b2c390cc 100644 --- a/fluree-db-api/src/merge_preview.rs +++ b/fluree-db-api/src/merge_preview.rs @@ -890,8 +890,8 @@ async fn build_conflict_details( current_asserted_for_key(source_state, &key), current_asserted_for_key(target_state, &key), )?; - let source_values = resolve_flake_list(&source_flakes, source_compactor)?; - let target_values = resolve_flake_list(&target_flakes, target_compactor)?; + let source_values = resolve_fact_list(&source_flakes, source_compactor)?; + let target_values = resolve_fact_list(&target_flakes, target_compactor)?; Ok::<_, ApiError>(ConflictDetail { key, @@ -906,13 +906,13 @@ async fn build_conflict_details( .await } -fn resolve_flake_list( - flakes: &[Flake], +fn resolve_fact_list( + facts: &[fluree_db_transact::StoredFact], compactor: &IriCompactor, ) -> Result> { - flakes + facts .iter() - .map(|flake| resolve_flake(compactor, flake)) + .map(|fact| resolve_flake(compactor, fact.flake())) .collect() } diff --git a/fluree-db-api/src/rebase.rs b/fluree-db-api/src/rebase.rs index 32705221a8..52c763ff94 100644 --- a/fluree-db-api/src/rebase.rs +++ b/fluree-db-api/src/rebase.rs @@ -8,13 +8,14 @@ use crate::commit_data::{key_of, OwnChanges}; use crate::error::{ApiError, Result}; use fluree_db_core::LedgerId; use fluree_db_core::{ - range_with_overlay, BranchedContentStore, Commit, ConflictKey, ContentId, ContentStore, Flake, - IndexType, RangeMatch, RangeOptions, RangeTest, DEFAULT_GRAPH_ID, + BranchedContentStore, Commit, ConflictKey, ContentId, ContentStore, Flake, DEFAULT_GRAPH_ID, }; use fluree_db_ledger::LedgerState; use fluree_db_nameservice::NsRecordSnapshot; use fluree_db_novelty::{delta_keys_of, FactKey}; -use fluree_db_transact::{CommitOpts, NamespaceRegistry, StagedCommit}; +use fluree_db_transact::{ + CommitOpts, CurrentFacts, NamespaceRegistry, Slot, StagedCommit, StoredFact, +}; use rustc_hash::FxHashSet; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -828,12 +829,9 @@ impl crate::Fluree { current_asserted_for_key(source_state, key) .await? .into_iter() - .filter(|flake| !reasserted.contains(&FactKey::of(flake))) - .map(|flake| Flake { - op: false, - t: 0, // restamped by StagedLedger::new - ..flake - }), + .filter(|fact| !reasserted.contains(&FactKey::of(fact.flake()))) + // `t` is restamped by StagedLedger::new. + .map(|fact| fact.retract(0).into_flake()), ); } @@ -931,10 +929,18 @@ fn find_conflicting_keys( .collect() } +/// The facts `state` currently holds for a conflict key's `(subject, +/// predicate, graph)`, graph stamped. +/// +/// Read through [`CurrentFacts`], which stamps the key's graph on index +/// rows. A range read returns an index-resident row with `g: None` whatever +/// graph it lives in, so filtering on `g == key.g` dropped every indexed +/// named-graph value: a take-source merge then left the target's value in +/// place next to the source's. pub(crate) async fn current_asserted_for_key( state: &LedgerState, key: &ConflictKey, -) -> Result> { +) -> Result> { let g_id = match &key.g { None => DEFAULT_GRAPH_ID, Some(g_sid) => match state @@ -946,26 +952,15 @@ pub(crate) async fn current_asserted_for_key( None => return Ok(Vec::new()), }, }; - - let match_val = RangeMatch::subject_predicate(key.s.clone(), key.p.clone()); - let opts = RangeOptions { - to_t: Some(state.t()), - ..Default::default() - }; - - let flakes = range_with_overlay( - &state.snapshot, + let slot = Slot { g_id, - state.novelty.as_ref(), - IndexType::Spot, - RangeTest::Eq, - match_val, - opts, - ) - .await?; - - Ok(flakes - .into_iter() - .filter(|flake| flake.op && flake.g == key.g) - .collect()) + g_sid: key.g.clone(), + s: key.s.clone(), + p: key.p.clone(), + }; + Ok(CurrentFacts::new(state) + .of_slots(std::slice::from_ref(&slot)) + .await? + .pop() + .unwrap_or_default()) } diff --git a/fluree-db-api/tests/grp_ledger.rs b/fluree-db-api/tests/grp_ledger.rs index c8ce45dba3..c12f369a6a 100644 --- a/fluree-db-api/tests/grp_ledger.rs +++ b/fluree-db-api/tests/grp_ledger.rs @@ -26,6 +26,8 @@ mod it_ledger_lifecycle; mod it_merge; #[path = "it_merge_preview.rs"] mod it_merge_preview; +#[path = "it_named_graph_conflict_retractions.rs"] +mod it_named_graph_conflict_retractions; #[path = "it_profile_ledger.rs"] mod it_profile_ledger; #[path = "it_push_after_merge.rs"] diff --git a/fluree-db-api/tests/it_named_graph_conflict_retractions.rs b/fluree-db-api/tests/it_named_graph_conflict_retractions.rs new file mode 100644 index 0000000000..280bc7d11a --- /dev/null +++ b/fluree-db-api/tests/it_named_graph_conflict_retractions.rs @@ -0,0 +1,195 @@ +//! Conflict retractions in a named graph read index-resident values. +//! +//! A take-source merge, a take-source revert and a take-branch rebase each +//! retract the losing side's current values on a conflicting `(subject, +//! predicate, graph)`. They read those values with a range read that +//! returns index-resident rows without their graph, then kept only the rows +//! whose graph equalled the key's. In a named graph that dropped every +//! indexed value, so the losing value survived next to the winning one +//! (novelty-resident values were read correctly). The read now goes through +//! the retraction resolver, which stamps the graph. + +use crate::support; +use fluree_db_api::{CommitRef, ConflictStrategy, FlureeBuilder, MergePreviewOpts}; +use serde_json::{json, Value as JsonValue}; + +const G1: &str = "http://example.org/g1"; + +fn set_value(v: &str) -> JsonValue { + json!({ + "@context": {"ex": "http://example.org/"}, + "graph": G1, + "where": {"@id": "ex:s", "ex:p": "?o"}, + "delete": {"@id": "ex:s", "ex:p": "?o"}, + "insert": {"@id": "ex:s", "ex:p": v} + }) +} + +async fn values(fluree: &support::MemoryFluree, ledger_id: &str) -> Vec { + let ledger = fluree.ledger(ledger_id).await.expect("load"); + let q = format!( + "SELECT ?o WHERE {{ GRAPH <{G1}> {{ ?o }} }}" + ); + let r = support::query_sparql(fluree, &ledger, &q) + .await + .expect("query"); + let mut out: Vec = r + .to_jsonld(&ledger.snapshot) + .expect("jsonld") + .as_array() + .expect("rows") + .iter() + .map(|row| { + let cell = row.as_array().and_then(|r| r.first()).unwrap_or(row); + cell.as_str() + .map(str::to_string) + .unwrap_or_else(|| cell.to_string()) + }) + .collect(); + out.sort(); + out +} + +async fn seed_graph(fluree: &support::MemoryFluree, ledger: fluree_db_api::LedgerState) { + fluree + .insert( + ledger, + &json!({ + "@context": {"ex": "http://example.org/"}, + "@graph": [{"@id": "ex:s", "@graph": G1, "ex:p": "orig"}] + }), + ) + .await + .expect("seed"); +} + +/// Take-source merge: the target's value is retracted whether it is in +/// novelty or in the index, and the preview shows it either way. +#[tokio::test] +async fn take_source_merge_replaces_the_targets_named_graph_value() { + for indexed in [false, true] { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger = fluree.create_ledger("mydb").await.unwrap(); + seed_graph(&fluree, ledger).await; + fluree + .create_branch("mydb", "dev", None, None) + .await + .unwrap(); + let dev = fluree.ledger("mydb:dev").await.unwrap(); + fluree.update(dev, &set_value("frombranch")).await.unwrap(); + let main = fluree.ledger("mydb:main").await.unwrap(); + fluree.update(main, &set_value("frommain")).await.unwrap(); + if indexed { + support::rebuild_and_publish_index(&fluree, "mydb:main").await; + } + + let preview = fluree + .merge_preview_with( + "mydb", + "dev", + None, + MergePreviewOpts { + include_conflict_details: true, + conflict_strategy: ConflictStrategy::TakeSource, + ..MergePreviewOpts::default() + }, + ) + .await + .unwrap(); + let detail = preview.conflicts.details.first().expect("one conflict"); + assert_eq!( + detail.target_values.len(), + 1, + "indexed={indexed}: the preview must show the target's value" + ); + + fluree + .merge_branch("mydb", "dev", None, ConflictStrategy::TakeSource) + .await + .unwrap(); + assert_eq!( + values(&fluree, "mydb:main").await, + ["frombranch"], + "indexed={indexed}" + ); + } +} + +/// Take-source revert: reverting the commit that wrote the value also +/// retracts the value HEAD holds now. +#[tokio::test] +async fn take_source_revert_retracts_the_indexed_head_value() { + for indexed in [false, true] { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger = fluree.create_ledger("mydb").await.unwrap(); + let r1 = fluree + .insert( + ledger, + &json!({ + "@context": {"ex": "http://example.org/"}, + "@graph": [{"@id": "ex:anchor", "ex:name": "anchor"}] + }), + ) + .await + .unwrap(); + let r2 = fluree + .insert( + r1.ledger, + &json!({ + "@context": {"ex": "http://example.org/"}, + "@graph": [{"@id": "ex:s", "@graph": G1, "ex:p": "v1"}] + }), + ) + .await + .unwrap(); + let v1_commit = r2.receipt.commit_id.clone(); + fluree.update(r2.ledger, &set_value("v2")).await.unwrap(); + if indexed { + support::rebuild_and_publish_index(&fluree, "mydb:main").await; + } + fluree + .revert_commits( + "mydb", + "main", + vec![CommitRef::Exact(v1_commit)], + ConflictStrategy::TakeSource, + ) + .await + .unwrap(); + assert!( + values(&fluree, "mydb:main").await.is_empty(), + "indexed={indexed}: the revert wins, so v2 goes too" + ); + } +} + +/// Take-branch rebase: the branch's value wins and the source's value, +/// index-resident on the source, is retracted. +#[tokio::test] +async fn take_branch_rebase_retracts_the_sources_indexed_value() { + for indexed in [false, true] { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger = fluree.create_ledger("mydb").await.unwrap(); + seed_graph(&fluree, ledger).await; + fluree + .create_branch("mydb", "dev", None, None) + .await + .unwrap(); + let dev = fluree.ledger("mydb:dev").await.unwrap(); + fluree.update(dev, &set_value("dev-val")).await.unwrap(); + let main = fluree.ledger("mydb:main").await.unwrap(); + fluree.update(main, &set_value("main-val")).await.unwrap(); + if indexed { + support::rebuild_and_publish_index(&fluree, "mydb:main").await; + } + fluree + .rebase_branch("mydb", "dev", ConflictStrategy::TakeBranch) + .await + .unwrap(); + assert_eq!( + values(&fluree, "mydb:dev").await, + ["dev-val"], + "indexed={indexed}" + ); + } +} From 9d9d7e5b5e86e1012af7bb7be1d104c77d5cff60 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 02:03:10 -0400 Subject: [PATCH 09/36] fix(api): a revert does not invert a legacy upsert phantom Upserts written before the retraction resolver retracted "a"@en as a tagless "a"^^rdf:langString. That retraction matched nothing and is inert for reads and indexing, but a revert inverts every flake of the reverted commits, and inverting it asserted "a" as an rdf:langString with no language tag (LANG ""), a literal that cannot exist. The undo fold now skips any retraction whose shape no stored fact has: an rdf:langString without a tag, or a tag on another datatype. It logs how many it skipped. Legacy retractions of list entries without their position are well-formed plain literals, so no shape check can catch them; telling those apart needs a historical stored-fact read and is left as a follow-up. --- fluree-db-api/src/commit_data.rs | 77 +++++++++++++ fluree-db-api/tests/grp_ledger.rs | 2 + .../tests/it_revert_legacy_phantom.rs | 105 ++++++++++++++++++ 3 files changed, 184 insertions(+) create mode 100644 fluree-db-api/tests/it_revert_legacy_phantom.rs diff --git a/fluree-db-api/src/commit_data.rs b/fluree-db-api/src/commit_data.rs index 8a2076902d..f8ccd5b3de 100644 --- a/fluree-db-api/src/commit_data.rs +++ b/fluree-db-api/src/commit_data.rs @@ -126,13 +126,90 @@ where } } Fold::Undo => { + let mut skipped = 0usize; for flakes in &ordered { for flake in flakes { + if is_malformed_retraction(flake) { + skipped += 1; + continue; + } acc.push_newest_first(&flake.invert()); } } + if skipped > 0 { + tracing::warn!( + skipped, + "revert skipped retractions no stored fact could match (an \ + rdf:langString without a language tag, or a tag on another \ + datatype); undoing one would assert an invalid literal" + ); + } } } data.flakes = acc.finish(); data } + +/// A retraction whose shape no stored fact has: an `rdf:langString` +/// without a language tag, or a tag on any other datatype. +/// +/// Upserts written before the retraction resolver rebuilt each retraction +/// from a query binding and dropped the tag, so `"a"@en` was retracted as a +/// tagless `"a"^^rdf:langString`. Such a retraction matched nothing and is +/// inert for reads and indexing, but a revert inverts every flake of the +/// reverted commits, and inverting it asserts a literal that cannot exist. +/// Legacy retractions of list entries without their position are +/// well-formed plain literals, so no shape check can tell them apart. +fn is_malformed_retraction(flake: &Flake) -> bool { + if flake.op { + return false; + } + let lang_string = flake.dt.namespace_code == fluree_vocab::namespaces::RDF + && flake.dt.name.as_ref() == "langString"; + let tagged = flake.m.as_ref().is_some_and(|m| m.lang.is_some()); + lang_string != tagged +} + +#[cfg(test)] +mod tests { + use super::*; + use fluree_db_core::{FlakeMeta, FlakeValue, Sid}; + + fn label(v: &str, dt: &str, lang: Option<&str>, t: i64, op: bool) -> Flake { + Flake::new( + Sid::new(100, "s"), + Sid::new(100, "label"), + FlakeValue::String(v.to_string()), + Sid::new(fluree_vocab::namespaces::RDF, dt), + t, + op, + lang.map(FlakeMeta::with_lang), + ) + } + + /// Reverting an upsert written before the retraction resolver: its + /// retraction of `"a"@en` lost the tag. Inverting that retraction would + /// assert `"a"^^rdf:langString` with no tag, so the undo skips it; the + /// well-formed flakes of the same commit still invert. + #[test] + fn undo_skips_retractions_no_stored_fact_could_match() { + let legacy_upsert = Commit::new( + 2, + vec![ + // The phantom: tagless langString. + label("a", "langString", None, 2, false), + // Well-formed: the upsert's assertion. + label("b", "langString", Some("en"), 2, true), + // A tag on a non-langString datatype is the other malformed + // shape. + label("c", "HTML", Some("en"), 2, false), + ], + ); + let undone = collect_from_commits([legacy_upsert], Fold::Undo).flakes; + assert_eq!(undone.len(), 1, "{undone:?}"); + let f = &undone[0]; + assert!(!f.op, "the assertion of \"b\"@en inverts to its retraction"); + assert_eq!(f.o, FlakeValue::String("b".to_string())); + assert_eq!(f.m.as_ref().and_then(|m| m.lang.as_deref()), Some("en")); + } +} diff --git a/fluree-db-api/tests/grp_ledger.rs b/fluree-db-api/tests/grp_ledger.rs index c12f369a6a..4c96259170 100644 --- a/fluree-db-api/tests/grp_ledger.rs +++ b/fluree-db-api/tests/grp_ledger.rs @@ -40,6 +40,8 @@ mod it_rebase; mod it_refresh; #[path = "it_revert.rs"] mod it_revert; +#[path = "it_revert_legacy_phantom.rs"] +mod it_revert_legacy_phantom; #[path = "it_revert_preview.rs"] mod it_revert_preview; #[path = "it_stable_hashes.rs"] diff --git a/fluree-db-api/tests/it_revert_legacy_phantom.rs b/fluree-db-api/tests/it_revert_legacy_phantom.rs new file mode 100644 index 0000000000..2d8f02bf92 --- /dev/null +++ b/fluree-db-api/tests/it_revert_legacy_phantom.rs @@ -0,0 +1,105 @@ +//! Reverting a commit that holds a legacy upsert phantom must not assert an +//! invalid literal. +//! +//! Before the retraction resolver, an upsert over `"a"@en` retracted a +//! tagless `"a"^^rdf:langString`, which matched nothing and is inert for +//! reads and indexing. Commits written then still hold it. A revert +//! inverts every flake of the reverted commit, and inverting that one +//! asserted `"a"` as an `rdf:langString` with no language tag (LANG ""), +//! a literal that cannot exist. The revert now skips such retractions. + +use crate::support; +use fluree_db_api::{CommitRef, ConflictStrategy, FlureeBuilder, IndexConfig}; +use fluree_db_core::{Flake, FlakeMeta, FlakeValue, Sid}; +use fluree_db_transact::{CommitOpts, NamespaceRegistry, StageOptions}; +use serde_json::json; + +const EX: &str = "http://example.org/"; + +#[tokio::test] +async fn revert_skips_a_tagless_lang_string_retraction() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger = fluree.create_ledger("mydb").await.unwrap(); + let ledger = fluree + .insert( + ledger, + &json!({ + "@context": {"ex": EX}, + "@id": "ex:s", + "ex:label": {"@value": "a", "@language": "en"} + }), + ) + .await + .unwrap() + .ledger; + + // The commit an old upsert of `"b"@en` wrote: the phantom retraction of + // the tagless `"a"` plus the assertion of `"b"@en`. + let s = ledger.snapshot.encode_iri(&format!("{EX}s")).unwrap(); + let p = ledger.snapshot.encode_iri(&format!("{EX}label")).unwrap(); + let lang_string = Sid::new(fluree_vocab::namespaces::RDF, "langString"); + let t = ledger.t() + 1; + let legacy = vec![ + Flake::new( + s.clone(), + p.clone(), + FlakeValue::String("a".into()), + lang_string.clone(), + t, + false, + None, + ), + Flake::new( + s, + p, + FlakeValue::String("b".into()), + lang_string, + t, + true, + Some(FlakeMeta::with_lang("en")), + ), + ]; + let ns = NamespaceRegistry::from_db(&ledger.snapshot); + let view = fluree_db_transact::stage_flakes(ledger, legacy, StageOptions::new()) + .await + .expect("stage the legacy commit"); + let (receipt, _ledger) = fluree + .commit_staged( + view, + ns, + &IndexConfig { + reindex_min_bytes: 1 << 40, + reindex_max_bytes: 1 << 41, + }, + CommitOpts::default(), + ) + .await + .expect("commit the legacy commit"); + + fluree + .revert_commits( + "mydb", + "main", + vec![CommitRef::Exact(receipt.commit_id)], + ConflictStrategy::Abort, + ) + .await + .expect("revert"); + + let ledger = fluree.ledger("mydb:main").await.unwrap(); + let r = support::query_sparql( + &fluree, + &ledger, + &format!( + "SELECT ?v ?l WHERE {{ <{EX}s> <{EX}label> ?o BIND(STR(?o) AS ?v) BIND(LANG(?o) AS ?l) }}" + ), + ) + .await + .expect("query"); + let rows = r.to_jsonld(&ledger.snapshot).expect("jsonld"); + assert_eq!( + rows, + json!([["a", "en"]]), + "the revert undoes `\"b\"@en` and asserts no tagless literal" + ); +} From 6e4aac83580a662f37052dcf66695de81102e744 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 02:04:20 -0400 Subject: [PATCH 10/36] docs(transactions): upsert replaces stored values; a DELETE retracts only stored facts upsert.md: the replacement unit is the whole (graph, subject, predicate), all languages and list positions together, and an identical refresh commits nothing, blank nodes included. The "Empty Replacement" and comparison-table text said an upsert removes predicates the payload does not name; it never did, and the page's own summary said so. retractions.md: a DELETE of a triple that is not stored commits nothing, names terms exactly (datatype exact, language tag case-insensitive), and a DELETE/INSERT of an absent triple inserts it. update-where-delete-insert.md: the same, and the comparison table's upsert granularity. --- docs/transactions/retractions.md | 21 ++++- .../update-where-delete-insert.md | 10 ++- docs/transactions/upsert.md | 83 ++++++++++++------- 3 files changed, 78 insertions(+), 36 deletions(-) diff --git a/docs/transactions/retractions.md b/docs/transactions/retractions.md index 9053a38d56..d4e506ee1d 100644 --- a/docs/transactions/retractions.md +++ b/docs/transactions/retractions.md @@ -304,14 +304,29 @@ Remove entities with specific property: ### Idempotent -Retracting a non-existent triple is a no-op: +Retracting a triple that is not stored is a no-op, and a transaction that +retracts nothing else commits nothing: ```text t=1: No triple exists -t=2: DELETE { ex:alice schema:age 30 } - Result: No change (triple didn't exist) +t=2: DELETE DATA { ex:alice schema:age 30 } + Result: nothing is committed; the ledger stays at t=1 ``` +A DELETE names RDF terms exactly. The datatype must match the stored one +(`30` is `xsd:integer`, so it does not retract `"30"^^xsd:int`); a language +tag matches case-insensitively (`"a"@EN` retracts a stored `"a"@en`). A +`DELETE`/`INSERT` of a triple that is not stored therefore inserts it — the +retraction names nothing, so it cannot cancel the insertion (SPARQL 1.1 +Update's `(DS − D) ∪ I`): + +```sparql +DELETE { ex:alice schema:age 30 } INSERT { ex:alice schema:age 30 } WHERE {} +``` + +stores `ex:alice schema:age 30` if it was absent, and commits nothing if it +was present. + ### No Cascading by Default Retracting an entity doesn't automatically retract references to it: diff --git a/docs/transactions/update-where-delete-insert.md b/docs/transactions/update-where-delete-insert.md index 515db39f16..4bd0657371 100644 --- a/docs/transactions/update-where-delete-insert.md +++ b/docs/transactions/update-where-delete-insert.md @@ -33,6 +33,12 @@ This: 2. Deletes that age value 3. Inserts the new age value +A DELETE retracts only facts that are stored. A template instantiated with a +value the ledger does not hold (a constant that is absent, or a value bound by +a different pattern) retracts nothing, and cannot cancel an INSERT of the same +triple: `DELETE { x } INSERT { x } WHERE {}` inserts `x` when it is absent. +See [Retraction Semantics](retractions.md#retraction-semantics). + ## WHERE clause capabilities The update transaction `where` clause uses the **same pattern grammar as JSON-LD queries**, so you can use rich patterns like OPTIONAL, UNION, FILTER, VALUES, and subqueries. @@ -688,8 +694,8 @@ Updates ALL entities with status="pending". | Feature | WHERE/DELETE/INSERT | Replace Mode | |---------|---------------------|--------------| -| **Granularity** | Property-level | Entity-level | -| **Other properties** | Preserved | Removed | +| **Granularity** | Value-level | Predicate-level (every value, every language) | +| **Other properties** | Preserved | Preserved | | **Conditional** | Yes (with filters) | No | | **Pattern matching** | Yes | No | | **Idempotent** | Depends on logic | Yes | diff --git a/docs/transactions/upsert.md b/docs/transactions/upsert.md index 335e741181..40f98ef361 100644 --- a/docs/transactions/upsert.md +++ b/docs/transactions/upsert.md @@ -76,8 +76,12 @@ ex:alice schema:email "alice@example.org" ``` **Operations:** -1. Retract ALL existing properties of ex:alice -2. Assert new properties +1. For each predicate in the payload (`rdf:type`, `schema:name`, `schema:email`, + `schema:age`), retract every value ex:alice currently holds for it +2. Assert the payload's values + +A value the payload restates cancels its own retraction, so it does not +appear in the commit. **Flakes:** ``` @@ -86,7 +90,6 @@ ex:alice schema:name "Alice" (retract) ex:alice schema:email "alice@example.org" (retract) # Assertions (t=2) -ex:alice rdf:type schema:Person (assert) ex:alice schema:name "Alice Smith" (assert) ex:alice schema:email "alice.smith@example.org" (assert) ex:alice schema:age 30 (assert) @@ -100,7 +103,34 @@ ex:alice schema:email "alice.smith@example.org" ex:alice schema:age 30 ``` -Note: The `@type` is re-asserted (types are always included in replace). +`rdf:type schema:Person` is unchanged: it is retracted and re-asserted in the +same transaction, and the two cancel. + +### Language Tags, Lists, and Blank Nodes + +The unit of replacement is the whole `(graph, subject, predicate)`: every +current value is retracted exactly as it is stored. + +- **Language tags.** All languages of a predicate are replaced together. An + upsert of `{"schema:name": {"@value": "Alicia", "@language": "es"}}` over + `"Alice"@en` and `"Alice"@fr` leaves only `"Alicia"@es`. To change one + language and keep the others, use + [WHERE/DELETE/INSERT](update-where-delete-insert.md) with a `FILTER` on + `lang(?v)`, or include every language in the payload. +- **Lists.** A `@list` value is replaced as a whole: every stored position is + retracted, including repeated values, and positions the new list restates + cancel. Replacing a list with plain values (or the reverse) works the same + way. +- **Blank nodes.** A blank node in an upsert payload is identified by the + payload itself, so upserting the same document again addresses the same + node, and its values are replaced like any other subject's. An identical + re-upsert of a document with blank nodes commits nothing. A changed + document mints new blank nodes; the old ones stay (see + [Stable blank-node ids](update-where-delete-insert.md#editing-blank-node-structures-stable-_fdb--ids) + for editing a stored blank node in place). +- **Named graphs.** The graph is part of the unit: an upsert into a named + graph replaces that graph's values and leaves the same predicate's values + in other graphs alone. ## Idempotency @@ -112,17 +142,13 @@ Replace mode is idempotent—repeated submissions produce the same result: ``` Result: Entity created. -**Second Submission (t=2):** +**Second Submission:** ```json {"@id": "ex:alice", "schema:name": "Alice", "schema:age": 30} ``` -Result: No actual changes (retracts and re-asserts same values). - -**Third Submission (t=3):** -```json -{"@id": "ex:alice", "schema:name": "Alice", "schema:age": 30} -``` -Result: No actual changes. +Result: nothing is committed. Each value's retraction and re-assertion cancel, +so the transaction stages no flakes and the ledger stays at t=1. This holds +for language-tagged values, lists, and the payload's blank nodes too. This makes upserts safe to retry. @@ -338,9 +364,9 @@ All types are replaced together. ## Edge Cases -### Empty Replacement +### Predicates Not in the Payload -Replacing with minimal data removes other properties: +Only the predicates the payload names are replaced: **Before (t=1):** ```json @@ -348,17 +374,15 @@ Replacing with minimal data removes other properties: "@id": "ex:alice", "schema:name": "Alice", "schema:email": "alice@example.org", - "schema:age": 30, - "schema:telephone": "+1-555-0100" + "schema:age": 30 } ``` -**Replace (t=2):** +**Upsert (t=2):** ```json { "@id": "ex:alice", - "@type": "schema:Person", - "schema:name": "Alice" + "schema:name": "Alice Smith" } ``` @@ -366,18 +390,15 @@ Replacing with minimal data removes other properties: ```json { "@id": "ex:alice", - "@type": "schema:Person", - "schema:name": "Alice" + "schema:name": "Alice Smith", + "schema:email": "alice@example.org", + "schema:age": 30 } ``` -Email, age, and telephone are removed. - -### Partial Updates Not Possible - -Replace mode replaces ALL properties—partial updates not supported. - -For partial updates, use [WHERE/DELETE/INSERT](update-where-delete-insert.md). +To remove a predicate's values, retract them with +[WHERE/DELETE/INSERT](update-where-delete-insert.md), or use +[graph sync](sync.md) to make a whole graph equal a document. ## Error Handling @@ -517,10 +538,10 @@ For partial updates, use WHERE/DELETE/INSERT: | Feature | Default Mode | Replace Mode | |---------|--------------|--------------| -| **Behavior** | Additive | Replace all | -| **Existing properties** | Preserved | Removed | +| **Behavior** | Additive | Replaces each named predicate's values | +| **Predicates not in the payload** | Preserved | Preserved | | **Idempotent** | No | Yes | -| **Partial updates** | Yes (with WHERE/DELETE/INSERT) | No | +| **Partial updates** | Yes (with WHERE/DELETE/INSERT) | Per predicate | | **Use case** | Adding data | Synchronization | | **Retry safety** | Requires care | Safe by default | | **Performance** | Fewer operations | More operations | From b06e214f7d41686225631b15a35174d4bdd0fde9 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 07:19:07 -0400 Subject: [PATCH 11/36] test(api): SPARQL DELETE DATA of a typed literal deletes the stored fact With #1988, SPARQL UPDATE lowers `"1"^^xsd:int`, `xsd:long` and `xsd:dateTime` literals as JSON-LD does, so a DELETE DATA intent names the stored term exactly and the retraction resolver deletes it, over novelty and over an index. Before #1988 the intent named a string, and the delete removed nothing. --- fluree-db-api/tests/it_delete_stored_facts.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/fluree-db-api/tests/it_delete_stored_facts.rs b/fluree-db-api/tests/it_delete_stored_facts.rs index 27005aa95d..1d48a749af 100644 --- a/fluree-db-api/tests/it_delete_stored_facts.rs +++ b/fluree-db-api/tests/it_delete_stored_facts.rs @@ -271,15 +271,13 @@ async fn delete_data_names_terms_exactly() { } } -/// SPARQL `DELETE DATA` of typed literals whose datatype SPARQL UPDATE does -/// not coerce today (`xsd:int`, `xsd:long`, `xsd:dateTime`): the lowering -/// keeps the lexical form as a string, so the intent names no stored term. -/// Before the retraction resolver that committed phantom retractions that -/// deleted nothing; with it alone nothing is committed and the values still -/// stay. #1988 lowers these literals through the same coercion JSON-LD uses, -/// after which the intent names the stored term and the resolver deletes it. +/// SPARQL `DELETE DATA` of typed literals (`xsd:int`, `xsd:long`, +/// `xsd:dateTime`). SPARQL UPDATE used to keep their lexical form as a +/// string, so the intent named no stored term and the delete committed +/// phantom retractions that removed nothing. The lowering now coerces them +/// as JSON-LD does (#1988), the intent names the stored term exactly, and +/// the resolver deletes it. #[tokio::test] -#[ignore = "needs #1988: SPARQL UPDATE keeps typed literal values"] async fn delete_data_of_typed_literals_deletes_them() { for indexed in [false, true] { let (_d, fluree, ledger) = seeded( From 0d944b0f86551de235a80752dc7bd9fa0c55a0a8 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 12:42:34 -0400 Subject: [PATCH 12/36] fix(transact): a DELETE names an indexed fact by its index key The persisted index keeps less than some terms carry. An integer too large for i64 is keyed by value alone (NUM_BIG), with no XSD subtype, and a dateTime or time is keyed to the microsecond. So an indexed fact's decode can differ from the term that names it. The index decode says xsd:integer where a big xsd:nonNegativeInteger was written, a query row says xsd:decimal, and a nanosecond literal decodes to its microsecond. Matching by term alone therefore missed these facts on an indexed ledger. DELETE DATA, a JSON-LD delete or a constant DELETE template naming a big integer under its subtype committed nothing. So did a DELETE row whose value an OPTIONAL, UNION or BIND bound. Cypher SET kept the old big value next to the new one, DETACH DELETE left it behind, and a sub-microsecond dateTime or time survived DELETE DATA. main deleted each of these, because its retraction carried the same index key. A fact read from the index is now also named by its index key: the intent's object encoded as the index keys it (o_type + o_key), compared with the fact's. Terms with one key are one fact to the index, which merges every retraction by that key. The resolver still retracts the fact as read, so every retraction names a stored fact. Term identity stays first, and novelty facts keep it alone. The key comparison runs only when no stored term matches, so a DELETE of a present term costs what it did. The witnessed-row list lookup uses the same rule. BinaryRangeProvider::persisted_object_key exposes the encoding the overlay merge already applies to retractions. --- docs/transactions/retractions.md | 6 + fluree-db-api/tests/it_delete_stored_facts.rs | 266 ++++++++++++++++++ fluree-db-query/src/binary_range.rs | 28 ++ fluree-db-query/src/binary_scan.rs | 2 +- fluree-db-transact/src/current_facts.rs | 154 +++++++--- 5 files changed, 423 insertions(+), 33 deletions(-) diff --git a/docs/transactions/retractions.md b/docs/transactions/retractions.md index d4e506ee1d..c7ece6f482 100644 --- a/docs/transactions/retractions.md +++ b/docs/transactions/retractions.md @@ -327,6 +327,12 @@ DELETE { ex:alice schema:age 30 } INSERT { ex:alice schema:age 30 } WHERE {} stores `ex:alice schema:age 30` if it was absent, and commits nothing if it was present. +Once a fact is indexed, a DELETE is matched against what the index stores, +which is less than was written for two kinds of value: an integer too large +for 64 bits keeps no XSD subtype, and a `dateTime` or `time` keeps six +fractional digits. A DELETE naming the same big integer under another integer +datatype, or the same instant to the microsecond, retracts such a fact. + ### No Cascading by Default Retracting an entity doesn't automatically retract references to it: diff --git a/fluree-db-api/tests/it_delete_stored_facts.rs b/fluree-db-api/tests/it_delete_stored_facts.rs index 1d48a749af..21510d3e35 100644 --- a/fluree-db-api/tests/it_delete_stored_facts.rs +++ b/fluree-db-api/tests/it_delete_stored_facts.rs @@ -14,6 +14,10 @@ //! nothing. Each case has SPARQL and JSON-LD twins (Cypher where its //! lowering produces the shape), on a novelty-only ledger and an indexed //! one. +//! +//! On an indexed ledger an intent also names a fact by its index key, which +//! keeps less than some terms carry: a big integer's XSD subtype, a +//! `dateTime`'s digits past the microsecond. use crate::support; use fluree_db_api::{Fluree, FlureeBuilder, LedgerState, TransactResult}; @@ -408,3 +412,265 @@ async fn witnessed_delete_of_an_indexed_big_integer_in_a_named_graph() { } } } + +// ============================================================================= +// Indexed facts are named by their index key +// ============================================================================= + +/// A big integer past the range of `i64`. The persisted index keys such a +/// value by the value alone: its XSD subtype is not stored. +const BIG: &str = "123456789012345678901234567890"; + +/// The subtypes the big-integer cells cover, each with a big value it can +/// hold. +fn big_subtypes() -> [(&'static str, String); 4] { + [ + ("nonNegativeInteger", BIG.to_string()), + ("positiveInteger", BIG.to_string()), + ("negativeInteger", format!("-{BIG}")), + ("unsignedLong", "18446744073709551615".to_string()), + ] +} + +fn node_seed() -> JsonValue { + json!({"@id": "ex:s", "@type": "ex:Node"}) +} + +const NODE_ONLY: [&str; 1] = ["type=http://example.org/ns/Node"]; + +/// A DELETE that names a big integer under its declared subtype deletes it, +/// on a novelty-only ledger and on an indexed one, where the index decodes +/// the value as `xsd:integer`: `DELETE DATA`, a JSON-LD delete, and a +/// constant `DELETE` template alike. +#[tokio::test] +async fn constant_deletes_name_big_integers_under_their_subtype() { + let mut seed = node_seed(); + let mut del = json!({"@id": "ex:s"}); + let mut triples = Vec::new(); + for (dt, v) in big_subtypes() { + let term = json!({"@value": v, "@type": format!("xsd:{dt}")}); + seed[format!("ex:{dt}")] = term.clone(); + del[format!("ex:{dt}")] = term; + triples.push(format!("ex:s ex:{dt} \"{v}\"^^xsd:{dt}")); + } + let triples = triples.join(" . "); + for indexed in [false, true] { + for form in ["data", "jsonld", "template"] { + let (_d, fluree, ledger) = seeded(&format!("big-{form}"), seed.clone(), indexed).await; + let r = match form { + "data" => { + sparql_update(&fluree, ledger, &format!("DELETE DATA {{ {triples} }}")).await + } + "template" => { + sparql_update( + &fluree, + ledger, + &format!("DELETE {{ {triples} }} WHERE {{ ex:s a ex:Node }}"), + ) + .await + } + _ => jsonld_update(&fluree, ledger, json!({"delete": del.clone()})).await, + }; + assert_eq!( + facts(&fluree, &r.ledger).await, + NODE_ONLY, + "{form}, indexed={indexed}" + ); + assert_eq!(r.receipt.retract_count, 4, "{form}, indexed={indexed}"); + } + } +} + +/// A value bound by OPTIONAL, UNION or BIND is not a witness, so its DELETE +/// row is matched against the stored facts of its slot. The row carries the +/// query's decode of the value; on an indexed ledger that decode and the +/// stored fact's differ in datatype, and the row still names the fact. +/// +/// Not covered: BIND on a novelty-only ledger with a subtype. BIND gives a +/// big integer `xsd:integer` whatever its declared subtype, so there the row +/// names a term that is not stored. That is a query-side gap, on `main` too. +#[tokio::test] +async fn where_bound_rows_name_big_integers() { + for indexed in [false, true] { + for dt in ["integer", "nonNegativeInteger"] { + let mut seed = node_seed(); + seed["ex:n"] = json!({"@value": BIG, "@type": format!("xsd:{dt}")}); + for (shape, body) in [ + ( + "optional", + "DELETE { ex:s ex:n ?o } WHERE { ex:s a ex:Node OPTIONAL { ex:s ex:n ?o } }", + ), + ( + "union", + "DELETE { ex:s ex:n ?o } WHERE { { ex:s ex:n ?o } UNION { ex:s ex:none ?o } }", + ), + ( + "bind", + "DELETE { ex:s ex:n ?o } WHERE { ex:s ex:n ?v BIND(?v AS ?o) }", + ), + ] { + if shape == "bind" && dt != "integer" && !indexed { + continue; + } + let (_d, fluree, ledger) = + seeded(&format!("big-{shape}-{dt}"), seed.clone(), indexed).await; + let r = sparql_update(&fluree, ledger, body).await; + assert_eq!( + facts(&fluree, &r.ledger).await, + NODE_ONLY, + "{shape}, xsd:{dt}, indexed={indexed}" + ); + } + let (_d, fluree, ledger) = + seeded(&format!("big-optional-jsonld-{dt}"), seed.clone(), indexed).await; + let r = jsonld_update( + &fluree, + ledger, + json!({ + "where": [{"@id": "ex:s", "@type": "ex:Node"}, ["optional", {"@id": "ex:s", "ex:n": "?o"}]], + "delete": {"@id": "ex:s", "ex:n": "?o"} + }), + ) + .await; + assert_eq!( + facts(&fluree, &r.ledger).await, + NODE_ONLY, + "JSON-LD optional, xsd:{dt}, indexed={indexed}" + ); + } + } +} + +/// Cypher `SET` replaces a big-integer property and `DETACH DELETE` removes +/// it with its node, on both lanes: both read the old value through an +/// OPTIONAL row. +#[tokio::test] +async fn cypher_set_and_detach_delete_reach_big_integers() { + for indexed in [false, true] { + let mut seed = node_seed(); + seed["ex:big"] = json!({"@value": BIG, "@type": "xsd:integer"}); + + let (_d, fluree, ledger) = seeded("big-cypher-set", seed.clone(), indexed).await; + let r = fluree + .transact_cypher( + ledger, + "MATCH (n:`http://example.org/ns/Node`) SET n.`http://example.org/ns/big` = 5", + ) + .await + .expect("cypher SET"); + assert_eq!( + facts(&fluree, &r.ledger).await, + ["big=5", "type=http://example.org/ns/Node"], + "SET, indexed={indexed}" + ); + + let (_d, fluree, ledger) = seeded("big-cypher-detach", seed, indexed).await; + let r = fluree + .transact_cypher( + ledger, + "MATCH (n:`http://example.org/ns/Node`) DETACH DELETE n", + ) + .await + .expect("cypher DETACH DELETE"); + assert!( + facts(&fluree, &r.ledger).await.is_empty(), + "DETACH DELETE, indexed={indexed}: {:?}", + facts(&fluree, &r.ledger).await + ); + } +} + +/// A `dateTime` or `time` written past the microsecond is deleted by the +/// same literal on both lanes; the index keeps microseconds. +#[tokio::test] +async fn delete_data_names_sub_microsecond_temporals() { + let seed = json!({ + "@id": "ex:s", + "ex:d": {"@value": "2020-01-01T00:00:00.123456789Z", "@type": "xsd:dateTime"}, + "ex:t": {"@value": "12:00:00.123456789", "@type": "xsd:time"} + }); + for indexed in [false, true] { + let (_d, fluree, ledger) = seeded("sub-us-sparql", seed.clone(), indexed).await; + let r = sparql_update( + &fluree, + ledger, + "DELETE DATA { ex:s ex:d \"2020-01-01T00:00:00.123456789Z\"^^xsd:dateTime . \ + ex:s ex:t \"12:00:00.123456789\"^^xsd:time }", + ) + .await; + assert_eq!(r.receipt.retract_count, 2, "SPARQL, indexed={indexed}"); + assert!( + facts(&fluree, &r.ledger).await.is_empty(), + "SPARQL, indexed={indexed}" + ); + + let (_d, fluree, ledger) = seeded("sub-us-jsonld", seed.clone(), indexed).await; + let mut del = seed.clone(); + del["@id"] = json!("ex:s"); + let r = jsonld_update(&fluree, ledger, json!({"delete": del})).await; + assert_eq!(r.receipt.retract_count, 2, "JSON-LD, indexed={indexed}"); + assert!( + facts(&fluree, &r.ledger).await.is_empty(), + "JSON-LD, indexed={indexed}" + ); + } +} + +/// The index key names one value: a DELETE of another big integer under +/// the same subtype, or of the same digits as a decimal, names nothing and +/// commits nothing. +#[tokio::test] +async fn a_different_big_value_is_not_deleted() { + let mut seed = node_seed(); + seed["ex:n"] = json!({"@value": BIG, "@type": "xsd:nonNegativeInteger"}); + for indexed in [false, true] { + let (_d, fluree, ledger) = seeded("big-other", seed.clone(), indexed).await; + let t = ledger.t(); + let r = sparql_update( + &fluree, + ledger, + &format!( + "DELETE DATA {{ ex:s ex:n \"{BIG}1\"^^xsd:nonNegativeInteger . \ + ex:s ex:n \"{BIG}.0\"^^xsd:decimal }}" + ), + ) + .await; + no_commit(&r, t, &format!("indexed={indexed}")); + assert_eq!( + facts(&fluree, &r.ledger).await, + [ + format!("n={BIG}"), + "type=http://example.org/ns/Node".to_string() + ], + "indexed={indexed}" + ); + } +} + +/// The index keys a big integer per graph and predicate, so a named graph's +/// fact is named by its own graph's key. +#[tokio::test] +async fn a_delete_names_a_big_integer_in_a_named_graph() { + let seed = json!({"@graph": [{ + "@id": "ex:s", "@graph": G1, + "ex:n": {"@value": BIG, "@type": "xsd:nonNegativeInteger"} + }]}); + for indexed in [false, true] { + let (_d, fluree, ledger) = seeded("big-graph", seed.clone(), indexed).await; + assert_eq!(numbig_rows(&fluree, &ledger).await, 1, "precondition"); + let r = sparql_update( + &fluree, + ledger, + &format!( + "DELETE DATA {{ GRAPH <{G1}> {{ ex:s ex:n \"{BIG}\"^^xsd:nonNegativeInteger }} }}" + ), + ) + .await; + assert_eq!(r.receipt.retract_count, 1, "indexed={indexed}"); + assert_eq!( + numbig_rows(&fluree, &r.ledger).await, + 0, + "indexed={indexed}" + ); + } +} diff --git a/fluree-db-query/src/binary_range.rs b/fluree-db-query/src/binary_range.rs index 0fe0c2f4b6..63009cbe00 100644 --- a/fluree-db-query/src/binary_range.rs +++ b/fluree-db-query/src/binary_range.rs @@ -320,6 +320,34 @@ impl BinaryRangeProvider { pub fn runtime_small_dicts(&self) -> &Arc { &self.runtime_small_dicts } + + /// The key a persisted row holding `flake`'s object carries in graph + /// `g_id` under the flake's predicate: `(o_type, o_key)`, the index's + /// identity for the object, as overlay retractions are matched against + /// it. + /// + /// The index keeps less than some values carry — a big integer keeps no + /// XSD subtype, a `dateTime` or `time` keeps microseconds — so distinct + /// terms can share a key. Terms that share one are one stored fact. + /// + /// `None` when no persisted row can hold the object: its predicate, + /// string, subject, language tag, datatype or big number is not in the + /// persisted dictionaries, or it is a vector (keyed by fact, not by + /// value). + pub fn persisted_object_key(&self, g_id: GraphId, flake: &Flake) -> Option<(u16, u64)> { + let p_id = self.store.sid_to_p_id(&flake.p)?; + let lang = flake.m.as_ref().and_then(|m| m.lang.as_deref()); + crate::binary_scan::value_to_otype_okey( + &flake.o, + &flake.dt, + lang, + &self.store, + None, + Some((g_id, p_id)), + ) + .ok() + .map(|(o_type, o_key)| (o_type.as_u16(), o_key)) + } } impl RangeProvider for BinaryRangeProvider { diff --git a/fluree-db-query/src/binary_scan.rs b/fluree-db-query/src/binary_scan.rs index 4acbdb858f..1d769b8784 100644 --- a/fluree-db-query/src/binary_scan.rs +++ b/fluree-db-query/src/binary_scan.rs @@ -3743,7 +3743,7 @@ fn string_not_found_error(value: &str) -> std::io::Error { /// - langString: OType must embed the lang_id, not use XSD_STRING /// - numeric subtypes: xsd:int vs xsd:integer can share the same FlakeValue::Long /// - string subtypes: xsd:anyURI vs xsd:string share FlakeValue::String -fn value_to_otype_okey( +pub(crate) fn value_to_otype_okey( val: &FlakeValue, dt_sid: &Sid, lang: Option<&str>, diff --git a/fluree-db-transact/src/current_facts.rs b/fluree-db-transact/src/current_facts.rs index 11d695eaec..27029df68a 100644 --- a/fluree-db-transact/src/current_facts.rs +++ b/fluree-db-transact/src/current_facts.rs @@ -19,6 +19,11 @@ //! [`FlakeAccumulator::push_retractions`](crate::generate::FlakeAccumulator::push_retractions) //! accepts. //! +//! An intent names a stored fact when it holds the same term: value, exact +//! datatype, and language tag compared case-insensitively. A fact read from +//! the persisted index is also named by its index key, which keeps less +//! than some terms carry (see [`IndexKeys`]). +//! //! ## The read //! //! Per `(graph, subject, predicate)` slot (or subject): one seek into the @@ -43,6 +48,7 @@ use fluree_db_core::{ RangeTest, Sid, Tracker, }; use fluree_db_ledger::LedgerState; +use fluree_db_query::BinaryRangeProvider; use std::collections::{HashMap, HashSet}; /// Seek-versus-walk break-even for a graph's novelty. Slots are answered by @@ -56,9 +62,9 @@ pub const NOVELTY_WALK_RATIO: usize = 16; /// Where a stored fact's surviving op was read from. /// /// Index rows are decodes, and some decodes are lossy — a big integer's -/// declared XSD subtype is not stored, so it decodes as `xsd:integer` or -/// `xsd:decimal`. Recording the origin keeps any value-only identity rule -/// for those rows in this one type. +/// declared XSD subtype is not stored, so it decodes as `xsd:integer` (and +/// as `xsd:decimal` in a query row). The origin tells the resolver which +/// facts it may name by index key ([`IndexKeys`]). #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Origin { /// The persisted index. @@ -218,28 +224,92 @@ fn same_term(stored: &Flake, intent: &Flake) -> bool { && lang_matches(lang_of(stored), lang_of(intent)) } -/// The stored fact an intent names, if any. An intent with a list position -/// names that position. One without names the first list entry holding the -/// term, in index order, else the plain value: the contract hydration had, -/// under which a value asserted at N list positions loses exactly one entry -/// per distinct DELETE row. -fn match_stored<'f>(facts: &'f [StoredFact], intent: &Flake) -> Option<&'f StoredFact> { - let term = |f: &&StoredFact| same_term(&f.flake, intent); - match list_index_of(intent) { - Some(i) => facts - .iter() - .filter(|f| list_index_of(&f.flake) == Some(i)) - .find(term), - None => facts +/// The stored fact an intent names, if any, where `names(n, fact)` says +/// whether `facts[n]` holds the intent's term. An intent with a list +/// position names that position. One without names the first list entry +/// holding the term, in index order, else the plain value: the contract +/// hydration had, under which a value asserted at N list positions loses +/// exactly one entry per distinct DELETE row. +fn match_stored<'f>( + facts: &'f [StoredFact], + intent: &Flake, + names: impl Fn(usize, &StoredFact) -> bool, +) -> Option<&'f StoredFact> { + let first = |position: &dyn Fn(Option) -> bool| { + facts .iter() - .filter(|f| list_index_of(&f.flake).is_some()) - .find(term) - .or_else(|| { - facts - .iter() - .filter(|f| list_index_of(&f.flake).is_none()) - .find(term) - }), + .enumerate() + .find(|(n, f)| position(list_index_of(&f.flake)) && names(*n, f)) + .map(|(_, f)| f) + }; + match list_index_of(intent) { + Some(i) => first(&|p| p == Some(i)), + None => first(&|p| p.is_some()).or_else(|| first(&|p| p.is_none())), + } +} + +/// The stored fact an intent names by term identity. +fn match_term<'f>(facts: &'f [StoredFact], intent: &Flake) -> Option<&'f StoredFact> { + match_stored(facts, intent, |_, f| same_term(&f.flake, intent)) +} + +/// Names facts read from the persisted index by their index key. +/// +/// The index keeps less than some terms carry: a big integer keeps no XSD +/// subtype, a `dateTime` or `time` keeps microseconds. An indexed fact's +/// decode can therefore differ from the term that was written, and from +/// the term a query row carries for it. Terms with one index key are one +/// stored fact to the index, which merges every retraction by that key; so +/// an intent whose key is an indexed fact's key names that fact, and the +/// resolver retracts the fact as read. +struct IndexKeys<'l> { + provider: Option<&'l BinaryRangeProvider>, + /// Per slot, computed on first use. + slot_keys: Vec>, +} + +/// The index key `(o_type, o_key)` of each fact of a slot that was read +/// from the index; `None` for a novelty fact, or one the index cannot key. +type SlotKeys = Vec>; + +impl<'l> IndexKeys<'l> { + fn new(ledger: &'l LedgerState, slots: usize) -> Self { + Self { + provider: ledger + .snapshot + .range_provider + .as_ref() + .and_then(|rp| rp.as_any().downcast_ref::()), + slot_keys: vec![None; slots], + } + } + + /// Per fact of slot `slot` (in graph `g_id`), whether it was read from + /// the index and has `intent`'s key. `None` when no fact can: no index, + /// no indexed fact in the slot, or no key for the intent (its value is + /// in no persisted dictionary, so no indexed row holds it). + fn hits( + &mut self, + slot: usize, + g_id: GraphId, + facts: &[StoredFact], + intent: &Flake, + ) -> Option> { + let provider = self.provider?; + if !facts.iter().any(|f| f.origin == Origin::Index) { + return None; + } + let key = provider.persisted_object_key(g_id, intent)?; + let keys = self.slot_keys[slot].get_or_insert_with(|| { + facts + .iter() + .map(|f| match f.origin { + Origin::Index => provider.persisted_object_key(g_id, &f.flake), + Origin::Novelty => None, + }) + .collect() + }); + Some(keys.iter().map(|k| *k == Some(key)).collect()) } } @@ -361,12 +431,14 @@ impl<'l> CurrentFacts<'l> { /// decoded, with no read: the WHERE just matched that stored fact. /// - A witnessed row on a list-bearing ledger reads its slot for the one /// thing a WHERE binding does not carry, the list position: it takes - /// the position of the first stored list entry holding the same term, - /// and otherwise stays as decoded. + /// the position of the first stored list entry holding the same term + /// (or, for an indexed entry, the same index key), and otherwise stays + /// as decoded. /// - Every other intent — a template constant, `DELETE DATA`, a value /// bound by another pattern — is matched against the stored facts of - /// its slot ([`match_stored`]). An intent that names no stored fact - /// stages nothing. + /// its slot ([`match_stored`]): by term, else, for a fact read from + /// the index, by index key ([`IndexKeys`]). The fact is retracted as + /// read. An intent that names no stored fact stages nothing. /// /// `graph_of` gives the ledger graph id of an intent's graph Sid. pub(crate) async fn resolve_intents( @@ -411,23 +483,41 @@ impl<'l> CurrentFacts<'l> { } stats.slot_reads = slots.len(); let facts = self.of_slots(&slots).await?; + let mut keys = IndexKeys::new(self.ledger, slots.len()); for (intent, idx) in pending.into_iter().zip(pending_slots) { let slot_facts = &facts[idx]; + let g_id = slots[idx].g_id; let t = intent.flake.t; if intent.witnessed { let mut flake = intent.flake; if list_index_of(&flake).is_none() { - if let Some(entry) = slot_facts + let is_entry = |f: &StoredFact| list_index_of(&f.flake).is_some(); + let entry = slot_facts .iter() - .find(|f| list_index_of(&f.flake).is_some() && same_term(&f.flake, &flake)) - { + .find(|f| is_entry(f) && same_term(&f.flake, &flake)) + .or_else(|| { + // Most witnessed rows are not list entries: no + // key encoding unless an indexed entry could match. + slot_facts + .iter() + .any(|f| is_entry(f) && f.origin == Origin::Index) + .then_some(())?; + let hits = keys.hits(idx, g_id, slot_facts, &flake)?; + let n = (0..slot_facts.len()) + .find(|&n| hits[n] && is_entry(&slot_facts[n]))?; + Some(&slot_facts[n]) + }); + if let Some(entry) = entry { flake.m = entry.flake.m.clone(); stats.enriched += 1; } } out.push(Retraction(flake)); - } else if let Some(stored) = match_stored(slot_facts, &intent.flake) { + } else if let Some(stored) = match_term(slot_facts, &intent.flake).or_else(|| { + let hits = keys.hits(idx, g_id, slot_facts, &intent.flake)?; + match_stored(slot_facts, &intent.flake, |n, _| hits[n]) + }) { stats.matched += 1; out.push(stored.clone().retract(t)); } else { From 0b05671dc55cb32a8a7d7d4ce2acf50d4cd57459 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 13:17:47 -0400 Subject: [PATCH 13/36] fix(transact): modify policy judges every delete intent, stored or not A DELETE intent that names no stored fact stages nothing. Under a policy, that also kept it from the modify-policy check, which only saw the staged flakes. A delete the identity may not perform was then refused when its target was stored and committed nothing when it was not. Intents that name no stored fact now go through the modify-policy check with the staged flakes, and are still dropped from the commit. A delete the identity may not perform is refused the same way whether or not its target is stored, as it was before intents were resolved against storage. A permitted delete of an absent value still commits nothing. Only a non-root policy context collects the unmatched intents. --- fluree-db-api/tests/it_delete_stored_facts.rs | 168 +++++++++++++++++- fluree-db-transact/src/current_facts.rs | 8 +- fluree-db-transact/src/stage.rs | 34 +++- 3 files changed, 203 insertions(+), 7 deletions(-) diff --git a/fluree-db-api/tests/it_delete_stored_facts.rs b/fluree-db-api/tests/it_delete_stored_facts.rs index 21510d3e35..4bd7ad1e45 100644 --- a/fluree-db-api/tests/it_delete_stored_facts.rs +++ b/fluree-db-api/tests/it_delete_stored_facts.rs @@ -20,7 +20,10 @@ //! `dateTime`'s digits past the microsecond. use crate::support; -use fluree_db_api::{Fluree, FlureeBuilder, LedgerState, TransactResult}; +use fluree_db_api::policy_builder::build_policy_context_from_opts; +use fluree_db_api::{ + Fluree, FlureeBuilder, GovernanceOptions, LedgerState, PolicyContext, TransactResult, +}; use serde_json::{json, Value as JsonValue}; const EX: &str = "http://example.org/ns/"; @@ -674,3 +677,166 @@ async fn a_delete_names_a_big_integer_in_a_named_graph() { ); } } + +// ============================================================================= +// Modify policy judges every delete a transaction asks for +// ============================================================================= + +async fn policy(ledger: &LedgerState, opts: GovernanceOptions) -> PolicyContext { + build_policy_context_from_opts( + &ledger.snapshot, + ledger.novelty.as_ref(), + Some(ledger.novelty.as_ref()), + ledger.t(), + &opts, + &[0], + ) + .await + .expect("build policy context") +} + +/// The identity may view and modify `ex:name` and nothing else. +async fn name_only(ledger: &LedgerState) -> PolicyContext { + policy( + ledger, + GovernanceOptions { + policy: Some(json!([{ + "@id": "http://example.org/ns/nameOnly", + "f:onProperty": [{"@id": "http://example.org/ns/name"}], + "f:action": [{"@id": "f:view"}, {"@id": "f:modify"}], + "f:allow": true + }])), + default_allow: Some(false), + ..Default::default() + }, + ) + .await +} + +/// The identity may modify everything but `ex:ssn`. +async fn all_but_ssn(ledger: &LedgerState) -> PolicyContext { + policy( + ledger, + GovernanceOptions { + policy: Some(json!([{ + "@id": "http://example.org/ns/denySsn", + "f:required": true, + "f:onProperty": [{"@id": "http://example.org/ns/ssn"}], + "f:action": "f:modify", + "f:allow": false + }])), + default_allow: Some(true), + ..Default::default() + }, + ) + .await +} + +async fn sparql_update_as( + fluree: &Fluree, + ledger: LedgerState, + body: &str, + policy: PolicyContext, +) -> Result { + let text = + format!("PREFIX ex: <{EX}>\nPREFIX xsd: \n{body}"); + let parsed = fluree_db_sparql::parse_sparql(&text); + assert!(!parsed.has_errors(), "{text}: {:?}", parsed.diagnostics); + let mut ns = fluree_db_transact::NamespaceRegistry::from_db(&ledger.snapshot); + let txn = fluree_db_transact::lower_sparql_update_ast( + &parsed.ast.expect("ast"), + &mut ns, + fluree_db_transact::TxnOpts::default(), + ) + .expect("lower"); + fluree + .stage_owned(ledger) + .txn(txn) + .policy(policy) + .execute() + .await +} + +fn salary_seed() -> JsonValue { + json!({"@id": "ex:alice", "ex:name": "Alice", "ex:salary": 100}) +} + +/// A delete the identity may not perform is refused with one answer whether +/// or not the value it names is stored: here `ex:salary 100` is stored and +/// 999 and 50 are not. +#[tokio::test] +async fn delete_refused_by_policy_regardless_of_stored_value() { + for indexed in [false, true] { + for body in [ + "DELETE DATA { ex:alice ex:salary 100 }", + "DELETE DATA { ex:alice ex:salary 999 }", + "DELETE DATA { ex:alice ex:salary 50 }", + "DELETE { ?s ex:salary 100 } WHERE { ?s ex:name ?n }", + "DELETE { ?s ex:salary 999 } WHERE { ?s ex:name ?n }", + ] { + let (_d, fluree, ledger) = seeded("policy-sparql", salary_seed(), indexed).await; + let policy = name_only(&ledger).await; + let err = sparql_update_as(&fluree, ledger, body, policy) + .await + .map(|r| r.receipt.flake_count) + .expect_err(body); + assert!( + err.to_string() + .contains("Policy enforcement prevents modification"), + "{body}, indexed={indexed}: {err}" + ); + } + for salary in [100, 999] { + let (_d, fluree, ledger) = seeded("policy-jsonld", salary_seed(), indexed).await; + let policy = name_only(&ledger).await; + let txn = json!({ + "@context": ctx(), + "delete": {"@id": "ex:alice", "ex:salary": salary} + }); + let err = fluree + .stage_owned(ledger) + .update(&txn) + .policy(policy) + .execute() + .await + .map(|r| r.receipt.flake_count) + .expect_err("JSON-LD delete"); + assert!( + err.to_string() + .contains("Policy enforcement prevents modification"), + "JSON-LD {salary}, indexed={indexed}: {err}" + ); + } + } +} + +/// An identity the policy lets modify the value: deleting it retracts it, +/// and deleting a value that is not stored commits nothing. +#[tokio::test] +async fn a_permitted_delete_of_an_absent_value_commits_nothing() { + for indexed in [false, true] { + let (_d, fluree, ledger) = seeded("policy-allowed", salary_seed(), indexed).await; + let t = ledger.t(); + let policy = all_but_ssn(&ledger).await; + let r = sparql_update_as( + &fluree, + ledger, + "DELETE DATA { ex:alice ex:salary 999 }", + policy, + ) + .await + .expect("permitted"); + no_commit(&r, t, &format!("absent value, indexed={indexed}")); + + let policy = all_but_ssn(&r.ledger).await; + let r = sparql_update_as( + &fluree, + r.ledger, + "DELETE DATA { ex:alice ex:salary 100 }", + policy, + ) + .await + .expect("permitted"); + assert_eq!(r.receipt.retract_count, 1, "indexed={indexed}"); + } +} diff --git a/fluree-db-transact/src/current_facts.rs b/fluree-db-transact/src/current_facts.rs index 27029df68a..40fb19c6b1 100644 --- a/fluree-db-transact/src/current_facts.rs +++ b/fluree-db-transact/src/current_facts.rs @@ -438,7 +438,9 @@ impl<'l> CurrentFacts<'l> { /// bound by another pattern — is matched against the stored facts of /// its slot ([`match_stored`]): by term, else, for a fact read from /// the index, by index key ([`IndexKeys`]). The fact is retracted as - /// read. An intent that names no stored fact stages nothing. + /// read. An intent that names no stored fact stages nothing; with + /// `unmatched`, it is also appended there as instantiated, for the + /// caller's modify-policy check. /// /// `graph_of` gives the ledger graph id of an intent's graph Sid. pub(crate) async fn resolve_intents( @@ -446,6 +448,7 @@ impl<'l> CurrentFacts<'l> { intents: Vec, list_free: Option<&ListFree>, graph_of: impl Fn(&Flake) -> Result, + mut unmatched: Option<&mut Vec>, ) -> Result<(Vec, ResolveStats)> { let mut stats = ResolveStats::default(); let mut out = Vec::with_capacity(intents.len()); @@ -522,6 +525,9 @@ impl<'l> CurrentFacts<'l> { out.push(stored.clone().retract(t)); } else { stats.phantom_intents += 1; + if let Some(unmatched) = unmatched.as_deref_mut() { + unmatched.push(intent.flake); + } } } Ok((out, stats)) diff --git a/fluree-db-transact/src/stage.rs b/fluree-db-transact/src/stage.rs index cb2e897caf..e3ee8adcd1 100644 --- a/fluree-db-transact/src/stage.rs +++ b/fluree-db-transact/src/stage.rs @@ -1055,13 +1055,26 @@ pub async fn stage_with_graph_delta( tracker.consume_fuel(flakes.len() as u64)?; } - // Enforce modify policies (if policy context provided and not root) + // Enforce modify policies (if policy context provided and not root). + // + // Modify policy judges every delete the transaction asked for, + // including DELETE intents that named no stored fact and stage + // nothing: a delete the identity may not perform is refused the same + // way whether or not its target is stored. if let Some(policy) = options.policy_ctx { if !policy.wrapper().is_root() { + let judged: std::borrow::Cow<'_, [Flake]> = + if stream_stats.unmatched_intents.is_empty() { + std::borrow::Cow::Borrowed(&flakes) + } else { + let mut all = flakes.clone(); + all.extend(stream_stats.unmatched_intents.iter().cloned()); + std::borrow::Cow::Owned(all) + }; let policy_span = tracing::debug_span!("policy_enforce"); async { enforce_modify_policies( - &flakes, + &judged, policy, &ledger, options.tracker, @@ -2467,6 +2480,9 @@ struct WhereStreamStats { total_binding_rows: u64, retraction_count: usize, assertion_count: usize, + /// DELETE intents that named no stored fact, as instantiated. Collected + /// only under a non-root policy, for the modify-policy check. + unmatched_intents: Vec, } /// Stream the WHERE result into `acc`, projecting → materializing encoded @@ -2791,6 +2807,10 @@ async fn stream_where_into_accumulator( } let current_facts = CurrentFacts::new(ledger); let mut resolve_stats = ResolveStats::default(); + // Under a non-root policy, intents that name no stored fact still go to + // the modify-policy check (see `stage_with_graph_delta`). + let collect_unmatched = view_policy.is_some_and(|p| !p.wrapper().is_root()); + let mut unmatched_intents: Vec = Vec::new(); generator.set_graph_aliases(graph_aliases); // Open the streaming WHERE cursor. For empty patterns it emits one @@ -2845,9 +2865,12 @@ async fn stream_where_into_accumulator( // Resolve BEFORE push (see the doc comment above). let routing: &HashMap = reverse_graph; let (retractions, stats) = current_facts - .resolve_intents(intents, list_free.as_ref(), |f| { - resolve_flake_graph_id(f, routing) - }) + .resolve_intents( + intents, + list_free.as_ref(), + |f| resolve_flake_graph_id(f, routing), + collect_unmatched.then_some(&mut unmatched_intents), + ) .instrument(delete_span.clone()) .await?; delete_span.record("retraction_count", retractions.len() as u64); @@ -2895,6 +2918,7 @@ async fn stream_where_into_accumulator( total_binding_rows, retraction_count, assertion_count, + unmatched_intents, }) } From b336fc201f0de1f2717fc6fe69b3b9999eb73209 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 13:39:54 -0400 Subject: [PATCH 14/36] fix(transact): a witness reads the graph its template writes, OPTIONAL and UNION rows included Two changes to the rule that lets a DELETE row retract without a read. Graphs compare as ledger graph ids. The rule used to compare the WHERE default's IRI with the template's IRI, and a dataset alias by name. It now takes the graph ids the WHERE dataset actually reads: its default graph when that is one ledger graph (the ledger's own with no USING, WITH or from, else the one graph those name resolves to, where the ledger's own address is its default graph), and each GRAPH 's graph. It compares them with the graph the ledger has under the template's IRI. A name the WHERE resolves to one graph and a template writes to another is never a witness. USING or WITH naming the ledger's own address reads the default graph, while a GRAPH
template writes a graph registered under that name. A test covers a ledger holding such a graph. OPTIONAL and UNION rows can witness. A triple that is the only pattern of a top-level OPTIONAL, or of one branch of a top-level UNION (or of a UNION an OPTIONAL holds alone), witnesses a template whose object variable nothing else binds. A row that binds that variable is the decode of the fact the triple matched, and a row that does not emits no intent. These rows, which include Cypher SET and the outbound half of DETACH DELETE, now retract without reading every slot. A UNION whose other branch binds the object from another predicate still goes through the resolver. --- fluree-db-api/tests/it_delete_stored_facts.rs | 105 ++++++++ .../tests/it_delete_witness_routing.rs | 113 ++++++++ fluree-db-transact/src/delete_witness.rs | 255 ++++++++++++------ fluree-db-transact/src/stage.rs | 32 ++- 4 files changed, 413 insertions(+), 92 deletions(-) diff --git a/fluree-db-api/tests/it_delete_stored_facts.rs b/fluree-db-api/tests/it_delete_stored_facts.rs index 4bd7ad1e45..3f740d3b56 100644 --- a/fluree-db-api/tests/it_delete_stored_facts.rs +++ b/fluree-db-api/tests/it_delete_stored_facts.rs @@ -24,6 +24,8 @@ use fluree_db_api::policy_builder::build_policy_context_from_opts; use fluree_db_api::{ Fluree, FlureeBuilder, GovernanceOptions, LedgerState, PolicyContext, TransactResult, }; +use fluree_db_core::comparator::IndexType; +use fluree_db_core::{range_with_overlay, FlakeValue, RangeMatch, RangeOptions, RangeTest}; use serde_json::{json, Value as JsonValue}; const EX: &str = "http://example.org/ns/"; @@ -840,3 +842,106 @@ async fn a_permitted_delete_of_an_absent_value_commits_nothing() { assert_eq!(r.receipt.retract_count, 1, "indexed={indexed}"); } } + +// ============================================================================= +// A witness reads the graph its template writes +// ============================================================================= + +/// The strings stored as `ex:s ex:p` in the graph the ledger has under +/// `iri` (`None`: the default graph), read at the flake level: a query's +/// `GRAPH ` may resolve the name another way. +async fn graph_objects(ledger: &LedgerState, iri: Option<&str>) -> Vec { + let g_id = match iri { + None => 0, + Some(iri) => ledger + .snapshot + .graph_registry + .graph_id_for_iri(iri) + .expect("graph"), + }; + let s = ledger.snapshot.encode_iri(&format!("{EX}s")).expect("s"); + let p = ledger.snapshot.encode_iri(&format!("{EX}p")).expect("p"); + let flakes = range_with_overlay( + &ledger.snapshot, + g_id, + ledger.novelty.as_ref(), + IndexType::Spot, + RangeTest::Eq, + RangeMatch::subject_predicate(s, p), + RangeOptions::new().with_to_t(ledger.t()), + ) + .await + .expect("range"); + let mut out: Vec = flakes + .iter() + .filter(|f| f.op) + .map(|f| match &f.o { + FlakeValue::String(v) => v.clone(), + other => format!("{other:?}"), + }) + .collect(); + out.sort(); + out +} + +/// A ledger with a named graph registered under its own address, which a +/// TriG block can create. `USING
` with `GRAPH
` +/// templates: the templates write that graph, and a WHERE row witnesses a +/// template only when the graph the WHERE read is the graph the template +/// writes, compared as graph ids. The row's value is inserted into that +/// graph whichever graph the WHERE reads for the address, and the default +/// graph is left alone. +#[tokio::test] +async fn a_graph_named_by_the_ledger_address_gets_what_its_templates_write() { + for indexed in [false, true] { + let dir = tempfile::tempdir().expect("tempdir"); + let fluree = FlureeBuilder::file(dir.path().to_string_lossy().to_string()) + .build() + .expect("file fluree"); + let id = format!("legacyaddress{indexed}:main"); + let ledger = fluree.create_ledger(&id).await.expect("create"); + let trig = format!( + "@prefix ex: <{EX}> .\nex:s ex:p \"d\" .\nGRAPH <{id}> {{ ex:s ex:p \"l\" . }}\n" + ); + let mut ledger = fluree + .insert_turtle(ledger, &trig) + .await + .expect("seed") + .ledger; + assert!( + ledger + .snapshot + .graph_registry + .graph_id_for_iri(&id) + .is_some(), + "the fixture registers a graph under the ledger's address" + ); + if indexed { + drop(ledger); + support::rebuild_and_publish_index(&fluree, &id).await; + ledger = fluree.ledger(&id).await.expect("reload"); + } + let r = sparql_update( + &fluree, + ledger, + &format!( + "DELETE {{ GRAPH <{id}> {{ ?s ex:p ?o }} }} \ + INSERT {{ GRAPH <{id}> {{ ?s ex:p \"d\" }} }} \ + USING <{id}> WHERE {{ ?s ex:p ?o }}" + ), + ) + .await; + assert!( + graph_objects(&r.ledger, Some(&id)) + .await + .contains(&"d".to_string()), + "indexed={indexed}: {:?}", + graph_objects(&r.ledger, Some(&id)).await + ); + assert_eq!( + graph_objects(&r.ledger, None).await, + ["d"], + "indexed={indexed}" + ); + } +} diff --git a/fluree-db-api/tests/it_delete_witness_routing.rs b/fluree-db-api/tests/it_delete_witness_routing.rs index e03fc0de54..5d135a5ad8 100644 --- a/fluree-db-api/tests/it_delete_witness_routing.rs +++ b/fluree-db-api/tests/it_delete_witness_routing.rs @@ -114,3 +114,116 @@ async fn delete_witnessed_lane_fires_only_on_a_list_free_ledger() { "a list-bearing ledger must read list positions" ); } + +/// A value bound by the sole triple of an OPTIONAL, or of one UNION branch, +/// is the decode of that triple's fact, so its row takes the witnessed lane: +/// the JSON-LD and SPARQL shapes, and Cypher `SET` and `DETACH DELETE`, +/// which read the old value through an OPTIONAL. A UNION whose other branch +/// binds the object from another predicate stays on the matched lane. +#[tokio::test(flavor = "current_thread")] +async fn optional_and_union_rows_take_the_witnessed_lane() { + let (store, _guard) = span_capture::init_test_tracing(); + let site = fluree_db_transact::DELETE_WITNESSED_SITE; + let outcomes = |from: usize| -> Vec { + store.find_events("fast-path outcome")[from..] + .iter() + .filter(|e| e.fields.get("site").map(String::as_str) == Some(site)) + .filter_map(|e| e.fields.get("outcome").cloned()) + .collect() + }; + + let fluree = FlureeBuilder::memory().build_memory(); + let ledger = fluree + .create_ledger("it/delete-witness-optional:main") + .await + .expect("create"); + let mut ledger = fluree + .insert( + ledger, + &json!({ + "@context": ctx(), + "@graph": [ + {"@id": "ex:a", "@type": "ex:Node", "ex:tag": "x"}, + {"@id": "ex:b", "@type": "ex:Node", "ex:tag": "y"}, + {"@id": "ex:c", "@type": "ex:Node", "ex:tag": "z"}, + {"@id": "ex:d", "@type": "ex:Node", "ex:tag": "w", "ex:other": "v"}, + {"@id": "ex:e", "@type": "ex:Node", "ex:tag": "u"} + ] + }), + ) + .await + .expect("seed") + .ledger; + + let cases: Vec<(&str, JsonValue, &str)> = vec![ + ( + "JSON-LD OPTIONAL", + json!({ + "@context": ctx(), + "where": [{"@id": "ex:a", "@type": "ex:Node"}, + ["optional", {"@id": "ex:a", "ex:tag": "?o"}]], + "delete": {"@id": "ex:a", "ex:tag": "?o"} + }), + "proceed", + ), + ( + "JSON-LD UNION", + json!({ + "@context": ctx(), + "where": [["union", {"@id": "ex:b", "ex:tag": "?o"}, {"@id": "ex:b", "@type": "?t"}]], + "delete": {"@id": "ex:b", "ex:tag": "?o"} + }), + "proceed", + ), + ]; + for (what, txn, expected) in cases { + let before = store.find_events("fast-path outcome").len(); + let r = fluree.update(ledger, &txn).await.expect(what); + assert_eq!(r.receipt.retract_count, 1, "{what}"); + assert_eq!(outcomes(before), [expected], "{what}"); + ledger = r.ledger; + } + + // The other branch binds the object from another predicate: matched. + let before = store.find_events("fast-path outcome").len(); + let r = fluree + .update( + ledger, + &json!({ + "@context": ctx(), + "where": [["union", {"@id": "ex:d", "ex:tag": "?o"}, {"@id": "ex:d", "ex:other": "?o"}]], + "delete": {"@id": "ex:d", "ex:tag": "?o"} + }), + ) + .await + .expect("UNION binding the object twice"); + assert_eq!(r.receipt.retract_count, 1, "the delete still retracts"); + assert!( + outcomes(before).is_empty(), + "a UNION binding the object twice is not a witness" + ); + ledger = r.ledger; + + for (what, stmt) in [ + ( + "Cypher SET", + "MATCH (n:`http://example.org/ns/Node`) WHERE n.`http://example.org/ns/tag` = 'z' \ + SET n.`http://example.org/ns/tag` = 'zz'", + ), + ( + "Cypher DETACH DELETE", + "MATCH (n:`http://example.org/ns/Node`) WHERE n.`http://example.org/ns/tag` = 'u' \ + DETACH DELETE n", + ), + ] { + let before = store.find_events("fast-path outcome").len(); + let r = fluree.transact_cypher(ledger, stmt).await.expect(what); + assert!(r.receipt.retract_count >= 1, "{what}"); + assert!( + outcomes(before).iter().any(|o| o == "proceed"), + "{what}: {:?}", + outcomes(before) + ); + ledger = r.ledger; + } +} diff --git a/fluree-db-transact/src/delete_witness.rs b/fluree-db-transact/src/delete_witness.rs index 1e898acc04..654cb83ac6 100644 --- a/fluree-db-transact/src/delete_witness.rs +++ b/fluree-db-transact/src/delete_witness.rs @@ -9,8 +9,13 @@ //! //! A template `T` is **witnessed** by a WHERE triple `W` when: //! - (a) `W` sits at the top level of the WHERE, or directly inside a -//! top-level `GRAPH`: never under OPTIONAL, UNION, MINUS, EXISTS, a -//! subquery, SERVICE or a property path; +//! top-level `GRAPH`; or it is the only pattern of an OPTIONAL there, or +//! of one branch of a top-level UNION (or of a UNION a top-level OPTIONAL +//! holds alone). Never under MINUS, EXISTS, a subquery, SERVICE or a +//! property path. An OPTIONAL or UNION row that binds `W`'s object is the +//! decode of the fact `W` matched; one that does not leaves the object +//! unbound (by (d) nothing else binds it), and a template with an unbound +//! variable emits no intent; //! - (b) `T`'s subject and predicate are `W`'s — the same variable or the //! same Sid; //! - (c) `T`'s object is a variable `v` and `W`'s object is the same `v`, @@ -23,41 +28,32 @@ //! term identity, so the row could carry the other fact's datatype or //! language tag; a filter equating `v` with another variable could be //! folded into exactly such a join; -//! - (e) `T` writes to the graph `W` read: the default-graph template with -//! the ledger's own default graph as the WHERE default; `GRAPH ` -//! templates with a WHERE default of exactly `g` (WITH, or a single -//! USING / JSON-LD `graph`/`from`) or with `W` inside `GRAPH `, `g` -//! a graph the ledger has; `GRAPH ?g` templates with `W` inside -//! `GRAPH ?g`. A `GRAPH ` whose name the WHERE dataset aliases to -//! another graph is never a witness: it reads that graph, while a -//! template naming the same IRI writes to the IRI's own graph. +//! - (e) `T` writes to the graph `W` read, compared as ledger graph ids: the +//! graph the WHERE resolved for `W` (its single default graph, or the +//! dataset's graph for `GRAPH `) is the graph the ledger has under +//! the template's IRI (the default graph for a default-graph template). +//! A graph name read one way and written another — a dataset alias, or a +//! name the WHERE maps to the default graph — is therefore never a +//! witness. `GRAPH ?g` templates need `W` inside `GRAPH ?g`. //! //! Subject and predicate are always nodes, and joins compare nodes //! exactly, so (b) needs no condition like (d). use crate::ir::{TemplateGraph, TemplateTerm, TripleTemplate}; +use fluree_db_core::GraphId; use fluree_db_query::ir::GraphName; use fluree_db_query::{Pattern, Ref, Term, TriplePattern, VarId}; -/// The WHERE clause's default graph. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum WhereDefault<'a> { - /// No USING, WITH or JSON-LD `graph`/`from`: the ledger's default graph. - Ledger, - /// Exactly one graph, named by this IRI. - Graph(&'a str), - /// Anything else: several graphs, or none (USING NAMED alone). - Other, -} - -/// What the rule needs to know about the WHERE's dataset. +/// What the rule needs to know about the WHERE's dataset, in ledger graph +/// ids: the same resolution the WHERE reads through. pub(crate) struct WitnessContext<'a> { - pub(crate) default: WhereDefault<'a>, - /// Whether the WHERE dataset resolves this graph name to a different - /// graph (a dataset alias). - pub(crate) aliased: &'a dyn Fn(&str) -> bool, - /// Whether an IRI names a graph the ledger has. - pub(crate) registered: &'a dyn Fn(&str) -> bool, + /// The graph the WHERE's default graph reads, when it reads exactly one + /// (`None`: several, or none — USING NAMED alone). + pub(crate) default: Option, + /// The graph `GRAPH ` reads in the WHERE dataset. + pub(crate) read: &'a dyn Fn(&str) -> Option, + /// The graph a template's `GRAPH ` writes, when the ledger has it. + pub(crate) written: &'a dyn Fn(&str) -> Option, } /// Where a candidate witness reads. @@ -67,12 +63,46 @@ enum ReadGraph<'p> { Named(&'p GraphName), } -/// A candidate witness and its position: top-level index, and index inside -/// a top-level `GRAPH` when it sits in one. +/// Where a candidate witness sits, so the (d) check can skip it. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Site { + /// Top-level pattern `i`: the triple, or an OPTIONAL holding only it. + Top(usize), + /// Pattern `j` of the top-level `GRAPH` at `i`: the triple, or an + /// OPTIONAL holding only it. + InGraph(usize, usize), + /// Branch `b` of the UNION at top-level `i` (see [`union_branches`]), + /// which holds only the triple. + UnionBranch(usize, usize), +} + +/// The branches of a top-level UNION, or of a UNION a top-level OPTIONAL +/// holds alone. +fn union_branches(p: &Pattern) -> Option<&[Vec]> { + match p { + Pattern::Union(branches) => Some(branches), + Pattern::Optional(group) => match group.as_slice() { + [Pattern::Union(branches)] => Some(branches), + _ => None, + }, + _ => None, + } +} + +/// A candidate witness. struct Candidate<'p> { triple: &'p TriplePattern, graph: ReadGraph<'p>, - at: (usize, Option), + at: Site, +} + +/// The triple an OPTIONAL group or UNION branch holds, when it holds only +/// that. +fn sole_triple(group: &[Pattern]) -> Option<&TriplePattern> { + match group { + [Pattern::Triple(triple)] => Some(triple), + _ => None, + } } /// For each template, whether a WHERE triple witnesses it. @@ -95,15 +125,40 @@ fn candidates(patterns: &[Pattern]) -> Vec> { Pattern::Triple(triple) => out.push(Candidate { triple, graph: ReadGraph::Default, - at: (i, None), + at: Site::Top(i), }), + Pattern::Optional(group) if sole_triple(group).is_some() => { + if let Some(triple) = sole_triple(group) { + out.push(Candidate { + triple, + graph: ReadGraph::Default, + at: Site::Top(i), + }); + } + } + Pattern::Optional(_) | Pattern::Union(_) => { + for (b, branch) in union_branches(p).into_iter().flatten().enumerate() { + if let Some(triple) = sole_triple(branch) { + out.push(Candidate { + triple, + graph: ReadGraph::Default, + at: Site::UnionBranch(i, b), + }); + } + } + } Pattern::Graph { name, patterns } => { for (j, inner) in patterns.iter().enumerate() { - if let Pattern::Triple(triple) = inner { + let triple = match inner { + Pattern::Triple(triple) => Some(triple), + Pattern::Optional(group) => sole_triple(group), + _ => None, + }; + if let Some(triple) = triple { out.push(Candidate { triple, graph: ReadGraph::Named(name), - at: (i, Some(j)), + at: Site::InGraph(i, j), }); } } @@ -133,12 +188,13 @@ fn witnesses( } // (e) let same_graph = match (w.graph, &t.graph) { - (ReadGraph::Default, TemplateGraph::Default) => cx.default == WhereDefault::Ledger, + (ReadGraph::Default, TemplateGraph::Default) => cx.default == Some(0), (ReadGraph::Default, TemplateGraph::Iri(g)) => { - cx.default == WhereDefault::Graph(g) && (cx.registered)(g) + cx.default.is_some() && cx.default == (cx.written)(g) } (ReadGraph::Named(GraphName::Iri(read)), TemplateGraph::Iri(g)) => { - read == g && !(cx.aliased)(g) && (cx.registered)(g) + let read = (cx.read)(read); + read.is_some() && read == (cx.written)(g) } (ReadGraph::Named(GraphName::Var(read)), TemplateGraph::Var(g)) => read == g, _ => false, @@ -156,18 +212,23 @@ fn same_node(t: &TemplateTerm, r: &Ref) -> bool { } /// Whether anything but the witness at `skip` mentions `v`, other than a -/// top-level FILTER reading `v` alone (see (d)). -fn mentioned_elsewhere(v: VarId, patterns: &[Pattern], skip: (usize, Option)) -> bool { +/// FILTER reading `v` alone (see (d)). +fn mentioned_elsewhere(v: VarId, patterns: &[Pattern], skip: Site) -> bool { patterns.iter().enumerate().any(|(i, p)| match p { - Pattern::Triple(_) if skip == (i, None) => false, + _ if skip == Site::Top(i) => false, Pattern::Graph { name, patterns } => { matches!(name, GraphName::Var(g) if *g == v) || patterns .iter() .enumerate() - .any(|(j, inner)| skip != (i, Some(j)) && mentions(inner, v)) + .any(|(j, inner)| skip != Site::InGraph(i, j) && mentions(inner, v)) } - other => mentions(other, v), + other => match union_branches(other) { + Some(branches) => branches.iter().enumerate().any(|(b, branch)| { + skip != Site::UnionBranch(i, b) && branch.iter().any(|inner| mentions(inner, v)) + }), + None => mentions(other, v), + }, }) } @@ -217,19 +278,20 @@ mod tests { ) } - fn registered(iri: &str) -> bool { - iri == GRAPH_IRI - } + /// The ledger has one named graph, `GRAPH_IRI`, as graph id 1. + const G1: GraphId = 1; - fn not_aliased(_: &str) -> bool { - false + fn written(iri: &str) -> Option { + (iri == GRAPH_IRI).then_some(G1) } - fn cx(default: WhereDefault<'_>) -> WitnessContext<'_> { + /// A WHERE dataset that reads every name as the graph the ledger has + /// under it. + fn cx(default: Option) -> WitnessContext<'static> { WitnessContext { default, - aliased: ¬_aliased, - registered: ®istered, + read: &written, + written: &written, } } @@ -239,7 +301,7 @@ mod tests { #[test] fn delete_where_and_same_shape_updates_are_witnessed() { - let ledger = cx(WhereDefault::Ledger); + let ledger = cx(Some(0)); assert!(check(&[where_sp_o()], t_sp_o(), &ledger)); // DELETE WHERE { ?s ?p ?o } let spo = triple(Ref::Var(S), Ref::Var(P), Term::Var(O)); @@ -272,7 +334,7 @@ mod tests { #[test] fn constants_and_constraints_are_not_witnessed() { - let ledger = cx(WhereDefault::Ledger); + let ledger = cx(Some(0)); // DELETE DATA: no WHERE at all. assert!(!check(&[], t_sp_o(), &ledger)); // A constant object. @@ -303,7 +365,7 @@ mod tests { #[test] fn an_object_bound_twice_is_not_witnessed() { - let ledger = cx(WhereDefault::Ledger); + let ledger = cx(Some(0)); // Cross-pattern: the object is joined with another predicate's value. let other = triple(Ref::Var(X), Ref::Sid(Sid::new(100, "q")), Term::Var(O)); assert!(!check(&[where_sp_o(), other], t_sp_o(), &ledger)); @@ -334,12 +396,53 @@ mod tests { } #[test] - fn optional_union_and_minus_do_not_witness() { - let ledger = cx(WhereDefault::Ledger); + fn a_sole_optional_or_union_triple_witnesses() { + let ledger = cx(Some(0)); + // `?s ex:q ?x OPTIONAL { ?s ex:p ?o }`: the Cypher SET shape. + let q = triple(Ref::Var(S), Ref::Sid(Sid::new(100, "q")), Term::Var(X)); let optional = Pattern::Optional(vec![where_sp_o()]); + assert!(check(&[q.clone(), optional.clone()], t_sp_o(), &ledger)); + assert!(check(&[optional], t_sp_o(), &ledger)); + // One UNION branch, the object bound in no other. + let union = Pattern::Union(vec![vec![where_sp_o()], vec![q.clone()]]); + assert!(check(&[union], t_sp_o(), &ledger)); + // A UNION an OPTIONAL holds alone: the Cypher DETACH DELETE shape, + // whose inbound branch binds the object elsewhere. + let s_p_n = triple(Ref::Var(X), Ref::Var(P), Term::Var(S)); + let detach = Pattern::Optional(vec![Pattern::Union(vec![ + vec![triple(Ref::Var(S), Ref::Var(P), Term::Var(O))], + vec![s_p_n], + ])]); + let out_t = TripleTemplate::new( + TemplateTerm::Var(S), + TemplateTerm::Var(P), + TemplateTerm::Var(O), + ); + assert!(check(&[detach], out_t, &ledger)); + // Inside a top-level GRAPH. + let in_graph = Pattern::Graph { + name: GraphName::Iri(Arc::from(GRAPH_IRI)), + patterns: vec![q, Pattern::Optional(vec![where_sp_o()])], + }; + assert!(check(&[in_graph], t_sp_o().in_graph(GRAPH_IRI), &ledger)); + } + + #[test] + fn optional_union_and_minus_do_not_witness_otherwise() { + let ledger = cx(Some(0)); + let q_o = triple(Ref::Var(S), Ref::Sid(Sid::new(100, "q")), Term::Var(O)); + // An OPTIONAL holding more than the triple. + let optional = Pattern::Optional(vec![where_sp_o(), q_o.clone()]); assert!(!check(&[optional], t_sp_o(), &ledger)); - let union = Pattern::Union(vec![vec![where_sp_o()], vec![]]); + // Another UNION branch binds the object from another predicate. + let union = Pattern::Union(vec![vec![where_sp_o()], vec![q_o.clone()]]); assert!(!check(&[union], t_sp_o(), &ledger)); + // A UNION branch holding more than the triple. + let union = Pattern::Union(vec![vec![where_sp_o(), q_o], vec![]]); + assert!(!check(&[union], t_sp_o(), &ledger)); + // A nested OPTIONAL. + let nested = Pattern::Optional(vec![Pattern::Optional(vec![where_sp_o()])]); + assert!(!check(&[nested], t_sp_o(), &ledger)); // A top-level witness plus an OPTIONAL that mentions the object. let optional = Pattern::Optional(vec![triple( Ref::Var(S), @@ -359,39 +462,27 @@ mod tests { fn graph_contexts_must_agree() { let in_graph = t_sp_o().in_graph(GRAPH_IRI); // WITH / a single USING . - assert!(check( - &[where_sp_o()], - in_graph.clone(), - &cx(WhereDefault::Graph(GRAPH_IRI)) - )); + assert!(check(&[where_sp_o()], in_graph.clone(), &cx(Some(G1)))); // ... but the ledger default read by a named-graph template is not. - assert!(!check( - &[where_sp_o()], - in_graph.clone(), - &cx(WhereDefault::Ledger) - )); + assert!(!check(&[where_sp_o()], in_graph.clone(), &cx(Some(0)))); // USING with a default-graph template. - assert!(!check( - &[where_sp_o()], - t_sp_o(), - &cx(WhereDefault::Graph(GRAPH_IRI)) - )); + assert!(!check(&[where_sp_o()], t_sp_o(), &cx(Some(G1)))); // Several USING graphs. - assert!(!check(&[where_sp_o()], t_sp_o(), &cx(WhereDefault::Other))); - // A graph the ledger does not have. + assert!(!check(&[where_sp_o()], t_sp_o(), &cx(None))); + // A graph the ledger does not have: the WHERE reads some graph, the + // template writes a new one. let unknown = t_sp_o().in_graph("http://example.org/nope"); - assert!(!check( - &[where_sp_o()], - unknown, - &cx(WhereDefault::Graph("http://example.org/nope")) - )); + assert!(!check(&[where_sp_o()], unknown, &cx(Some(0)))); + // One name, read as the default graph and written as a named graph + // the ledger has under it: not the same graph. + assert!(!check(&[where_sp_o()], in_graph.clone(), &cx(Some(0)))); // GRAPH { … } let graph_block = |name: GraphName| Pattern::Graph { name, patterns: vec![where_sp_o()], }; - let ledger = cx(WhereDefault::Ledger); + let ledger = cx(Some(0)); assert!(check( &[graph_block(GraphName::Iri(Arc::from(GRAPH_IRI)))], in_graph.clone(), @@ -403,10 +494,10 @@ mod tests { &ledger )); // A name the dataset aliases to another graph reads that graph. - let alias_of_g1 = |iri: &str| iri == GRAPH_IRI; + let alias_of_g2 = |iri: &str| (iri == GRAPH_IRI).then_some(2); let aliased = WitnessContext { - aliased: &alias_of_g1, - ..cx(WhereDefault::Ledger) + read: &alias_of_g2, + ..cx(Some(0)) }; assert!(!check( &[graph_block(GraphName::Iri(Arc::from(GRAPH_IRI)))], diff --git a/fluree-db-transact/src/stage.rs b/fluree-db-transact/src/stage.rs index e3ee8adcd1..d533cba3a5 100644 --- a/fluree-db-transact/src/stage.rs +++ b/fluree-db-transact/src/stage.rs @@ -9,7 +9,7 @@ //! staged view against SHACL shapes. use crate::current_facts::{CurrentFacts, ListFree, ResolveStats, Retraction, Slot, StoredFact}; -use crate::delete_witness::{witnessed_templates, WhereDefault, WitnessContext}; +use crate::delete_witness::{witnessed_templates, WitnessContext}; use crate::error::{Result, TransactError}; use crate::generate::{infer_datatype, FlakeAccumulator, FlakeGenerator}; use crate::ir::InlineValues; @@ -2758,10 +2758,13 @@ async fn stream_where_into_accumulator( } let mut seen_named_keys: HashSet> = HashSet::new(); let mut graph_aliases: HashMap, Arc> = HashMap::new(); + // The graph each `GRAPH ` reads, for the witness rule. + let mut named_reads: HashMap, GraphId> = HashMap::new(); for (name, g_id, enumerable, canonical) in named { if !seen_named_keys.insert(name.clone()) { continue; } + named_reads.insert(name.clone(), g_id); runtime_dataset = if enumerable { runtime_dataset.with_named_graph(name, make_graph_ref(g_id)) } else { @@ -2774,24 +2777,33 @@ async fn stream_where_into_accumulator( // Which DELETE templates a WHERE triple witnesses, and whether the // ledger can hold a list position at all: together they decide which // retractions need no read (see `CurrentFacts::resolve_intents`). + // + // Graphs compare as the ledger graph ids the dataset above reads: its + // default graph when that is one ledger graph, and `named_reads` for + // `GRAPH `. The default graph is the ledger's own with no + // `USING`/`WITH`/`from`, and otherwise the one graph those name resolves + // to (`resolve_where_default_graph`, where the ledger's own address is its + // default graph). An empty default graph (`USING NAMED` alone, or a graph + // that does not exist) or a union of several witnesses nothing. let where_default = if where_default_is_empty { - WhereDefault::Other + None + } else if desired_where_default_graph_iris.is_empty() { + Some(base_db.g_id) } else { - match desired_where_default_graph_iris.as_slice() { - [] => WhereDefault::Ledger, - [iri] => WhereDefault::Graph(iri), - _ => WhereDefault::Other, + match where_default_g_ids.as_slice() { + [Some(g_id)] => Some(*g_id), + _ => None, } }; - let registered = |iri: &str| resolve_graph_id(iri).is_some(); - let aliased = |iri: &str| graph_aliases.contains_key(iri); + let read = |iri: &str| named_reads.get(iri).copied(); + let written = |iri: &str| resolve_graph_id(iri); let witnessed = witnessed_templates( &query_patterns, &txn.delete_templates, &WitnessContext { default: where_default, - aliased: &aliased, - registered: ®istered, + read: &read, + written: &written, }, ); let list_free = ListFree::check(ledger); From a693f2cbcc506d8529af3ea273f8b4f133a583bf Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 13:46:04 -0400 Subject: [PATCH 15/36] docs(transactions): upsert is retry-safe except for values stored with less detail An identical re-upsert commits nothing, except for a value stored with less detail than it was written: an integer beyond 64 bits written with an XSD subtype, or a dateTime or time with digits past the microsecond. Re-upserting such a value as written commits a retraction and an assertion of the same stored fact, and the value can be lost. This is not new in this branch. The upsert page now says so and says how to write such values. A second ignored U5 test pins the temporal case next to the big-integer one. Also: - A test pins that a refresh upsert of every language's label satisfies sh:uniqueLang, and that a second label in a language is still refused. - The of_slots doc no longer claims duplicate slots are read once. - A stale comment about SPARQL typed literals is removed. --- docs/transactions/upsert.md | 12 +++- fluree-db-api/src/shacl_tests.rs | 59 +++++++++++++++++++ fluree-db-api/tests/it_delete_stored_facts.rs | 5 +- .../tests/it_upsert_stored_retractions.rs | 19 ++++++ fluree-db-transact/src/current_facts.rs | 6 +- 5 files changed, 93 insertions(+), 8 deletions(-) diff --git a/docs/transactions/upsert.md b/docs/transactions/upsert.md index 40f98ef361..5191a94a5d 100644 --- a/docs/transactions/upsert.md +++ b/docs/transactions/upsert.md @@ -8,7 +8,7 @@ Upsert operations provide idempotent transactions by **replacing the values of t - If the entity exists: for each predicate present in your payload, retract existing values for that predicate and assert the new value(s) - If the entity doesn’t exist: create it with the supplied triples -This makes upserts safe to retry: sending the same upsert repeatedly produces the same current-state values for those predicates. +This makes upserts safe to retry: sending the same upsert repeatedly produces the same current-state values for those predicates, with one exception for values stored with less detail than they were written (see [Idempotency](#idempotency)). ## HTTP Endpoint @@ -150,7 +150,15 @@ Result: nothing is committed. Each value's retraction and re-assertion cancel, so the transaction stages no flakes and the ledger stays at t=1. This holds for language-tagged values, lists, and the payload's blank nodes too. -This makes upserts safe to retry. +This makes upserts safe to retry, with one exception. A value stored with +less detail than it was written is not re-upserted as a no-op: an integer too +large for 64 bits written with an XSD subtype such as `xsd:nonNegativeInteger` +(the index keeps no subtype), or a `dateTime` or `time` with digits past the +microsecond (storage keeps six). Re-upserting such a value as written commits +a retraction and an assertion of the same stored fact, and the value can be +lost. Write these values at the precision they are stored with: `xsd:integer` +for big integers, and at most six fractional digits for temporals. Graph sync +behaves the same way. ## Comparison: Insert vs Update vs Upsert diff --git a/fluree-db-api/src/shacl_tests.rs b/fluree-db-api/src/shacl_tests.rs index ea177af83e..adcd4df2fb 100644 --- a/fluree-db-api/src/shacl_tests.rs +++ b/fluree-db-api/src/shacl_tests.rs @@ -3718,6 +3718,65 @@ async fn shacl_unique_lang() { assert_shacl_violation(err, "more than one value"); } +/// An upsert that replaces every language's label satisfies `sh:uniqueLang`: +/// the replaced values are retracted with their language tags, so the +/// staged state holds one label per language. A second label in a language +/// is still refused. +#[tokio::test] +async fn a_refresh_upsert_satisfies_unique_lang() { + let fluree = FlureeBuilder::memory().build_memory(); + let context = shacl_context(); + let shape_txn = json!({ + "@context": context.clone(), + "@id": "ex:UniqueLabelShape", + "@type": "sh:NodeShape", + "sh:targetClass": {"@id": "ex:Translated"}, + "sh:property": [{ + "@id": "ex:pshape_unique_label", + "sh:path": {"@id": "ex:label"}, + "sh:uniqueLang": true + }] + }); + let ledger = fluree + .create_ledger("shacl/uniq-refresh:main") + .await + .unwrap(); + let ledger = fluree.upsert(ledger, &shape_txn).await.unwrap().ledger; + let doc = |en: &str, fr: &str| { + json!({ + "@context": context.clone(), + "@id": "ex:doc1", + "@type": "ex:Translated", + "ex:label": [ + {"@value": en, "@language": "en"}, + {"@value": fr, "@language": "fr"} + ] + }) + }; + let ledger = fluree + .upsert(ledger, &doc("colour", "couleur")) + .await + .expect("first labels") + .ledger; + let ledger = fluree + .upsert(ledger, &doc("color", "coloris")) + .await + .expect("a refresh replaces each language's label") + .ledger; + let err = fluree + .insert( + ledger, + &json!({ + "@context": context.clone(), + "@id": "ex:doc1", + "ex:label": {"@value": "hue", "@language": "en"} + }), + ) + .await + .unwrap_err(); + assert_shacl_violation(err, "more than one value"); +} + /// `sh:pattern` on IRI values matches the full decoded IRI (SPARQL `STR()`), /// not the SID name fragment — and non-matching IRIs violate. #[tokio::test] diff --git a/fluree-db-api/tests/it_delete_stored_facts.rs b/fluree-db-api/tests/it_delete_stored_facts.rs index 3f740d3b56..f9e70bd07a 100644 --- a/fluree-db-api/tests/it_delete_stored_facts.rs +++ b/fluree-db-api/tests/it_delete_stored_facts.rs @@ -264,9 +264,8 @@ async fn delete_data_names_terms_exactly() { &format!("xsd:integer vs stored xsd:int (indexed={indexed})"), ); assert_eq!(facts(&fluree, &r.ledger).await, ["n=1"]); - // The same term deletes it. (Spelled in JSON-LD: SPARQL UPDATE lowers - // `"1"^^xsd:int` to a string value, a separate gap that deletes - // nothing before or after this change.) + // The same term deletes it (spelled in JSON-LD here; + // `delete_data_of_typed_literals_deletes_them` covers SPARQL). let r = jsonld_update( &fluree, r.ledger, diff --git a/fluree-db-api/tests/it_upsert_stored_retractions.rs b/fluree-db-api/tests/it_upsert_stored_retractions.rs index a4402bd8e4..bb02f1743f 100644 --- a/fluree-db-api/tests/it_upsert_stored_retractions.rs +++ b/fluree-db-api/tests/it_upsert_stored_retractions.rs @@ -559,3 +559,22 @@ async fn an_identical_reupsert_keeps_an_indexed_big_integer_subtype() { h.upsert(&doc).await; assert_eq!(h.facts(None, "s").await.len(), 1, "the value survives"); } + +/// The same class for temporals: storage keeps a `dateTime` to the +/// microsecond, so a value written with more digits decodes differently from +/// the term that is re-upserted, and the retraction of the decode and the +/// assertion of the written term share one storage key. +#[tokio::test] +#[ignore = "U5: storage keeps temporals to the microsecond (filing held, AJ-4)"] +async fn an_identical_reupsert_keeps_a_sub_microsecond_datetime() { + let mut h = Harness::new("u5-datetime").await; + let doc = jsonld(json!({ + "@context": {"ex": EX, "xsd": "http://www.w3.org/2001/XMLSchema#"}, + "@id": "ex:s", + "ex:d": {"@value": "2020-01-01T00:00:00.123456789Z", "@type": "xsd:dateTime"} + })); + h.upsert(&doc).await; + h.reindex().await; + h.upsert(&doc).await; + assert_eq!(h.facts(None, "s").await.len(), 1, "the value survives"); +} diff --git a/fluree-db-transact/src/current_facts.rs b/fluree-db-transact/src/current_facts.rs index 40fb19c6b1..6c92220401 100644 --- a/fluree-db-transact/src/current_facts.rs +++ b/fluree-db-transact/src/current_facts.rs @@ -344,9 +344,9 @@ impl<'l> CurrentFacts<'l> { /// The current facts of every slot, in slot order: one `Vec` per slot, /// each in SPOT order. /// - /// Duplicate slots are answered once and cloned. Slots of one graph - /// share one decision between per-slot novelty seeks and a single - /// filtered novelty walk. + /// Each slot is read on its own (callers pass distinct slots). Slots of + /// one graph share one decision between per-slot novelty seeks and a + /// single filtered novelty walk. pub async fn of_slots(&self, slots: &[Slot]) -> Result>> { let novelty = NoveltyPlan::for_slots(self.ledger, slots, self.to_t); let mut out = Vec::with_capacity(slots.len()); From 41579486aa3844061f51f1236d83174ab2f27b20 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 02:09:16 -0400 Subject: [PATCH 16/36] bench(api): insert_formats measures Turtle upsert and mixed TriG insert Adds three columns over the existing people data: upsert_turtle (the Turtle bodies upserted into a fresh ledger), upsert_turtle_replace (the same upserts over a ledger that already holds them, so every subject's values are read and restated), and trig_mixed (a TriG insert of each body with its second half in a trailing graph block, the documents the streaming Turtle parse stops on). These are the RDF-text lanes the next commits move off the JSON-LD round trip; this records them first. --- BENCHMARKING.md | 2 +- fluree-db-api/benches/insert_formats.rs | 213 +++++++++++++++++++++++- 2 files changed, 213 insertions(+), 2 deletions(-) diff --git a/BENCHMARKING.md b/BENCHMARKING.md index ca7e326dcd..337ed3e572 100644 --- a/BENCHMARKING.md +++ b/BENCHMARKING.md @@ -72,7 +72,7 @@ this table, so it is the one that rots. Add a row when you add a bench file. | Crate | Bench file | Topic | |---|---|---| -| `fluree-db-api` | `insert_formats.rs` | JSON-LD vs Turtle insert throughput, matrix of (format × txn count × nodes/txn) | +| `fluree-db-api` | `insert_formats.rs` | JSON-LD vs Turtle insert throughput, matrix of (format × txn count × nodes/txn); also Turtle upsert (new subjects and over existing ones) and TriG insert with a trailing graph block | | `fluree-db-api` | `vector_query.rs` | End-to-end vector similarity through the query engine, 1K/5K articles, 768-dim | | `fluree-db-api` | `fulltext_query.rs` | Full-text query through novelty + index | | `fluree-db-api` | `import_bulk.rs` | Bulk Turtle import via `fluree.create(id).import(path).execute()`; single- vs default-threaded | diff --git a/fluree-db-api/benches/insert_formats.rs b/fluree-db-api/benches/insert_formats.rs index 35a1e364a2..b8571aa13d 100644 --- a/fluree-db-api/benches/insert_formats.rs +++ b/fluree-db-api/benches/insert_formats.rs @@ -14,7 +14,14 @@ // // formats: jsonld, turtle, trig (the same Turtle body inside one // `GRAPH { … }` block — exercises the named-graph -// structured path, which Turtle and JSON-LD never touch) +// structured path, which Turtle and JSON-LD never touch), +// upsert_turtle (the Turtle bodies upserted into a fresh +// ledger: new subjects), upsert_turtle_replace (the same +// upserts over a ledger that already holds them: every +// subject's values are read and restated), and trig_mixed +// (TriG insert of each body with its first half in the +// default graph and its second half in a trailing block — +// the documents the streaming Turtle parse stops on) // txn counts: 10, 100 // nodes/txn: 10, 100, 1000 // @@ -82,6 +89,11 @@ struct PregenData { turtle_txns: Vec, /// `turtle_txns` with the body wrapped in a single named-graph block. trig_txns: Vec, + /// `turtle_txns` with the second half of the body in a trailing block. + trig_mixed_txns: Vec, + /// `turtle_txns` with every statement in its own block, under extra + /// prefix declarations. + trig_blocks_txns: Vec, /// Total flakes produced by inserting all transactions (calibrated once). total_flakes: u64, } @@ -99,12 +111,16 @@ fn pregen( let mut jsonld_txns = Vec::with_capacity(txn_count); let mut turtle_txns = Vec::with_capacity(txn_count); let mut trig_txns = Vec::with_capacity(txn_count); + let mut trig_mixed_txns = Vec::with_capacity(txn_count); + let mut trig_blocks_txns = Vec::with_capacity(txn_count); for txn_idx in 0..txn_count { let data = generate_txn_data(txn_idx, nodes_per_txn); jsonld_txns.push(txn_data_to_jsonld(&data)); let turtle = txn_data_to_turtle(&data); trig_txns.push(turtle_to_trig(&turtle)); + trig_mixed_txns.push(turtle_to_trig_mixed(&turtle)); + trig_blocks_txns.push(turtle_to_trig_blocks(&turtle)); turtle_txns.push(turtle); } @@ -135,6 +151,8 @@ fn pregen( jsonld_txns, turtle_txns, trig_txns, + trig_mixed_txns, + trig_blocks_txns, total_flakes, } } @@ -159,6 +177,70 @@ fn turtle_to_trig(turtle: &str) -> String { out } +/// Keep the first half of a Turtle document's statements in the default +/// graph and move the second half into one trailing `GRAPH { … }` block. +/// Statements are the blank-line-separated paragraphs `txn_data_to_turtle` +/// writes, after its `@prefix` lines. +fn turtle_to_trig_mixed(turtle: &str) -> String { + let (prefixes, body): (Vec<&str>, Vec<&str>) = + turtle.lines().partition(|l| l.starts_with("@prefix")); + let body = body.join("\n"); + let statements: Vec<&str> = body + .split("\n\n") + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect(); + let split = statements.len() / 2; + let mut out = String::with_capacity(turtle.len() + 64); + for p in prefixes { + out.push_str(p); + out.push('\n'); + } + for st in &statements[..split] { + out.push_str(st); + out.push_str("\n\n"); + } + out.push_str("GRAPH {\n"); + for st in &statements[split..] { + out.push_str(st); + out.push_str("\n\n"); + } + out.push_str("}\n"); + out +} + +/// Every statement of a Turtle document in its own `GRAPH { … }` block, +/// over ten graphs, with sixteen more prefix declarations than the document +/// uses: the many-small-blocks shape, where per-block parser setup would +/// dominate. +fn turtle_to_trig_blocks(turtle: &str) -> String { + let (prefixes, body): (Vec<&str>, Vec<&str>) = + turtle.lines().partition(|l| l.starts_with("@prefix")); + let body = body.join("\n"); + let mut out = String::with_capacity(turtle.len() * 2); + for p in prefixes { + out.push_str(p); + out.push('\n'); + } + for i in 0..16 { + out.push_str(&format!( + "@prefix x{i}: .\n" + )); + } + for (k, st) in body + .split("\n\n") + .map(str::trim) + .filter(|s| !s.is_empty()) + .enumerate() + { + out.push_str(&format!( + "GRAPH {{\n{st}\n}}\n", + k % 10 + )); + } + out +} + // --------------------------------------------------------------------------- // Insert runners // --------------------------------------------------------------------------- @@ -210,6 +292,29 @@ async fn run_turtle_inserts( ledger } +async fn run_turtle_upserts( + fluree: &BenchFluree, + ledger: BenchLedger, + txns: &[String], + index_config: &IndexConfig, +) -> BenchLedger { + let mut ledger = ledger; + for txn in txns { + let result = fluree + .upsert_turtle_with_opts( + ledger, + txn, + TxnOpts::default(), + CommitOpts::default(), + index_config, + ) + .await + .unwrap(); + ledger = result.ledger; + } + ledger +} + /// TriG goes through the transact builder's `upsert_turtle`: the plain /// `insert_turtle_with_opts` path and the builder's `insert_turtle` fast path /// both skip `GRAPH`-block extraction. The ledger is addressed by alias, so no @@ -419,6 +524,112 @@ fn bench_insert_formats(c: &mut Criterion) { }); }); + // --- Turtle upsert, new subjects --- + group.bench_with_input( + BenchmarkId::new("upsert_turtle", ¶m_label), + &data, + |b, data| { + b.iter(|| { + let alias = next_ledger_alias("ttl-up"); + rt.block_on(async { + let ledger = fluree.create_ledger(&alias).await.unwrap(); + black_box( + run_turtle_upserts( + &fluree, + ledger, + &data.turtle_txns, + &index_config, + ) + .await, + ) + }) + }); + }, + ); + + // --- Turtle upsert over the same data (read + restate) --- + group.bench_with_input( + BenchmarkId::new("upsert_turtle_replace", ¶m_label), + &data, + |b, data| { + b.iter_batched( + || { + rt.block_on(async { + let alias = next_ledger_alias("ttl-up-rep"); + let ledger = fluree.create_ledger(&alias).await.unwrap(); + run_turtle_inserts( + &fluree, + ledger, + &data.turtle_txns, + &index_config, + ) + .await + }) + }, + |ledger| { + rt.block_on(async { + black_box( + run_turtle_upserts( + &fluree, + ledger, + &data.turtle_txns, + &index_config, + ) + .await, + ) + }) + }, + criterion::BatchSize::PerIteration, + ); + }, + ); + + // --- TriG insert: default graph + trailing block --- + group.bench_with_input( + BenchmarkId::new("trig_mixed", ¶m_label), + &data, + |b, data| { + b.iter(|| { + let alias = next_ledger_alias("trig-mixed"); + rt.block_on(async { + let ledger = fluree.create_ledger(&alias).await.unwrap(); + black_box( + run_turtle_inserts( + &fluree, + ledger, + &data.trig_mixed_txns, + &index_config, + ) + .await, + ) + }) + }); + }, + ); + + // --- TriG insert: every statement in its own block --- + group.bench_with_input( + BenchmarkId::new("trig_blocks", ¶m_label), + &data, + |b, data| { + b.iter(|| { + let alias = next_ledger_alias("trig-blocks"); + rt.block_on(async { + let ledger = fluree.create_ledger(&alias).await.unwrap(); + black_box( + run_turtle_inserts( + &fluree, + ledger, + &data.trig_blocks_txns, + &index_config, + ) + .await, + ) + }) + }); + }, + ); + // Collect a single timed run for the summary table. let jsonld_ms = time_jsonld_run(&rt, &fluree, &data.jsonld_txns, &index_config); let turtle_ms = time_turtle_run(&rt, &fluree, &data.turtle_txns, &index_config); From 45994cb8bb4a5cb73dba355228a7bff3d1237114 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 02:36:36 -0400 Subject: [PATCH 17/36] feat(transact): locate a TriG document's graph blocks from tokens The locator finds each graph block's label, as written, and its extent by reading tokens only. It never interprets a term, so it cannot disagree with the parser about what the text means: a brace or the word "graph" inside a literal is not a block. It blanks the block syntax in place, so every segment it returns is plain Turtle at the document's own byte offsets, and a parse error inside a block can report where it is in the document. It names the construct it refuses (a nested block, a directive inside a block, a blank-node label, an unclosed block, a stray `}`) as a Turtle parse error at the offending token. `has_graph_blocks` exposes it for callers that must tell TriG from Turtle before choosing a lane. --- fluree-db-transact/src/error.rs | 5 + fluree-db-transact/src/lib.rs | 1 + fluree-db-transact/src/parse/mod.rs | 2 + .../src/parse/rdf_text/locate.rs | 347 ++++++++++++++++++ fluree-db-transact/src/parse/rdf_text/mod.rs | 10 + 5 files changed, 365 insertions(+) create mode 100644 fluree-db-transact/src/parse/rdf_text/locate.rs create mode 100644 fluree-db-transact/src/parse/rdf_text/mod.rs diff --git a/fluree-db-transact/src/error.rs b/fluree-db-transact/src/error.rs index b82de949fb..333ee22d04 100644 --- a/fluree-db-transact/src/error.rs +++ b/fluree-db-transact/src/error.rs @@ -37,6 +37,11 @@ pub enum TransactError { #[error("Parse error: {0}")] Parse(String), + /// RDF text (Turtle, TriG) that does not parse. Positions are byte + /// offsets into the document as the caller sent it. + #[error("Turtle parse error: {0}")] + Turtle(#[from] fluree_graph_turtle::TurtleError), + /// A syntactically valid transaction feature is not yet implemented. /// /// Distinct from [`Self::Parse`] (which is user-error) and diff --git a/fluree-db-transact/src/lib.rs b/fluree-db-transact/src/lib.rs index 3de24e67b9..4d170ac58e 100644 --- a/fluree-db-transact/src/lib.rs +++ b/fluree-db-transact/src/lib.rs @@ -79,6 +79,7 @@ pub use namespace::{ stable_blank_node_sid, stable_blank_node_sid_from_label, NamespaceRegistry, SharedNamespaceAllocator, BLANK_NODE_ID_PREFIX, BLANK_NODE_PREFIX, }; +pub use parse::has_graph_blocks; pub use parse::{ might_contain_graph_block, parse_graph_insert, parse_sync_transaction, parse_transaction, parse_trig_phase1, resolve_trig_meta, unwrap_trig_graph_blocks, NamedGraphBlock, RawObject, diff --git a/fluree-db-transact/src/parse/mod.rs b/fluree-db-transact/src/parse/mod.rs index e469fdd73a..26ab13aca7 100644 --- a/fluree-db-transact/src/parse/mod.rs +++ b/fluree-db-transact/src/parse/mod.rs @@ -6,6 +6,7 @@ pub mod edge_annotations; pub mod jsonld; pub mod nquads; +pub mod rdf_text; pub mod trig_meta; pub mod txn_meta; @@ -49,6 +50,7 @@ pub(crate) const CLAUSE_KEYS: &[&str] = &["where", "delete", "insert", "upsert", pub use jsonld::{parse_graph_insert, parse_sync_transaction, parse_transaction}; pub use nquads::nquads_to_trig; +pub use rdf_text::has_graph_blocks; pub use trig_meta::{ extract_trig_txn_meta, might_contain_graph_block, parse_trig_phase1, resolve_trig_meta, unwrap_trig_graph_blocks, NamedGraphBlock, RawObject, RawTerm, RawTrigMeta, RawTriple, diff --git a/fluree-db-transact/src/parse/rdf_text/locate.rs b/fluree-db-transact/src/parse/rdf_text/locate.rs new file mode 100644 index 0000000000..72c0bafdb8 --- /dev/null +++ b/fluree-db-transact/src/parse/rdf_text/locate.rs @@ -0,0 +1,347 @@ +//! The TriG locator: where a document's graph blocks are, read from tokens. +//! +//! It never interprets a term, so it cannot disagree with the parser about +//! what the text means. It finds each block's label (as written) and extent, +//! blanks the block syntax so every segment is plain Turtle at its original +//! byte offsets, and names the construct it refuses: a nested block, a +//! directive inside a block, a blank-node label, an unclosed block, a stray +//! `}`. + +use crate::error::{Result, TransactError}; +use crate::parse::trig_meta::might_contain_graph_block; +use fluree_graph_turtle::{tokenize, Token, TokenKind, TurtleError}; +use std::borrow::Cow; +use std::ops::Range; + +/// A stretch of the document the parser reads in one call. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) enum SegmentKind { + /// Default-graph statements and directives, as written. + Default, + /// `{ … }`: an anonymous block, which TriG puts in the default graph. + DefaultBlock, + /// A labeled block. `label` is the label as written (``, `ex:g`), + /// resolved by the driver; `at` is its byte offset. + Named { label: String, at: usize }, +} + +#[derive(Clone, Debug)] +pub(super) struct Segment { + pub(super) kind: SegmentKind, + pub(super) range: Range, +} + +/// A located document. +#[derive(Debug)] +pub(super) struct Located<'a> { + /// The document with each block's header (`GRAPH {`) blanked and its + /// `}` blanked, or turned into the `.` a block's last statement may + /// omit. Every edit keeps the length, so byte offsets are the + /// document's. + pub(super) text: Cow<'a, str>, + pub(super) segments: Vec, +} + +fn locate_error(position: usize, message: impl Into) -> TransactError { + TransactError::Turtle(TurtleError::parse(position, message)) +} + +/// Nesting delta of a token inside a statement. +fn nesting(kind: &TokenKind) -> i32 { + match kind { + TokenKind::LBracket + | TokenKind::LParen + | TokenKind::ReifiedTripleStart + | TokenKind::TripleTermStart + | TokenKind::AnnotationOpen => 1, + TokenKind::RBracket + | TokenKind::RParen + | TokenKind::ReifiedTripleEnd + | TokenKind::TripleTermEnd + | TokenKind::AnnotationClose => -1, + _ => 0, + } +} + +fn is_label(kind: &TokenKind) -> bool { + matches!( + kind, + TokenKind::Iri + | TokenKind::IriEscaped(_) + | TokenKind::PrefixedName + | TokenKind::PrefixedNameNs + ) +} + +fn is_directive(kind: &TokenKind) -> bool { + matches!( + kind, + TokenKind::KwPrefix + | TokenKind::KwBase + | TokenKind::KwSparqlPrefix + | TokenKind::KwSparqlBase + | TokenKind::KwVersion + | TokenKind::KwSparqlVersion + ) +} + +/// Find a TriG document's segments by reading tokens only. +pub(super) fn locate(input: &str) -> Result> { + let tokens = tokenize(input)?; + let kind_at = |i: usize| tokens.get(i).map(|t: &Token| &t.kind); + let mut segments = Vec::new(); + let mut edits: Vec<(Range, Option)> = Vec::new(); + let mut default_start = 0usize; + let mut i = 0usize; + + while i < tokens.len() { + let tok = &tokens[i]; + let start = tok.start as usize; + match &tok.kind { + TokenKind::Eof => break, + TokenKind::KwSparqlPrefix => { + // `PREFIX p: ` has no terminating dot. + i += 3; + } + TokenKind::KwSparqlBase | TokenKind::KwSparqlVersion => { + i += 2; + } + TokenKind::RBrace => { + return Err(locate_error(start, "'}' with no graph block to close")); + } + TokenKind::BlankNodeLabel | TokenKind::Anon + if matches!(kind_at(i + 1), Some(TokenKind::LBrace)) => + { + return Err(locate_error( + start, + "blank-node graph label: a graph block needs an IRI label, e.g. ` { ... }`", + )); + } + kind => { + // A block: `GRAPH label {`, `label {`, or `{`. + let label_at = match kind { + TokenKind::KwGraph => { + let Some(next) = kind_at(i + 1) else { + return Err(locate_error(start, "expected a graph label after GRAPH")); + }; + if matches!(next, TokenKind::BlankNodeLabel | TokenKind::Anon) { + return Err(locate_error( + tokens[i + 1].start as usize, + "blank-node graph label: a graph block needs an IRI label, e.g. \ + `GRAPH { ... }`", + )); + } + if !is_label(next) || !matches!(kind_at(i + 2), Some(TokenKind::LBrace)) { + return Err(locate_error(start, "expected `GRAPH { ... }`")); + } + Some(i + 1) + } + k if is_label(k) && matches!(kind_at(i + 1), Some(TokenKind::LBrace)) => { + Some(i) + } + TokenKind::LBrace => None, + _ => { + // A statement: skip to its terminating dot. + i = skip_statement(&tokens, i); + continue; + } + }; + let brace = match label_at { + Some(l) => l + 1, + None => i, + }; + if start > default_start { + segments.push(Segment { + kind: SegmentKind::Default, + range: default_start..start, + }); + } + let content_start = tokens[brace].end as usize; + let (close, needs_dot, next) = scan_block(&tokens, brace + 1)?; + edits.push((start..content_start, None)); + edits.push((close..close + 1, needs_dot.then_some(b'.'))); + let kind = match label_at { + Some(l) => SegmentKind::Named { + label: input[tokens[l].start as usize..tokens[l].end as usize].to_string(), + at: tokens[l].start as usize, + }, + None => SegmentKind::DefaultBlock, + }; + segments.push(Segment { + kind, + range: content_start..close + 1, + }); + default_start = close + 1; + i = next; + } + } + } + if default_start < input.len() { + segments.push(Segment { + kind: SegmentKind::Default, + range: default_start..input.len(), + }); + } + + let text = if edits.is_empty() { + Cow::Borrowed(input) + } else { + let mut bytes = input.as_bytes().to_vec(); + for (range, replacement) in edits { + match replacement { + Some(b) => bytes[range.start] = b, + None => { + for b in &mut bytes[range] { + if !matches!(*b, b'\n' | b'\r') { + *b = b' '; + } + } + } + } + } + // Only ASCII bytes were replaced, by ASCII bytes. + Cow::Owned(String::from_utf8(bytes).expect("ASCII-for-ASCII edits keep UTF-8")) + }; + Ok(Located { text, segments }) +} + +/// Skip a top-level statement starting at `i`; returns the index after its +/// terminating dot (or of the next graph-block brace, where the parser will +/// report the missing dot). +fn skip_statement(tokens: &[Token], mut i: usize) -> usize { + let mut depth = 0i32; + while i < tokens.len() { + match &tokens[i].kind { + TokenKind::Eof => return i, + TokenKind::Dot if depth == 0 => return i + 1, + TokenKind::LBrace | TokenKind::RBrace if depth == 0 => return i, + kind => depth = (depth + nesting(kind)).max(0), + } + i += 1; + } + i +} + +/// Scan a block's contents from `i` (just past its `{`). Returns the byte +/// offset of its `}`, whether its last statement omits the terminating dot, +/// and the token index after the `}`. +fn scan_block(tokens: &[Token], mut i: usize) -> Result<(usize, bool, usize)> { + let mut depth = 0i32; + let mut last: Option<&TokenKind> = None; + while i < tokens.len() { + let tok = &tokens[i]; + match &tok.kind { + TokenKind::Eof => break, + TokenKind::RBrace if depth == 0 => { + let needs_dot = last.is_some_and(|k| !matches!(k, TokenKind::Dot)); + return Ok((tok.start as usize, needs_dot, i + 1)); + } + TokenKind::LBrace | TokenKind::KwGraph => { + return Err(locate_error( + tok.start as usize, + "nested graph block: TriG graph blocks cannot contain graph blocks", + )); + } + kind if is_directive(kind) => { + return Err(locate_error( + tok.start as usize, + "a prefix, base or version directive cannot appear inside a graph block", + )); + } + kind => depth = (depth + nesting(kind)).max(0), + } + last = Some(&tok.kind); + i += 1; + } + let at = tokens.last().map_or(0, |t| t.end as usize); + Err(locate_error(at, "unclosed graph block: expected '}'")) +} + +/// Whether `text` holds at least one graph block, read by the locator: a +/// token pass, no parse. For callers that must tell TriG from Turtle before +/// choosing a lane. +pub fn has_graph_blocks(text: &str) -> Result { + if !might_contain_graph_block(text) { + return Ok(false); + } + Ok(locate(text)? + .segments + .iter() + .any(|s| !matches!(s.kind, SegmentKind::Default))) +} + +#[cfg(test)] +mod tests { + use super::*; + + const EX: &str = "@prefix ex: .\n"; + + fn kinds(text: &str) -> Vec { + locate(text) + .unwrap() + .segments + .into_iter() + .map(|s| s.kind) + .collect() + } + + #[test] + fn the_locator_finds_blocks_in_every_spelling() { + let doc = format!( + "{EX}ex:a ex:p 1 .\nGRAPH {{ ex:b ex:p 2 }}\nex:g2 {{ ex:c ex:p 3 . }}\n\ + {{ ex:d ex:p 4 }}\nex:e ex:p \"has {{ a brace\" .\n" + ); + assert_eq!( + kinds(&doc), + vec![ + SegmentKind::Default, + SegmentKind::Named { + label: "".to_string(), + at: doc.find("").unwrap() + }, + SegmentKind::Default, + SegmentKind::Named { + label: "ex:g2".to_string(), + at: doc.find("ex:g2").unwrap() + }, + SegmentKind::Default, + SegmentKind::DefaultBlock, + SegmentKind::Default, + ] + ); + // A brace or the word "graph" inside a literal is not a block. + assert_eq!( + kinds(&format!("{EX}ex:a ex:p \"{{ graph }}\" .\n")), + vec![SegmentKind::Default] + ); + } + + #[test] + fn the_locator_names_the_construct_it_refuses() { + let refused = [ + ( + "GRAPH { GRAPH { } }", + "nested graph block", + ), + (" { { } }", "nested graph block"), + ("_:b { }", "blank-node graph label"), + ("GRAPH [] { }", "blank-node graph label"), + ( + "GRAPH { @prefix ex: . }", + "directive", + ), + ( + "GRAPH { 1 .", + "unclosed graph block", + ), + ( + "GRAPH { } 1 . }", + "no graph block to close", + ), + ]; + for (doc, needle) in refused { + let e = locate(doc).expect_err("should be refused").to_string(); + assert!(e.contains(needle), "{doc}: {e}"); + } + } +} diff --git a/fluree-db-transact/src/parse/rdf_text/mod.rs b/fluree-db-transact/src/parse/rdf_text/mod.rs new file mode 100644 index 0000000000..93f3fb2c2e --- /dev/null +++ b/fluree-db-transact/src/parse/rdf_text/mod.rs @@ -0,0 +1,10 @@ +//! RDF text, located once and parsed once. +//! +//! A TriG document is Turtle statements interleaved with graph blocks. The +//! locator (the `locate` module) reads tokens only: it finds each block's +//! label and extent and never interprets a term, so it cannot disagree with +//! the parser about what the text means. + +mod locate; + +pub use locate::has_graph_blocks; From 1ba743666b249959232ed02bfc870dc31a726aa7 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 02:37:48 -0400 Subject: [PATCH 18/36] feat(transact): parse RDF text once into transaction templates `parse_rdf_text` reads Turtle and TriG with the conformant parser into one `TemplateSink`, for every verb that writes RDF text. Plain Turtle takes a single parse. A TriG document is parsed segment by segment in document order from the locator's segments, each seeded with the prefix and base declarations made before it: a redefinition applies to what follows it and to nothing before it. A block label resolves through the parser, under the declarations in force where the block appears; parse errors report the document's byte offset. The sink converts literals exactly as `FlakeSink` does (lenient: an ill-typed lexical form is kept with its declared datatype), keeps list positions, takes blank-node and literal `rdf:type` objects like any other predicate's, and applies the reserved-predicate firewall. Blank-node labels are document-scoped across default statements and blocks; anonymous nodes get fresh `-b{N}` labels. RDF 1.2 annotations become the `f:reifies*` bundle through `bundle_templates`, which states no `f:reifiesGraph`: the graph scope that places a template in a named graph adds the anchor, here through a local stand-in with the contract of the graph-scope emitter. `<#txn-meta>` (with no base in force) becomes commit metadata with the refusals the TriG txn-meta parser applies. `content_id` is a commutative sum of per-statement 128-bit hashes, so it ignores statement order; `parse_rdf_text_txn` derives the upsert blank-node skolem scope from it. `Placement::Into` homes every statement in one graph for graph-scoped writes and refuses a block for another graph or a mix of blocks and default-graph statements, naming the graph. --- Cargo.lock | 1 + fluree-db-transact/Cargo.toml | 3 + fluree-db-transact/src/generate/flakes.rs | 52 +- fluree-db-transact/src/generate/mod.rs | 1 + fluree-db-transact/src/lib.rs | 6 +- fluree-db-transact/src/parse/mod.rs | 4 +- fluree-db-transact/src/parse/rdf_text/mod.rs | 782 ++++++++++++++++++- fluree-db-transact/src/parse/trig_meta.rs | 2 +- fluree-db-transact/src/template_sink.rs | 621 +++++++++++++++ 9 files changed, 1465 insertions(+), 7 deletions(-) create mode 100644 fluree-db-transact/src/template_sink.rs diff --git a/Cargo.lock b/Cargo.lock index 801761fd70..1f6faa6cec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3924,6 +3924,7 @@ dependencies = [ "tokio", "tracing", "ulid", + "xxhash-rust", "zstd", ] diff --git a/fluree-db-transact/Cargo.toml b/fluree-db-transact/Cargo.toml index 01afced237..df55450485 100644 --- a/fluree-db-transact/Cargo.toml +++ b/fluree-db-transact/Cargo.toml @@ -59,6 +59,9 @@ fluree-graph-turtle = { path = "../fluree-graph-turtle" } # Hash map for trig_meta parser + FxHashSet for worker cache rustc-hash = "2" +# RDF text content identity (upsert blank-node scope) +xxhash-rust = { workspace = true, features = ["xxh3"] } + # Thread-safe namespace allocator for parallel import parking_lot = "0.12" diff --git a/fluree-db-transact/src/generate/flakes.rs b/fluree-db-transact/src/generate/flakes.rs index 2af9b0118f..c0b598e7ed 100644 --- a/fluree-db-transact/src/generate/flakes.rs +++ b/fluree-db-transact/src/generate/flakes.rs @@ -579,9 +579,9 @@ pub(crate) fn validate_value_dt_pair(val: &FlakeValue, dt: &Sid) -> Result<()> { /// `@annotation` at the flake level; cascade retracts cancel either /// surface) cannot drift between those callers — each keeps only its own /// error channel and emission (Vec-extend vs commit-writer/spool). -/// Transactional TriG (`convert_named_graphs_to_templates` in -/// `fluree-db-api`) is a second builder emitting templates; its parity is -/// pinned by `trig_star_in_graph_block_matches_jsonld_named_graph_annotation`. +/// RDF text staged as templates (upsert, sync, the TriG insert fallback) +/// builds the same bundle with [`bundle_templates`]; the parity is pinned by +/// `trig_star_in_graph_block_matches_jsonld_named_graph_annotation`. /// /// `g` is `None` for the Turtle sinks (default graph) and the block's graph /// for TriG import; list-occurrence annotations are deferred in v1 → @@ -616,6 +616,52 @@ pub(crate) fn reified_triple_bundle( Ok(key.to_reifies_facts_jsonld_compatible(ann, t, true)) } +/// The `f:reifies*` bundle of a reifier attachment, as templates: `ann` +/// reifies `(s, p, o)`, where `o_dtc` is the object's datatype constraint +/// (`None` for a node object). +/// +/// The shape is the one the JSON-LD `@annotation` lowering writes and +/// [`reified_triple_bundle`] builds as flakes: `f:reifiesSubject`, +/// `f:reifiesPredicate`, `f:reifiesObject` (carrying the object's datatype +/// and language tag), and `f:reifiesLang` for a language-tagged object; no +/// `f:reifiesDatatype`. The `f:reifiesGraph` anchor is deliberately not +/// here: the scope that places the templates in a named graph adds it, so +/// no template builder can state an anchor that disagrees with the graph +/// the bundle lands in. +pub(crate) fn bundle_templates( + s: TemplateTerm, + p: TemplateTerm, + o: TemplateTerm, + o_dtc: Option, + ann: TemplateTerm, +) -> Vec { + use fluree_db_core::namespaces::{ + reifies_lang_sid, reifies_object_sid, reifies_predicate_sid, reifies_subject_sid, + }; + let lang = match &o_dtc { + Some(fluree_db_core::DatatypeConstraint::LangTag(lang)) => Some(lang.to_string()), + _ => None, + }; + let slot = |pred: &Sid, obj: TemplateTerm| { + TripleTemplate::new(ann.clone(), TemplateTerm::Sid(pred.clone()), obj) + }; + let mut out = Vec::with_capacity(4); + out.push(slot(reifies_subject_sid(), s)); + out.push(slot(reifies_predicate_sid(), p)); + if let Some(lang) = lang { + out.push(slot( + reifies_lang_sid(), + TemplateTerm::Value(FlakeValue::String(lang)), + )); + } + let mut object = slot(reifies_object_sid(), o); + if let Some(dtc) = o_dtc { + object = object.with_dtc(dtc); + } + out.push(object); + out +} + /// Infer datatype from a FlakeValue /// /// Returns the appropriate XSD/RDF datatype Sid for the given value. diff --git a/fluree-db-transact/src/generate/mod.rs b/fluree-db-transact/src/generate/mod.rs index 1dd86d6d21..1bb8dc6fd5 100644 --- a/fluree-db-transact/src/generate/mod.rs +++ b/fluree-db-transact/src/generate/mod.rs @@ -9,6 +9,7 @@ pub mod accumulator; pub mod flakes; pub use accumulator::FlakeAccumulator; +pub(crate) use flakes::bundle_templates; #[cfg(feature = "import")] pub(crate) use flakes::DT_ID; pub use flakes::{infer_datatype, FlakeGenerator}; diff --git a/fluree-db-transact/src/lib.rs b/fluree-db-transact/src/lib.rs index 4d170ac58e..fe7d202ca7 100644 --- a/fluree-db-transact/src/lib.rs +++ b/fluree-db-transact/src/lib.rs @@ -48,6 +48,7 @@ pub mod parse; pub mod raw_txn_upload; pub mod stage; pub mod staged_dicts; +pub mod template_sink; pub mod value_convert; #[cfg(feature = "import")] @@ -79,7 +80,9 @@ pub use namespace::{ stable_blank_node_sid, stable_blank_node_sid_from_label, NamespaceRegistry, SharedNamespaceAllocator, BLANK_NODE_ID_PREFIX, BLANK_NODE_PREFIX, }; -pub use parse::has_graph_blocks; +pub use parse::{ + has_graph_blocks, parse_rdf_text, parse_rdf_text_txn, Placement, RdfText, RdfTextSummary, +}; pub use parse::{ might_contain_graph_block, parse_graph_insert, parse_sync_transaction, parse_transaction, parse_trig_phase1, resolve_trig_meta, unwrap_trig_graph_blocks, NamedGraphBlock, RawObject, @@ -93,6 +96,7 @@ pub use stage::{ pub use staged_dicts::{ attach_binary_provider, attach_staged_dicts, detach_binary_provider, staged_dicts, StagedDicts, }; +pub use template_sink::{RdfTextParts, TemplateSink}; #[cfg(feature = "shacl")] pub use stage::{validate_view_with_shacl, ShaclGraphPolicy, ShaclValidationOutcome}; diff --git a/fluree-db-transact/src/parse/mod.rs b/fluree-db-transact/src/parse/mod.rs index 26ab13aca7..1b34e99498 100644 --- a/fluree-db-transact/src/parse/mod.rs +++ b/fluree-db-transact/src/parse/mod.rs @@ -50,7 +50,9 @@ pub(crate) const CLAUSE_KEYS: &[&str] = &["where", "delete", "insert", "upsert", pub use jsonld::{parse_graph_insert, parse_sync_transaction, parse_transaction}; pub use nquads::nquads_to_trig; -pub use rdf_text::has_graph_blocks; +pub use rdf_text::{ + has_graph_blocks, parse_rdf_text, parse_rdf_text_txn, Placement, RdfText, RdfTextSummary, +}; pub use trig_meta::{ extract_trig_txn_meta, might_contain_graph_block, parse_trig_phase1, resolve_trig_meta, unwrap_trig_graph_blocks, NamedGraphBlock, RawObject, RawTerm, RawTrigMeta, RawTriple, diff --git a/fluree-db-transact/src/parse/rdf_text/mod.rs b/fluree-db-transact/src/parse/rdf_text/mod.rs index 93f3fb2c2e..6b377f6aac 100644 --- a/fluree-db-transact/src/parse/rdf_text/mod.rs +++ b/fluree-db-transact/src/parse/rdf_text/mod.rs @@ -1,10 +1,790 @@ //! RDF text, located once and parsed once. //! +//! Every write verb that takes RDF text — upsert, the TriG insert fallback, +//! graph sync and graph insert — reads it here, with the conformant Turtle +//! parser, into one [`TemplateSink`]. Nothing re-encodes the text as +//! JSON-LD, reorders it, or reads block contents with a second parser. +//! //! A TriG document is Turtle statements interleaved with graph blocks. The //! locator (the `locate` module) reads tokens only: it finds each block's //! label and extent and never interprets a term, so it cannot disagree with -//! the parser about what the text means. +//! the parser about what the text means. The driver then parses each segment +//! in document order, carrying the prefix and base declarations forward from +//! where they appear: a redefinition applies to what follows it and to +//! nothing before it. Block labels are resolved by the parser too, under the +//! declarations in force where the block appears. One sink serves the whole +//! document, so a blank node label means one node everywhere in it. +//! +//! Plain Turtle — no `{` and no `GRAPH` anywhere — takes a single parse. + +use crate::error::{Result, TransactError}; +use crate::ir::{GraphSel, Txn, TxnOpts, TxnType}; +use crate::namespace::NamespaceRegistry; +use crate::parse::trig_meta::{might_contain_graph_block, TXN_META_GRAPH_IRI}; +use crate::template_sink::{RdfTextParts, Scope, TemplateSink}; +use fluree_graph_ir::{Datatype, GraphSink, LiteralValue, SinkResult, TermId}; +use fluree_graph_turtle::{ParserOptions, TurtleError}; +use std::sync::Arc; mod locate; pub use locate::has_graph_blocks; +use locate::{locate, SegmentKind}; + +/// Where a document's statements land. +#[derive(Clone, Copy, Debug)] +pub enum Placement<'a> { + /// As written: default-graph statements in the default graph, each + /// block's statements in the block's graph. + AsWritten, + /// Every statement in one graph (`None`: the default graph), for a write + /// scoped to a graph the request names. A block must name that graph, + /// and a document holds the graph's statements either in blocks or as + /// default-graph statements, not both. + Into(Option<&'a str>), +} + +/// A document parsed for one transaction: the templates and graphs it +/// writes, its `<#txn-meta>` entries, and what it held. +#[derive(Debug)] +pub struct RdfText { + pub parts: RdfTextParts, + /// Data statements in the document (not counting `<#txn-meta>`). + pub statements: usize, +} + +/// What [`parse_rdf_text_txn`] read, beside the transaction it built. +#[derive(Clone, Copy, Debug)] +pub struct RdfTextSummary { + /// See [`RdfTextParts::content_id`]. + pub content_id: u128, + /// Data statements in the document (not counting `<#txn-meta>`). + pub statements: usize, +} + +/// Parse `text` into a [`Txn`] of `txn_type`. +/// +/// Upsert gets a blank-node skolem scope derived from the document's +/// content (see [`TemplateSink`]'s content identity) unless `opts` already +/// names one, so the same document upserted again addresses the same blank +/// nodes. A graph-scoped [`Placement::Into`] write registers its target +/// graph; a sync additionally gets `sync_graph`. +pub fn parse_rdf_text_txn( + text: &str, + txn_type: TxnType, + placement: Placement<'_>, + sync: bool, + mut opts: TxnOpts, + ns: &mut NamespaceRegistry, +) -> Result<(Txn, RdfTextSummary)> { + let RdfText { parts, statements } = parse_rdf_text(text, placement, ns)?; + let summary = RdfTextSummary { + content_id: parts.content_id, + statements, + }; + if txn_type == TxnType::Upsert && opts.skolem_txn_id.is_none() { + let id = parts.content_id; + let folded = (id as u64) ^ ((id >> 64) as u64); + opts.skolem_txn_id = Some(format!( + "upsert{}", + fluree_db_core::skolem::doc_scope(folded) + )); + } + let mut txn = match txn_type { + TxnType::Upsert => Txn::upsert(), + TxnType::Insert => Txn::insert(), + TxnType::Update => { + return Err(TransactError::Parse( + "RDF text cannot express an update; use SPARQL UPDATE or JSON-LD".to_string(), + )) + } + } + .with_opts(opts); + txn.insert_templates = parts.templates; + txn.write_graphs = parts.write_graphs; + txn.txn_meta = parts.txn_meta; + if sync { + txn.sync_graph = Some(match placement { + Placement::Into(Some(iri)) => GraphSel::Graph(iri.to_string()), + Placement::Into(None) | Placement::AsWritten => GraphSel::Default, + }); + } + Ok((txn, summary)) +} + +/// Parse `text` into templates under `placement`. +pub fn parse_rdf_text( + text: &str, + placement: Placement<'_>, + ns: &mut NamespaceRegistry, +) -> Result { + let mut sink = TemplateSink::new(ns); + let target_scope = |sink_target: Option<&str>| match sink_target { + Some(iri) => Scope::Named(Arc::from(iri)), + None => Scope::Default, + }; + let default_scope = match placement { + Placement::AsWritten => Scope::Default, + Placement::Into(target) => target_scope(target), + }; + + if !might_contain_graph_block(text) { + sink.set_scope(default_scope); + fluree_graph_turtle::parse(text, &mut sink)?; + let statements = sink.statements(); + return Ok(RdfText { + parts: sink.into_parts()?, + statements, + }); + } + + let located = locate(text)?; + let mut prefixes: Vec<(String, String)> = Vec::new(); + let mut base: Option = None; + let mut default_statements = 0usize; + let mut named_blocks = 0usize; + for segment in &located.segments { + let before = sink.statements(); + match &segment.kind { + SegmentKind::Default | SegmentKind::DefaultBlock => { + sink.set_scope(default_scope.clone()); + } + SegmentKind::Named { label, at } => { + let iri = resolve_label(label, *at, &prefixes, base.as_deref())?; + if iri == TXN_META_GRAPH_IRI { + sink.set_scope(Scope::TxnMeta); + } else { + named_blocks += 1; + match placement { + Placement::AsWritten => sink.set_scope(Scope::Named(Arc::from(iri))), + Placement::Into(target) if target == Some(iri.as_str()) => { + sink.set_scope(default_scope.clone()); + } + Placement::Into(target) => { + return Err(TransactError::Parse(format!( + "the request targets one graph, {}; the body also has a \ + GRAPH block for <{iri}>", + describe_target(target) + ))); + } + } + } + } + } + let slice = &located.text[segment.range.clone()]; + let seeded_prefixes = prefixes.clone(); + let seeded_base = base.clone(); + let mut capture = Capture { + sink: &mut sink, + prefixes: &mut prefixes, + base: &mut base, + }; + fluree_graph_turtle::parse_with_prefixes_base_options( + slice, + &mut capture, + &seeded_prefixes, + seeded_base.as_deref(), + ParserOptions::default(), + ) + .map_err(|e| shift(e, segment.range.start))?; + if matches!( + segment.kind, + SegmentKind::Default | SegmentKind::DefaultBlock + ) { + default_statements += sink.statements() - before; + } + } + if let Placement::Into(target) = placement { + if default_statements > 0 && named_blocks > 0 { + let target = describe_target(target); + return Err(TransactError::Parse(format!( + "a TriG body holds {target}'s triples either in GRAPH {target} blocks or as \ + default-graph triples, not both" + ))); + } + } + let statements = sink.statements(); + Ok(RdfText { + parts: sink.into_parts()?, + statements, + }) +} + +fn describe_target(target: Option<&str>) -> String { + match target { + Some(iri) => format!("<{iri}>"), + None => "the default graph".to_string(), + } +} + +/// Move a parser error's byte position from a segment into the document. +fn shift(e: TurtleError, offset: usize) -> TransactError { + TransactError::Turtle(match e { + TurtleError::Parse { position, message } => TurtleError::Parse { + position: position + offset, + message, + }, + TurtleError::Lexer { position, message } => TurtleError::Lexer { + position: position + offset, + message, + }, + other => other, + }) +} + +/// Resolve a block label with the parser, under the declarations in force +/// where the block appears: exactly the expansion a subject IRI gets there +/// (prefixed-name unescaping, relative references against the base). +/// +/// `<#txn-meta>` with no base in force is the txn-meta block, as it always +/// was; under a base it resolves like any relative reference. +fn resolve_label( + label: &str, + at: usize, + prefixes: &[(String, String)], + base: Option<&str>, +) -> Result { + if base.is_none() && label == "<#txn-meta>" { + return Ok(TXN_META_GRAPH_IRI.to_string()); + } + let probe = format!("{label} ."); + let mut sink = LabelProbe(None); + fluree_graph_turtle::parse_with_prefixes_base_options( + &probe, + &mut sink, + prefixes, + base, + ParserOptions::default(), + ) + .map_err(|e| { + let message = match e { + TurtleError::Parse { message, .. } | TurtleError::Lexer { message, .. } => message, + other => other.to_string(), + }; + TransactError::Turtle(TurtleError::Parse { + position: at, + message: format!("graph label {label}: {message}"), + }) + })?; + sink.0.ok_or_else(|| { + TransactError::Turtle(TurtleError::parse( + at, + format!("graph label {label} is not an IRI"), + )) + }) +} + +/// Records the first IRI a probe statement names: the label. +struct LabelProbe(Option); + +impl GraphSink for LabelProbe { + fn on_base(&mut self, _: &str) {} + fn on_prefix(&mut self, _: &str, _: &str) {} + fn term_iri(&mut self, iri: &str) -> TermId { + if self.0.is_none() { + self.0 = Some(iri.to_string()); + } + TermId::new(0) + } + fn term_blank(&mut self, _: Option<&str>) -> TermId { + TermId::new(0) + } + fn term_literal(&mut self, _: &str, _: Datatype, _: Option<&str>) -> TermId { + TermId::new(0) + } + fn term_literal_value(&mut self, _: LiteralValue, _: Datatype) -> TermId { + TermId::new(0) + } + fn emit_triple(&mut self, _: TermId, _: TermId, _: TermId) -> SinkResult { + Ok(()) + } +} + +/// Forwards every event to the sink and records the prefix and base +/// declarations a segment makes, so the next segment is seeded with them. +struct Capture<'s, 'n> { + sink: &'s mut TemplateSink<'n>, + prefixes: &'s mut Vec<(String, String)>, + base: &'s mut Option, +} + +impl GraphSink for Capture<'_, '_> { + fn on_base(&mut self, base_iri: &str) { + *self.base = Some(base_iri.to_string()); + self.sink.on_base(base_iri); + } + fn on_prefix(&mut self, prefix: &str, namespace_iri: &str) { + match self.prefixes.iter_mut().find(|(p, _)| p == prefix) { + Some(entry) => entry.1 = namespace_iri.to_string(), + None => self + .prefixes + .push((prefix.to_string(), namespace_iri.to_string())), + } + self.sink.on_prefix(prefix, namespace_iri); + } + fn term_iri(&mut self, iri: &str) -> TermId { + self.sink.term_iri(iri) + } + fn term_blank(&mut self, label: Option<&str>) -> TermId { + self.sink.term_blank(label) + } + fn term_literal(&mut self, value: &str, datatype: Datatype, language: Option<&str>) -> TermId { + self.sink.term_literal(value, datatype, language) + } + fn term_literal_value(&mut self, value: LiteralValue, datatype: Datatype) -> TermId { + self.sink.term_literal_value(value, datatype) + } + fn emit_triple(&mut self, subject: TermId, predicate: TermId, object: TermId) -> SinkResult { + self.sink.emit_triple(subject, predicate, object) + } + fn emit_list_item( + &mut self, + subject: TermId, + predicate: TermId, + object: TermId, + index: i32, + ) -> SinkResult { + self.sink.emit_list_item(subject, predicate, object, index) + } + fn supports_quads(&self) -> bool { + self.sink.supports_quads() + } + fn emit_quad( + &mut self, + subject: TermId, + predicate: TermId, + object: TermId, + graph: TermId, + ) -> SinkResult { + self.sink.emit_quad(subject, predicate, object, graph) + } + fn end_statement(&mut self) { + self.sink.end_statement(); + } + fn abort_statement(&mut self) { + self.sink.abort_statement(); + } + fn supports_reified_triples(&self) -> bool { + self.sink.supports_reified_triples() + } + fn emit_reified_triple( + &mut self, + subject: TermId, + predicate: TermId, + object: TermId, + reifier: TermId, + ) -> SinkResult { + self.sink + .emit_reified_triple(subject, predicate, object, reifier) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::{TemplateGraph, TemplateTerm, TripleTemplate}; + use fluree_db_core::{DatatypeConstraint, FlakeValue, Sid}; + use fluree_db_novelty::TxnMetaValue; + + const EX: &str = "@prefix ex: .\n"; + const XSD_STRING: &str = "^^"; + + fn iri(ns: &NamespaceRegistry, sid: &Sid) -> String { + format!( + "{}{}", + ns.get_prefix(sid.namespace_code).unwrap_or("?"), + sid.name + ) + } + + fn term(ns: &NamespaceRegistry, t: &TemplateTerm) -> String { + match t { + TemplateTerm::Sid(sid) => format!("<{}>", iri(ns, sid)), + TemplateTerm::BlankNode(label) => format!("_:{label}"), + TemplateTerm::Value(FlakeValue::String(s)) => format!("{s:?}"), + TemplateTerm::Value(v) => format!("{v:?}"), + TemplateTerm::Var(v) => format!("?{v:?}"), + } + } + + /// `[graph] s p o[@lang|^^dt][#i]` + fn render(ns: &NamespaceRegistry, t: &TripleTemplate) -> String { + let graph = match &t.graph { + TemplateGraph::Default => String::new(), + TemplateGraph::Iri(g) => format!("[{g}] "), + TemplateGraph::Var(v) => format!("[?{v:?}] "), + }; + let dtc = match &t.dtc { + None => String::new(), + Some(DatatypeConstraint::LangTag(l)) => format!("@{l}"), + Some(DatatypeConstraint::Explicit(dt)) => format!("^^<{}>", iri(ns, dt)), + }; + let i = t.list_index.map(|i| format!("#{i}")).unwrap_or_default(); + format!( + "{graph}{} {} {}{dtc}{i}", + term(ns, &t.subject), + term(ns, &t.predicate), + term(ns, &t.object) + ) + } + + fn parse_ok(text: &str, placement: Placement<'_>) -> (RdfText, NamespaceRegistry) { + let mut ns = NamespaceRegistry::new(); + let parsed = parse_rdf_text(text, placement, &mut ns).unwrap_or_else(|e| panic!("{e}")); + (parsed, ns) + } + + fn rendered(text: &str) -> Vec { + let (parsed, ns) = parse_ok(text, Placement::AsWritten); + let mut out: Vec = parsed + .parts + .templates + .iter() + .map(|t| render(&ns, t)) + .collect(); + out.sort(); + out + } + + fn err(text: &str, placement: Placement<'_>) -> String { + let mut ns = NamespaceRegistry::new(); + parse_rdf_text(text, placement, &mut ns) + .expect_err("should be refused") + .to_string() + } + + #[test] + fn a_parse_error_in_a_block_reports_the_document_offset() { + let doc = format!( + "{EX}GRAPH {{ ex:a ex:p 1 }}\nGRAPH {{ ex:b ex:p ; }}\n" + ); + let e = err(&doc, Placement::AsWritten); + let semicolon = doc.rfind(';').unwrap(); + let position: usize = e + .split("position ") + .nth(1) + .and_then(|rest| rest.split(|c: char| !c.is_ascii_digit()).next()) + .and_then(|n| n.parse().ok()) + .unwrap_or_else(|| panic!("no position in {e}")); + let second_block = doc.find("GRAPH ").unwrap(); + assert!( + position >= second_block && position <= semicolon + 2, + "the offset points into the second block of the document: {e}" + ); + } + + // ---- Directives in document order (U1) ----------------------------- + + #[test] + fn a_redefined_prefix_or_base_applies_only_after_it() { + let doc = "@prefix ex: .\n\ + ex:x ex:p \"mentions the graph word\" .\n\ + GRAPH { ex:y ex:p \"1\" . }\n\ + @prefix ex: .\n\ + ex:z ex:p \"2\" .\n\ + GRAPH { ex:w ex:p \"3\" . }\n"; + assert_eq!( + rendered(doc), + vec![ + format!( + " \"mentions the graph word\"{XSD_STRING}" + ), + format!(" \"2\"{XSD_STRING}"), + format!("[http://g/1] \"1\"{XSD_STRING}"), + format!("[http://g/2] \"3\"{XSD_STRING}"), + ] + ); + let doc = "@base .\n

\"1\" .\nGRAPH {

\"2\" . }\n\ + @base .\n

\"3\" .\nGRAPH {

\"4\" . }\n"; + assert_eq!( + rendered(doc), + vec![ + format!(" \"1\"{XSD_STRING}"), + format!(" \"3\"{XSD_STRING}"), + format!("[http://a.org/g1] \"2\"{XSD_STRING}"), + format!("[http://b.org/g2] \"4\"{XSD_STRING}"), + ] + ); + } + + #[test] + fn relative_references_need_a_base() { + let e = err( + "GRAPH { 1 . }", + Placement::AsWritten, + ); + assert!(e.contains("relative"), "{e}"); + let e = err( + "GRAPH { 1 . }", + Placement::AsWritten, + ); + assert!(e.contains("graph label"), "{e}"); + } + + // ---- Block contents: the full grammar ------------------------------- + + #[test] + fn blocks_accept_the_full_turtle_grammar_and_match_the_default_graph() { + let body = "ex:s ex:p [ ex:q \"x\" ] ; ex:list ( \"c\" \"a\" \"b\" \"a\" ) ; \ + ex:nested ( ( 1 ) ) ; ex:empty () ; ex:anon [] ; a _:k , \"lit\" .\n\ + ex:s ex:r ex:o ~ ex:reifier {| ex:source \"wiki\" |} .\n"; + let default = rendered(&format!("{EX}{body}")); + let block = rendered(&format!("{EX}GRAPH {{\n{body}}}\n")); + let mut stripped: Vec = block + .iter() + .filter(|t| !t.contains("reifiesGraph")) + .map(|t| t.trim_start_matches("[http://g] ").to_string()) + .collect(); + stripped.sort(); + assert_eq!( + stripped, default, + "block contents parse exactly as default-graph Turtle" + ); + assert!( + block + .iter() + .any(|t| t.contains("reifiesGraph") && t.ends_with("")), + "a named-graph bundle carries its anchor: {block:?}" + ); + assert!( + !default.iter().any(|t| t.contains("reifiesGraph")), + "a default-graph bundle has none" + ); + // Collections keep order and duplicates. + for (i, v) in ["c", "a", "b", "a"].iter().enumerate() { + let needle = format!("\"{v}\"{XSD_STRING}#{i}"); + assert!( + default + .iter() + .any(|t| t.contains("/list>") && t.ends_with(&needle)), + "{needle}: {default:?}" + ); + } + // `rdf:type` takes a blank node and a literal like any predicate. + let rdf_type = ""; + assert!(default + .iter() + .any(|t| t.contains(rdf_type) && t.ends_with("_:k"))); + assert!(default + .iter() + .any(|t| t.contains(rdf_type) && t.contains("\"lit\""))); + } + + #[test] + fn blank_node_labels_are_document_scoped_and_anonymous_nodes_are_fresh() { + let (parsed, ns) = parse_ok( + &format!( + "{EX}_:b ex:p 1 .\nGRAPH {{ _:b ex:q 2 . [] ex:r 3 . }}\n\ + GRAPH {{ _:b ex:q 4 . [] ex:r 5 . }}\n" + ), + Placement::AsWritten, + ); + let subjects: Vec = parsed + .parts + .templates + .iter() + .map(|t| term(&ns, &t.subject)) + .collect(); + assert_eq!( + subjects.iter().filter(|s| *s == "_:b").count(), + 3, + "one label, one node, across the default graph and both blocks: {subjects:?}" + ); + let anon: Vec<&String> = subjects.iter().filter(|s| s.starts_with("_:-b")).collect(); + assert_eq!(anon.len(), 2); + assert_ne!(anon[0], anon[1], "`[]` in two blocks is two nodes"); + } + + #[test] + fn literals_convert_as_insert_converts_them() { + // An ill-typed lexical form is kept, with its declared datatype, as + // `FlakeSink` keeps it: one conversion on every RDF-text lane. + let got = rendered(&format!( + "{EX}@prefix xsd: .\n\ + ex:s ex:n \"abc\"^^xsd:integer ; ex:m \"12\"^^xsd:integer .\n" + )); + assert!( + got.iter() + .any(|t| t.contains("\"abc\"^^")), + "{got:?}" + ); + assert!(got.iter().any(|t| t.contains("Long(12)")), "{got:?}"); + } + + // ---- txn-meta ------------------------------------------------------ + + fn meta(doc: &str) -> Vec<(String, TxnMetaValue)> { + let (parsed, _) = parse_ok(doc, Placement::AsWritten); + parsed + .parts + .txn_meta + .iter() + .map(|e| (e.predicate_name.clone(), e.value.clone())) + .collect() + } + + const META_PREFIXES: &str = "@prefix ex: .\n\ + @prefix fluree: .\n\ + @prefix xsd: .\n"; + + #[test] + fn txn_meta_blocks_become_commit_metadata() { + let doc = format!( + "{META_PREFIXES}ex:alice ex:name \"Alice\" .\n\ + GRAPH <#txn-meta> {{\n\ + fluree:commit:this ex:machine \"server-01\" ; ex:batchId 42 ;\n\ + ex:validated true ; ex:ratio 1.5e0 ; ex:author ex:alice ;\n\ + ex:description \"Mise a jour\"@fr ;\n\ + ex:timestamp \"2025-01-15T10:30:00Z\"^^xsd:dateTime ;\n\ + ex:tags \"a\", \"b\" .\n\ + }}\n" + ); + let entries = meta(&doc); + let find = |p: &str| -> Vec<&TxnMetaValue> { + entries + .iter() + .filter(|(n, _)| n == p) + .map(|(_, v)| v) + .collect() + }; + assert!(matches!(find("machine")[..], [TxnMetaValue::String(s)] if s == "server-01")); + assert!(matches!(find("batchId")[..], [TxnMetaValue::Long(42)])); + assert!(matches!( + find("validated")[..], + [TxnMetaValue::Boolean(true)] + )); + assert!(matches!(find("ratio")[..], [TxnMetaValue::Double(d)] if (*d - 1.5).abs() < 1e-9)); + assert!(matches!(find("author")[..], [TxnMetaValue::Ref { name, .. }] if name == "alice")); + assert!(matches!(find("description")[..], + [TxnMetaValue::LangString { value, lang }] if value == "Mise a jour" && lang == "fr")); + assert!(matches!(find("timestamp")[..], + [TxnMetaValue::TypedLiteral { value, dt_name, .. }] + if value == "2025-01-15T10:30:00Z" && dt_name == "dateTime")); + assert_eq!(find("tags").len(), 2); + // The default-graph triple is data, not metadata. + let (parsed, _) = parse_ok(&doc, Placement::AsWritten); + assert_eq!(parsed.statements, 1); + // The scheme spelling, a full IRI subject, the compact block form and + // SPARQL-style prefixes all read the same. + for doc in [ + "PREFIX ex: \n<#txn-meta> { ex:k \"v\" }\n", + "@prefix ex: .\n\ + GRAPH <#txn-meta> { ex:k \"v\" . }\n", + ] { + assert!( + matches!(&meta(doc)[..], [(k, TxnMetaValue::String(v))] if k == "k" && v == "v"), + "{doc}" + ); + } + } + + #[test] + fn txn_meta_refuses_what_commit_metadata_cannot_hold() { + for (body, needle) in [ + ("ex:alice ex:machine \"server-01\" .", "fluree:commit:this"), + ("_:b1 ex:machine \"server-01\" .", "blank nodes not allowed"), + ( + "fluree:commit:this ex:source _:b1 .", + "blank nodes not allowed", + ), + ( + "fluree:commit:this ex:source [ ex:p 1 ] .", + "blank nodes not allowed", + ), + ( + "fluree:commit:this ex:items ( 1 2 ) .", + "lists are not allowed", + ), + ("fluree:commit:this ex:p ex:o ~ ex:r .", "reifiers"), + ] { + let doc = format!("{META_PREFIXES}GRAPH <#txn-meta> {{ {body} }}\n"); + let e = err(&doc, Placement::AsWritten); + assert!(e.contains(needle), "{body}: {e}"); + } + // Under a base, `<#txn-meta>` resolves like any relative reference. + let (parsed, _) = parse_ok( + "@base .\nGRAPH <#txn-meta> { 1 . }\n", + Placement::AsWritten, + ); + assert!(parsed.parts.txn_meta.is_empty()); + assert!(parsed + .parts + .write_graphs + .contains("http://b.org/doc#txn-meta")); + } + + // ---- Content identity ------------------------------------------------ + + fn content_id(doc: &str) -> u128 { + parse_ok(doc, Placement::AsWritten).0.parts.content_id + } + + #[test] + fn content_identity_ignores_statement_order_but_not_content() { + let a = format!( + "{EX}ex:s ex:p [ ex:q \"x\" ] .\nex:t ex:p \"y\" , \"z\" .\nex:l ex:p ( 1 2 ) .\n" + ); + let reordered = format!( + "{EX}ex:l ex:p ( 1 2 ) .\nex:t ex:p \"z\" , \"y\" .\nex:s ex:p [ ex:q \"x\" ] .\n" + ); + assert_eq!(content_id(&a), content_id(&reordered)); + let edited = a.replace("\"y\"", "\"y2\""); + assert_ne!(content_id(&a), content_id(&edited)); + let list_order = a.replace("( 1 2 )", "( 2 1 )"); + assert_ne!( + content_id(&a), + content_id(&list_order), + "list order is content" + ); + let meta_only = format!( + "{a}@prefix fluree: .\n\ + GRAPH <#txn-meta> {{ fluree:commit:this ex:at \"t1\" }}\n" + ); + assert_eq!( + content_id(&a), + content_id(&meta_only), + "commit metadata does not change the data's identity" + ); + } + + // ---- Placement --------------------------------------------------------- + + #[test] + fn a_graph_scoped_write_homes_every_statement_and_checks_the_blocks() { + let target = "http://g/t"; + let in_target = + |t: &TripleTemplate| matches!(&t.graph, TemplateGraph::Iri(g) if &**g == target); + let (parsed, ns) = parse_ok( + &format!("{EX}GRAPH <{target}> {{ ex:a ex:p 1 ~ ex:r . }}\n"), + Placement::Into(Some(target)), + ); + assert!(parsed.parts.templates.iter().all(in_target)); + assert!(parsed + .parts + .templates + .iter() + .any(|t| render(&ns, t).contains("reifiesGraph"))); + let (parsed, _) = parse_ok( + &format!("{EX}ex:a ex:p 1 .\n"), + Placement::Into(Some(target)), + ); + assert!(parsed.parts.templates.iter().all(in_target)); + assert!(parsed.parts.write_graphs.contains(target)); + + let e = err( + &format!("{EX}GRAPH {{ ex:a ex:p 1 }}\n"), + Placement::Into(Some(target)), + ); + assert!(e.contains(""), "{e}"); + let e = err( + &format!("{EX}ex:b ex:p 2 .\nGRAPH <{target}> {{ ex:a ex:p 1 }}\n"), + Placement::Into(Some(target)), + ); + assert!(e.contains("not both"), "{e}"); + let e = err( + &format!("{EX}GRAPH <{target}> {{ ex:a ex:p 1 }}\n"), + Placement::Into(None), + ); + assert!(e.contains("the default graph"), "{e}"); + } +} diff --git a/fluree-db-transact/src/parse/trig_meta.rs b/fluree-db-transact/src/parse/trig_meta.rs index 2caf10dd02..1aa48e24e6 100644 --- a/fluree-db-transact/src/parse/trig_meta.rs +++ b/fluree-db-transact/src/parse/trig_meta.rs @@ -1881,7 +1881,7 @@ fn split_prefixed_name(span: &str) -> (&str, &str) { } /// Validate txn-meta limits. -fn validate_limits(entries: &[TxnMetaEntry]) -> Result<()> { +pub(crate) fn validate_limits(entries: &[TxnMetaEntry]) -> Result<()> { if entries.len() > MAX_TXN_META_ENTRIES { return Err(TransactError::Parse(format!( "txn-meta entry count {} exceeds maximum {}", diff --git a/fluree-db-transact/src/template_sink.rs b/fluree-db-transact/src/template_sink.rs new file mode 100644 index 0000000000..e518e82bb8 --- /dev/null +++ b/fluree-db-transact/src/template_sink.rs @@ -0,0 +1,621 @@ +//! TemplateSink — a `GraphSink` that turns parsed RDF text into transaction +//! templates. +//! +//! RDF text (Turtle, TriG, N-Triples) is interpreted once, by +//! `fluree-graph-turtle`'s parser, and every verb that writes it — upsert, +//! the TriG insert fallback, graph sync, graph insert — stages from this +//! sink's templates. It replaces a round trip through JSON-LD that lost +//! list order, blank-node `rdf:type` objects and literal `rdf:type` values, +//! rejected IRIs the JSON-LD compact-IRI guard mistook for prefixed names, +//! and parsed literals more strictly than insert did. +//! +//! Literals convert exactly as [`FlakeSink`](crate::flake_sink::FlakeSink) +//! converts them (`value_convert`, lenient: an ill-typed lexical form is +//! stored as a string with its declared datatype), so a document means the +//! same thing on every RDF-text lane. +//! +//! The driver (`parse::rdf_text`) feeds one sink across a TriG document's +//! segments, setting the graph scope per segment, so labeled blank nodes are +//! document-scoped and anonymous ones are numbered across the whole +//! document. + +use crate::error::TransactError; +use crate::generate::{bundle_templates, infer_datatype, validate_value_dt_pair}; +use crate::ir::{TemplateTerm, TripleTemplate}; +use crate::namespace::{NamespaceRegistry, NsAllocator}; +use crate::value_convert::{convert_native_literal, convert_string_literal}; +use fluree_db_core::{DatatypeConstraint, FlakeValue, Sid}; +use fluree_db_novelty::{TxnMetaEntry, TxnMetaValue}; +use fluree_graph_ir::{Datatype, GraphSink, LiteralValue, SinkResult, TermId}; +use std::collections::{BTreeSet, HashMap}; +use std::sync::Arc; +use xxhash_rust::xxh3::Xxh3; + +/// The graph the statements being parsed belong to. +#[derive(Clone, Debug)] +pub(crate) enum Scope { + /// The default graph. + Default, + /// A named graph, by IRI. + Named(Arc), + /// The `<#txn-meta>` block: commit metadata, not graph data. + TxnMeta, +} + +/// A parsed term. +enum SinkTerm { + /// An IRI or blank node. + Node { + term: TemplateTerm, + /// The IRI, for IRIs (a quad's graph, a txn-meta subject check). + iri: Option>, + hash: u128, + }, + /// A literal, converted for storage. + Literal { + value: FlakeValue, + dtc: DatatypeConstraint, + hash: u128, + /// The txn-meta form of the literal, when parsed in the txn-meta + /// scope. + meta: Option>, + }, +} + +/// What a parsed RDF text document stages. +#[derive(Debug)] +pub struct RdfTextParts { + /// One template per statement, graph set, plus the `f:reifies*` + /// bundle templates of any RDF 1.2 annotations. + pub templates: Vec, + /// Every named graph the document writes to (its labeled blocks). + pub write_graphs: BTreeSet, + /// The `<#txn-meta>` block's entries. + pub txn_meta: Vec, + /// Order-insensitive hash of the document's data statements (see + /// [`TemplateSink`]). + pub content_id: u128, +} + +/// A `GraphSink` that builds [`TripleTemplate`]s. +/// +/// # Content identity +/// +/// `content_id` is the wrapping sum of a 128-bit hash of every data +/// statement — graph, subject, predicate, object (lexical form, datatype +/// and language for literals; IRI or label for nodes) and list position — +/// and of every reifier attachment. A sum is commutative, so the id does +/// not depend on statement order, and it is computed in one pass with O(1) +/// memory. Upsert derives its blank-node skolem scope from it, so the same +/// document upserted again addresses the same blank nodes. `<#txn-meta>` +/// entries are commit metadata and do not contribute. +pub struct TemplateSink<'a> { + ns: &'a mut NamespaceRegistry, + terms: Vec, + /// Labeled blank nodes, document-scoped. + blank_labels: HashMap, + /// Anonymous blank nodes minted so far (`-b{N}`, as `FlakeSink`). + anon: u32, + scope: Scope, + templates: Vec, + write_graphs: BTreeSet, + txn_meta: Vec, + content_id: u128, + /// Data statements accepted so far (not `<#txn-meta>` entries, not + /// annotation bundles). + statements: usize, + /// First error; surfaced by [`Self::into_parts`] so a bad document fails + /// the whole transaction rather than committing part of it. + error: Option, +} + +impl<'a> TemplateSink<'a> { + pub fn new(ns: &'a mut NamespaceRegistry) -> Self { + Self { + ns, + terms: Vec::new(), + blank_labels: HashMap::new(), + anon: 0, + scope: Scope::Default, + templates: Vec::new(), + write_graphs: BTreeSet::new(), + txn_meta: Vec::new(), + content_id: 0, + statements: 0, + error: None, + } + } + + /// Set the graph the next statements belong to. A named graph is + /// written even when its block holds no statements. + pub(crate) fn set_scope(&mut self, scope: Scope) { + if let Scope::Named(iri) = &scope { + self.write_graphs.insert(iri.to_string()); + } + self.scope = scope; + } + + /// Data statements accepted so far. + pub(crate) fn statements(&self) -> usize { + self.statements + } + + /// The templates, graphs, txn-meta and content id, or the first error + /// the document raised. + pub fn into_parts(self) -> Result { + if let Some(e) = self.error { + return Err(e); + } + crate::parse::trig_meta::validate_limits(&self.txn_meta)?; + Ok(RdfTextParts { + templates: self.templates, + write_graphs: self.write_graphs, + txn_meta: self.txn_meta, + content_id: self.content_id, + }) + } + + fn fail(&mut self, message: impl Into) { + if self.error.is_none() { + self.error = Some(TransactError::Parse(message.into())); + } + } + + fn add(&mut self, term: SinkTerm) -> TermId { + let id = TermId::new(self.terms.len() as u32); + self.terms.push(term); + id + } + + fn term(&self, id: TermId) -> &SinkTerm { + &self.terms[id.index() as usize] + } + + fn hash(&self, id: TermId) -> u128 { + match self.term(id) { + SinkTerm::Node { hash, .. } | SinkTerm::Literal { hash, .. } => *hash, + } + } + + /// The node at `id`, or `None` for a literal. + fn node(&self, id: TermId) -> Option { + match self.term(id) { + SinkTerm::Node { term, .. } => Some(term.clone()), + SinkTerm::Literal { .. } => None, + } + } + + /// An object: a node, or a literal value with its datatype constraint. + fn object(&self, id: TermId) -> (TemplateTerm, Option) { + match self.term(id) { + SinkTerm::Node { term, .. } => (term.clone(), None), + SinkTerm::Literal { value, dtc, .. } => { + (TemplateTerm::Value(value.clone()), Some(dtc.clone())) + } + } + } + + /// The graph's contribution to a statement hash. + fn graph_hash(&self) -> u128 { + match &self.scope { + Scope::Named(iri) => hash_parts(b"G", &[iri.as_bytes()]), + Scope::Default | Scope::TxnMeta => 0, + } + } + + fn record(&mut self, parts: &[&[u8]]) { + self.content_id = self.content_id.wrapping_add(hash_parts(b"S", parts)); + } + + /// Emit `template` in the current scope: the stand-in for `GraphScope::emit`. + fn emit(&mut self, template: TripleTemplate) { + match &self.scope { + Scope::Default => self.templates.push(template), + Scope::Named(iri) => { + let iri = Arc::clone(iri); + graph_scope_emit(&iri, template, self.ns, &mut self.templates); + } + Scope::TxnMeta => unreachable!("txn-meta statements are not templates"), + } + } + + fn push_statement(&mut self, s: TermId, p: TermId, o: TermId, list_index: Option) { + if matches!(self.scope, Scope::TxnMeta) { + if list_index.is_some() { + self.fail("lists are not allowed in the txn-meta graph"); + return; + } + self.push_txn_meta(s, p, o); + return; + } + let Some(subject) = self.node(s) else { + self.fail("a literal cannot be a subject"); + return; + }; + let predicate = match self.node(p) { + Some(TemplateTerm::Sid(p)) => p, + _ => { + self.fail("a predicate must be an IRI"); + return; + } + }; + // Reserved-predicate firewall, as `FlakeSink` and the JSON-LD and + // SPARQL UPDATE surfaces: a hand-written `f:reifies*` statement is + // refused; annotations are minted only through the RDF 1.2 + // annotation syntax, which arrives via `emit_reified_triple`. + if fluree_db_core::is_reserved_reifies_predicate(&predicate) { + let iri = format!( + "{}{}", + self.ns.get_prefix(predicate.namespace_code).unwrap_or(""), + predicate.name + ); + let e = TransactError::UnsupportedFeature(format!( + "'{iri}' is a system-controlled predicate; use the RDF 1.2 annotation \ + syntax (`~ {{| ... |}}` or `<< s p o >>`) instead of \ + writing f:reifies* triples by hand" + )); + if self.error.is_none() { + self.error = Some(e); + } + return; + } + let (object, dtc) = self.object(o); + if let (TemplateTerm::Value(v), Some(dtc)) = (&object, &dtc) { + if let Err(e) = validate_value_dt_pair(v, dtc.datatype()) { + if self.error.is_none() { + self.error = Some(e); + } + return; + } + } + let (gh, sh, ph, oh) = (self.graph_hash(), self.hash(s), self.hash(p), self.hash(o)); + let index = list_index.map_or([0u8; 5], |i| { + let b = i.to_le_bytes(); + [1, b[0], b[1], b[2], b[3]] + }); + self.record(&[ + &gh.to_le_bytes(), + &sh.to_le_bytes(), + &ph.to_le_bytes(), + &oh.to_le_bytes(), + &index, + ]); + + let mut template = TripleTemplate::new(subject, TemplateTerm::Sid(predicate), object); + if let Some(dtc) = dtc { + template = template.with_dtc(dtc); + } + if let Some(i) = list_index { + template = template.with_list_index(i); + } + self.statements += 1; + self.emit(template); + } + + fn push_txn_meta(&mut self, s: TermId, p: TermId, o: TermId) { + match self.term(s) { + SinkTerm::Node { iri: Some(iri), .. } if is_commit_this_iri(iri) => {} + SinkTerm::Node { iri: Some(iri), .. } => { + let iri = iri.clone(); + self.fail(format!( + "txn-meta subject must be fluree:commit:this, found: {iri}" + )); + return; + } + SinkTerm::Node { iri: None, .. } => { + self.fail("blank nodes not allowed as txn-meta subject"); + return; + } + SinkTerm::Literal { .. } => { + self.fail("a literal cannot be a subject"); + return; + } + } + let predicate = match self.node(p) { + Some(TemplateTerm::Sid(p)) => p, + _ => { + self.fail("a txn-meta predicate must be an IRI"); + return; + } + }; + let value = match self.term(o) { + SinkTerm::Node { + term: TemplateTerm::Sid(sid), + iri: Some(_), + .. + } => Ok(TxnMetaValue::Ref { + ns: sid.namespace_code, + name: sid.name.to_string(), + }), + SinkTerm::Node { .. } => Err("blank nodes not allowed in txn-meta objects".to_string()), + SinkTerm::Literal { meta, .. } => meta + .clone() + .unwrap_or_else(|| Err("unsupported txn-meta literal".to_string())), + }; + match value { + Ok(value) => self.txn_meta.push(TxnMetaEntry::new( + predicate.namespace_code, + predicate.name.to_string(), + value, + )), + Err(message) => self.fail(message), + } + } + + /// The txn-meta form of a literal parsed in the txn-meta scope. + fn meta_literal(&mut self, lexical: &str, dt_iri: &str, lang: Option<&str>) -> TxnMetaValue { + if let Some(lang) = lang { + return TxnMetaValue::LangString { + value: lexical.to_string(), + lang: lang.to_string(), + }; + } + if dt_iri == fluree_vocab::xsd::STRING { + return TxnMetaValue::String(lexical.to_string()); + } + let dt = self.ns.sid_for_iri(dt_iri); + TxnMetaValue::TypedLiteral { + value: lexical.to_string(), + dt_ns: dt.namespace_code, + dt_name: dt.name.to_string(), + } + } +} + +/// The one rule that puts a template in its graph and, when it writes +/// `f:reifiesSubject` in a named graph, adds the bundle's `f:reifiesGraph` +/// anchor, so a template bundle never states its own anchor. It stands in +/// for `GraphScope::emit`, which is to own this rule for every template +/// source; when that lands, the sink routes through it and this goes. +fn graph_scope_emit( + iri: &Arc, + template: TripleTemplate, + ns: &mut NamespaceRegistry, + out: &mut Vec, +) { + let anchors = matches!( + &template.predicate, + TemplateTerm::Sid(p) if fluree_db_core::is_reifies_subject(p) + ); + let template = template.in_graph(Arc::clone(iri)); + if anchors { + let anchor = TripleTemplate::new( + template.subject.clone(), + TemplateTerm::Sid(fluree_db_core::namespaces::reifies_graph_sid().clone()), + TemplateTerm::Sid(ns.sid_for_iri(iri)), + ) + .in_graph(Arc::clone(iri)); + out.push(anchor); + } + out.push(template); +} + +fn is_commit_this_iri(iri: &str) -> bool { + iri == fluree_vocab::fluree::COMMIT_THIS_HTTP || iri == fluree_vocab::fluree::COMMIT_THIS_SCHEME +} + +/// A 128-bit hash of a tagged sequence of parts. Stable across builds and +/// platforms: it hashes bytes, never a `Debug` rendering or a `Hash` impl. +fn hash_parts(tag: &[u8], parts: &[&[u8]]) -> u128 { + let mut h = Xxh3::new(); + h.update(tag); + for part in parts { + h.update(&(part.len() as u64).to_le_bytes()); + h.update(part); + } + h.digest128() +} + +impl GraphSink for TemplateSink<'_> { + fn on_base(&mut self, _base_iri: &str) { + // The parser resolves relative IRIs before calling term_iri. + } + + fn on_prefix(&mut self, _prefix: &str, namespace_iri: &str) { + // Pre-register the namespace, as FlakeSink does, so codes are + // allocated in declaration order. + self.ns.get_or_allocate(namespace_iri); + } + + fn term_iri(&mut self, iri: &str) -> TermId { + let sid = self.ns.sid_for_iri(iri); + let hash = hash_parts(b"I", &[iri.as_bytes()]); + self.add(SinkTerm::Node { + term: TemplateTerm::Sid(sid), + iri: Some(Arc::from(iri)), + hash, + }) + } + + fn term_blank(&mut self, label: Option<&str>) -> TermId { + // A blank node in the txn-meta block fails where it is used, with a + // message naming the position (subject or object). + match label { + Some(l) => { + if let Some(&id) = self.blank_labels.get(l) { + return id; + } + // Stable Fluree blank-node ids (`fdb-…`) address the stored + // node; other labels skolemize at staging. + let term = crate::namespace::stable_blank_node_sid_from_label(l) + .map_or_else(|| TemplateTerm::BlankNode(l.to_string()), TemplateTerm::Sid); + let hash = hash_parts(b"B", &[l.as_bytes()]); + let id = self.add(SinkTerm::Node { + term, + iri: None, + hash, + }); + self.blank_labels.insert(l.to_string(), id); + id + } + None => { + // The leading '-' keeps minted labels out of the + // user-writable BLANK_NODE_LABEL space (see FlakeSink). + self.anon += 1; + let label = format!("-b{}", self.anon); + let hash = hash_parts(b"B", &[label.as_bytes()]); + self.add(SinkTerm::Node { + term: TemplateTerm::BlankNode(label), + iri: None, + hash, + }) + } + } + } + + fn term_literal(&mut self, value: &str, datatype: Datatype, language: Option<&str>) -> TermId { + let dt_iri = datatype.as_iri(); + let (flake_value, dt_sid) = + convert_string_literal(value, dt_iri, &mut NsAllocator::Exclusive(self.ns)); + let dtc = match language { + Some(lang) => DatatypeConstraint::LangTag(Arc::from(lang)), + None => DatatypeConstraint::Explicit(dt_sid), + }; + let hash = hash_parts( + b"L", + &[ + value.as_bytes(), + dt_iri.as_bytes(), + language.unwrap_or("").as_bytes(), + ], + ); + let meta = matches!(self.scope, Scope::TxnMeta) + .then(|| Ok(self.meta_literal(value, dt_iri, language))); + self.add(SinkTerm::Literal { + value: flake_value, + dtc, + hash, + meta, + }) + } + + fn term_literal_value(&mut self, value: LiteralValue, datatype: Datatype) -> TermId { + let flake_value = convert_native_literal(&value); + let dt_sid = if datatype.is_json() { + Sid::new(fluree_vocab::namespaces::RDF, "JSON") + } else { + infer_datatype(&flake_value) + }; + let (kind, text): (&[u8], String) = match &value { + LiteralValue::Integer(i) => (b"i", i.to_string()), + LiteralValue::Double(d) => (b"d", format!("{:016x}", d.to_bits())), + LiteralValue::Boolean(b) => (b"b", b.to_string()), + LiteralValue::String(s) => (b"s", s.to_string()), + LiteralValue::Json(s) => (b"j", s.to_string()), + }; + let hash = hash_parts(b"N", &[kind, text.as_bytes(), datatype.as_iri().as_bytes()]); + let meta = matches!(self.scope, Scope::TxnMeta).then(|| match &value { + LiteralValue::Integer(i) => Ok(TxnMetaValue::Long(*i)), + LiteralValue::Double(d) if d.is_finite() => Ok(TxnMetaValue::Double(*d)), + LiteralValue::Double(_) => { + Err("txn-meta does not support non-finite double values".to_string()) + } + LiteralValue::Boolean(b) => Ok(TxnMetaValue::Boolean(*b)), + LiteralValue::String(s) => Ok(TxnMetaValue::String(s.to_string())), + LiteralValue::Json(s) => Ok(TxnMetaValue::TypedLiteral { + value: s.to_string(), + dt_ns: dt_sid.namespace_code, + dt_name: dt_sid.name.to_string(), + }), + }); + self.add(SinkTerm::Literal { + value: flake_value, + dtc: DatatypeConstraint::Explicit(dt_sid), + hash, + meta, + }) + } + + fn emit_triple(&mut self, subject: TermId, predicate: TermId, object: TermId) -> SinkResult { + self.push_statement(subject, predicate, object, None); + Ok(()) + } + + fn emit_list_item( + &mut self, + subject: TermId, + predicate: TermId, + object: TermId, + index: i32, + ) -> SinkResult { + self.push_statement(subject, predicate, object, Some(index)); + Ok(()) + } + + fn supports_quads(&self) -> bool { + true + } + + /// A quad: the triple in the named graph `graph`, for a parser that reads + /// TriG itself. + fn emit_quad( + &mut self, + subject: TermId, + predicate: TermId, + object: TermId, + graph: TermId, + ) -> SinkResult { + let iri = match self.term(graph) { + SinkTerm::Node { iri: Some(iri), .. } => Arc::clone(iri), + _ => { + self.fail("a graph name must be an IRI"); + return Ok(()); + } + }; + let outer = std::mem::replace(&mut self.scope, Scope::Default); + self.set_scope(Scope::Named(iri)); + self.push_statement(subject, predicate, object, None); + self.scope = outer; + Ok(()) + } + + fn supports_reified_triples(&self) -> bool { + true + } + + /// An RDF 1.2 reifier attachment: the `f:reifies*` bundle templates, in + /// the current graph. The parser has already emitted the base triple. + fn emit_reified_triple( + &mut self, + subject: TermId, + predicate: TermId, + object: TermId, + reifier: TermId, + ) -> SinkResult { + if matches!(self.scope, Scope::TxnMeta) { + self.fail( + "RDF 1.2 reifiers and annotations are not allowed in the txn-meta graph; \ + its triples become commit metadata, not graph edges", + ); + return Ok(()); + } + let (Some(s), Some(p), Some(ann)) = + (self.node(subject), self.node(predicate), self.node(reifier)) + else { + self.fail("a reified triple needs node subject, predicate and reifier"); + return Ok(()); + }; + let (o, dtc) = self.object(object); + let (gh, sh, ph, oh, rh) = ( + self.graph_hash(), + self.hash(subject), + self.hash(predicate), + self.hash(object), + self.hash(reifier), + ); + self.content_id = self.content_id.wrapping_add(hash_parts( + b"R", + &[ + &gh.to_le_bytes(), + &sh.to_le_bytes(), + &ph.to_le_bytes(), + &oh.to_le_bytes(), + &rh.to_le_bytes(), + ], + )); + for template in bundle_templates(s, p, o, dtc, ann) { + self.emit(template); + } + Ok(()) + } +} From 159284da55aedea4911add736effa464f8fc6d98 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 03:03:39 -0400 Subject: [PATCH 19/36] fix(api): upsert RDF text from one parse, without a JSON-LD round trip Turtle and TriG upserts were parsed into a graph, re-encoded as JSON-LD and expanded again, and TriG blocks went through a second parser. The round trip refused IRIs whose scheme the compact-IRI guard took for an undefined prefix (`tag:`, `kb:`), stored collections as unordered values, refused an annotated `rdf:type` edge, and parsed literals more strictly than insert. Every upsert entry point (`Fluree::upsert_turtle*`, the owned and cached builders with or without a policy, the graph builder) now stages from `parse_rdf_text_txn`, parsed against the state it is staged on: `OpPlan::Rdf` carries the text through the cached-handle retry loop. The upsert stores what a Turtle insert of the same document stores, blank nodes are scoped to the document's content, the raw transaction is the text, and a parse error is a Turtle parse error at the document's offset. `Fluree::upsert_turtle` now reads TriG as the builders do. Fuel is charged as for any upsert: the baseline plus one unit per staged flake, reported in the tally on every builder lane. --- fluree-db-api/src/tx.rs | 112 ++- fluree-db-api/src/tx_builder.rs | 76 ++- fluree-db-api/tests/grp_transact.rs | 2 + fluree-db-api/tests/it_rdf_text_parity.rs | 638 ++++++++++++++++++ .../tests/it_turtle_star_write_paths.rs | 38 +- 5 files changed, 819 insertions(+), 47 deletions(-) create mode 100644 fluree-db-api/tests/it_rdf_text_parity.rs diff --git a/fluree-db-api/src/tx.rs b/fluree-db-api/src/tx.rs index f84ffa0309..66d17c1f90 100644 --- a/fluree-db-api/src/tx.rs +++ b/fluree-db-api/src/tx.rs @@ -26,8 +26,8 @@ use fluree_db_transact::stage_with_graph_delta as stage_txn; use fluree_db_transact::validate_view_with_shacl; use fluree_db_transact::{ commit as commit_txn, parse_transaction, resolve_trig_meta, CommitOpts, CommitReceipt, - GraphSel, NamedGraphBlock, NamespaceRegistry, RawTrigMeta, StageOptions, TemplateTerm, - TripleTemplate, Txn, TxnOpts, TxnType, + GraphSel, NamedGraphBlock, NamespaceRegistry, Placement, RawTrigMeta, StageOptions, + TemplateTerm, TripleTemplate, Txn, TxnOpts, TxnType, }; use fluree_vocab::config_iris; use rustc_hash::{FxHashMap, FxHashSet}; @@ -342,6 +342,16 @@ impl<'a> TrackedTransactionInput<'a> { } } +/// A document that does not parse is a Turtle parse error (HTTP 400, +/// `TURTLE_PARSE`), whichever part of it failed; anything else the document +/// raised stays the transaction error it is. +fn rdf_text_error(e: fluree_db_transact::TransactError) -> ApiError { + match e { + fluree_db_transact::TransactError::Turtle(e) => ApiError::Turtle(e), + other => other.into(), + } +} + /// Create a tracker for fuel limits only (no time/policy tracking). /// /// This mirrors query behavior: even non-tracked transactions respect max-fuel. @@ -2994,6 +3004,55 @@ impl crate::Fluree { .await } + /// Stage RDF text (Turtle or TriG) as a `txn_type` transaction. + /// + /// The text is parsed once, by the Turtle parser, into templates + /// ([`fluree_db_transact::parse_rdf_text_txn`]); nothing is re-encoded + /// as JSON-LD. Statements land where the document puts them: the default + /// graph's in the default graph, each block's in its graph, and a + /// `<#txn-meta>` block's in the commit metadata. An upsert scopes its + /// blank nodes to the document's content, so the same document upserted + /// again addresses the same nodes. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn stage_rdf_text_tracked( + &self, + ledger: LedgerState, + txn_type: TxnType, + text: &str, + txn_opts: TxnOpts, + index_config: Option<&IndexConfig>, + external_tracker: Option<&Tracker>, + policy: Option<&crate::PolicyContext>, + ) -> Result { + let mut ns_registry = NamespaceRegistry::from_db(&ledger.snapshot); + let (txn, _) = { + let parse_span = + tracing::debug_span!("txn_parse", txn_type = ?txn_type, rdf_bytes = text.len()); + let _guard = parse_span.enter(); + fluree_db_transact::parse_rdf_text_txn( + text, + txn_type, + Placement::AsWritten, + false, + txn_opts, + &mut ns_registry, + ) + .map_err(rdf_text_error)? + }; + // RDF text has no JSON body: no `opts.maxFuel` to read and no + // `@context` to compact violation messages with. + self.stage_built_txn_tracked( + ledger, + txn, + ns_registry, + &JsonValue::Null, + index_config, + external_tracker, + policy, + ) + .await + } + /// Shared staging tail for a fully-built JSON-LD [`Txn`]: uniqueness / /// SHACL / reasoning validation and [`StageResult`] assembly. #[allow(clippy::too_many_arguments)] @@ -4058,16 +4117,18 @@ impl crate::Fluree { .await } - /// Upsert data from Turtle format + /// Upsert data from Turtle or TriG /// - /// Parses the Turtle input and upserts it into the ledger. - /// For each (subject, predicate) pair, existing values are retracted - /// before new values are asserted. + /// Parses the document once and upserts it into the ledger: for each + /// (graph, subject, predicate) it names, the current values are + /// retracted and the document's values asserted. A TriG document's + /// blocks upsert into their graphs, and a `<#txn-meta>` block becomes + /// commit metadata. /// /// # Arguments /// /// * `ledger` - The ledger state (consumed) - /// * `turtle` - Turtle (TTL) format data + /// * `turtle` - Turtle or TriG text /// /// # Example /// @@ -4078,11 +4139,18 @@ impl crate::Fluree { /// "#).await?; /// ``` pub async fn upsert_turtle(&self, ledger: LedgerState, turtle: &str) -> Result { - let data = fluree_graph_turtle::parse_to_json(turtle)?; - self.upsert(ledger, &data).await + let index_config = self.default_index_config(); + self.upsert_turtle_with_opts( + ledger, + turtle, + TxnOpts::default(), + CommitOpts::default(), + &index_config, + ) + .await } - /// Upsert data from Turtle format with options + /// Upsert data from Turtle or TriG with options /// /// Same as `upsert_turtle` but allows custom transaction and commit options. /// Prefer using the builder API: `fluree.transact(ledger).upsert_turtle(ttl).txn_opts(...).execute()`. @@ -4095,8 +4163,28 @@ impl crate::Fluree { commit_opts: CommitOpts, index_config: &IndexConfig, ) -> Result { - let data = fluree_graph_turtle::parse_to_json(turtle)?; - self.upsert_with_opts(ledger, &data, txn_opts, commit_opts, index_config) + let store_raw_txn = txn_opts.store_raw_txn.unwrap_or(false); + + // The raw transaction is the text, as for a Turtle insert. + let commit_opts = if commit_opts.raw_txn_upload.is_none() && store_raw_txn { + let content_store = self.content_store(ledger.ledger_id()); + commit_opts.with_raw_txn_spawned(content_store, JsonValue::String(turtle.to_string())) + } else { + commit_opts + }; + + let staged = self + .stage_rdf_text_tracked( + ledger, + TxnType::Upsert, + turtle, + txn_opts, + Some(index_config), + None, + None, + ) + .await?; + self.commit_stage_result(staged, TxnType::Upsert, commit_opts, index_config) .await } diff --git a/fluree-db-api/src/tx_builder.rs b/fluree-db-api/src/tx_builder.rs index c9ec4cf26d..cb52165842 100644 --- a/fluree-db-api/src/tx_builder.rs +++ b/fluree-db-api/src/tx_builder.rs @@ -33,8 +33,8 @@ use fluree_db_nameservice::NsRecord; use fluree_db_novelty::Novelty; use fluree_db_transact::GraphSel; use fluree_db_transact::{ - lower_sparql_update_request, parse_trig_phase1, CommitOpts, NamedGraphBlock, NamespaceRegistry, - RawTrigMeta, TransactError, Txn, TxnOpts, TxnType, + lower_sparql_update_request, CommitOpts, NamedGraphBlock, NamespaceRegistry, RawTrigMeta, + TransactError, Txn, TxnOpts, TxnType, }; use rustc_hash::FxHashSet; use std::collections::HashMap; @@ -458,19 +458,12 @@ impl TransactOperation<'_> { TransactOperation::Graph(_) => Err(ApiError::internal( "graph operations must be staged through stage_graph_op_tracked", )), - TransactOperation::InsertTurtle(ttl) | TransactOperation::UpsertTurtle(ttl) => { - // Phase 1: Extract TriG GRAPH block (if present) - let phase1 = parse_trig_phase1(ttl)?; - - // Parse cleaned Turtle to JSON - let json = fluree_graph_turtle::parse_to_json(&phase1.turtle)?; - - Ok(ParsedOperation { - json, - trig_meta: phase1.raw_meta, - named_graphs: phase1.named_graphs, - }) - } + // RDF text is parsed once, into templates, by the staging path + // (`stage_turtle_insert_with_opts`, `stage_rdf_text_tracked`); + // reading it here would re-encode it as JSON-LD. + TransactOperation::InsertTurtle(_) | TransactOperation::UpsertTurtle(_) => Err( + ApiError::internal("RDF text must be staged through its own lane, not as JSON-LD"), + ), } } } @@ -1175,6 +1168,13 @@ fn remap_sid(sid: &mut Sid, remap: &HashMap) { enum OpPlan<'a> { InsertTurtle(&'a str), + /// RDF text staged as templates (an upsert), parsed against whichever + /// state it is staged on: parsing allocates namespace codes relative to + /// that state's table. + Rdf { + txn_type: TxnType, + text: &'a str, + }, /// A SPARQL UPDATE request, parsed and lowered against whichever state /// it is staged on — lowering allocates namespace codes relative to /// that state's table. @@ -1195,6 +1195,10 @@ impl<'a> OpPlan<'a> { fn from_op(op: TransactOperation<'a>) -> Result { match op { TransactOperation::InsertTurtle(turtle) => Ok(OpPlan::InsertTurtle(turtle)), + TransactOperation::UpsertTurtle(text) => Ok(OpPlan::Rdf { + txn_type: TxnType::Upsert, + text, + }), TransactOperation::Graph(op) => Ok(OpPlan::Graph(op)), _ => { let txn_type = op.txn_type(); @@ -1472,6 +1476,28 @@ impl Fluree { return Ok((stage_result, TxnType::Insert, commit_opts, None)); } + // RDF text upsert: parsed once into templates. + if let TransactOperation::UpsertTurtle(text) = op { + let commit_opts = self.maybe_spawn_txn_upload( + core.commit_opts, + &ledger_id, + serde_json::Value::String(text.to_string()), + store_raw_txn, + ); + let stage_result = self + .stage_rdf_text_tracked( + ledger_state, + TxnType::Upsert, + text, + core.txn_opts, + Some(index_config), + tracker_ref, + core.policy.as_ref(), + ) + .await?; + return Ok((stage_result, TxnType::Upsert, commit_opts, None)); + } + // Graph ops: dedicated staging (graph scope, sync retraction wave). // Must be dispatched before the JSON-like fallthrough, which would // otherwise stage the payload as a plain default-graph insert. @@ -1577,6 +1603,26 @@ impl Fluree { .await?; Ok((stage_result, TxnType::Insert, commit_opts)) } + OpPlan::Rdf { txn_type, text } => { + let commit_opts = self.maybe_spawn_txn_upload( + commit_opts_base.clone(), + &ledger_id, + serde_json::Value::String((*text).to_string()), + store_raw_txn, + ); + let stage_result = self + .stage_rdf_text_tracked( + ledger_state, + *txn_type, + text, + txn_opts, + Some(index_config), + tracker_ref, + None, + ) + .await?; + Ok((stage_result, *txn_type, commit_opts)) + } OpPlan::JsonLike { txn_type, txn_json, diff --git a/fluree-db-api/tests/grp_transact.rs b/fluree-db-api/tests/grp_transact.rs index d058a9daf2..aa363da4a3 100644 --- a/fluree-db-api/tests/grp_transact.rs +++ b/fluree-db-api/tests/grp_transact.rs @@ -15,6 +15,8 @@ mod it_enforce_unique_upsert_indexed; mod it_jsonld_empty_list; #[path = "it_raw_txn_parallel_upload.rs"] mod it_raw_txn_parallel_upload; +#[path = "it_rdf_text_parity.rs"] +mod it_rdf_text_parity; #[path = "it_reserved_graph_writes.rs"] mod it_reserved_graph_writes; #[path = "it_stable_blank_nodes.rs"] diff --git a/fluree-db-api/tests/it_rdf_text_parity.rs b/fluree-db-api/tests/it_rdf_text_parity.rs new file mode 100644 index 0000000000..037285df42 --- /dev/null +++ b/fluree-db-api/tests/it_rdf_text_parity.rs @@ -0,0 +1,638 @@ +//! RDF text is parsed once, by the Turtle parser, on every write lane. +//! +//! Upsert read Turtle and TriG through a JSON-LD round trip: the text was +//! parsed into a graph, re-encoded as JSON-LD and expanded again. The round +//! trip refused IRIs whose scheme the compact-IRI guard took for an +//! undefined prefix (`tag:`, `kb:`; #1977), stored collections as unordered +//! values, and parsed literals more strictly than insert. Upsert now stages +//! from the parse itself, so it stores what insert stores. + +use crate::support::{genesis_ledger, rebuild_and_publish_index, MemoryFluree}; +use fluree_db_api::policy_builder::build_policy_context_from_opts; +use fluree_db_api::{ + CommitOpts, Fluree, FlureeBuilder, GovernanceOptions, GraphDb, LedgerState, PolicyContext, + TrackingOptions, +}; +use fluree_db_core::comparator::IndexType; +use fluree_db_core::{ + range_with_overlay, ContentStore as _, Flake, FlakeValue, RangeMatch, RangeOptions, RangeTest, + Sid, +}; +use fluree_db_transact::TxnOpts; +use serde_json::{json, Value as JsonValue}; + +const TAG: &str = "tag:example.org,2025:"; +const KB: &str = "kb:n#"; + +fn memory() -> MemoryFluree { + FlureeBuilder::memory().build_memory() +} + +// ============================================================================= +// Reading back what a lane stored +// ============================================================================= + +/// A node as written in the document: its IRI, or `_:label` for a blank +/// node, whatever skolem scope the lane minted it under +/// (`fdb--

\"mentions the graph word\" .\n\ @base .\n\

\"2\" .\n", ); for iri in ["http://a.org/x", "http://a.org/p", "http://b.org/y"] { - assert!(json.contains(iri), "{iri} missing: {json}"); + assert!(triples.contains(iri), "{iri} missing: {triples}"); } - assert!(!json.contains("http://b.org/x"), " was re-based: {json}"); + assert!( + !triples.contains("http://b.org/x"), + " was re-based: {triples}" + ); } /// The txn-meta block expands with the prefixes where it appears, on @@ -2856,53 +2578,4 @@ ex:alice ex:note "value with a { brace" . .to_string(); assert!(err.contains("version specifier"), "{err}"); } - - #[test] - fn unwrap_keeps_block_triples_in_place() { - let input = "@prefix ex: .\n\ - GRAPH ex:g { ex:a ex:p [ ex:q \"}\" ] }\n\ - GRAPH <#txn-meta> { ex:m \"x\" . }\n\ - {\n ex:b ex:p ( 1 2 ) .\n}\n"; - let unwrapped = unwrap_trig_graph_blocks(input).unwrap(); - assert_eq!(unwrapped.turtle.len(), input.len(), "edits are same-length"); - assert_eq!( - unwrapped.turtle.lines().count(), - input.lines().count(), - "newlines survive" - ); - assert_eq!( - unwrapped.graph_iris, - ["http://example.org/g", "http://example.org/g"] - ); - assert!(!unwrapped.mixes_default_and_named); - assert_eq!(unwrapped.raw_meta.map(|m| m.triples.len()), Some(1)); - - let lines: Vec<&str> = unwrapped.turtle.lines().map(str::trim).collect(); - // The `}` in the string literal is left alone; the block's own `}` - // becomes the missing `.`. - assert_eq!(lines[1], "ex:a ex:p [ ex:q \"}\" ] ."); - assert_eq!(lines[2], "", "the txn-meta block is removed"); - assert_eq!(lines[3], ""); - assert_eq!(lines[4], "ex:b ex:p ( 1 2 ) ."); - assert_eq!(lines[5], "", "a block ending in `.` needs no second one"); - } - - #[test] - fn unwrap_reports_default_triples_beside_blocks() { - let input = "@prefix ex: .\n\ - ex:a ex:p ex:o .\nGRAPH ex:g { ex:b ex:p ex:o . }\n"; - assert!( - unwrap_trig_graph_blocks(input) - .unwrap() - .mixes_default_and_named - ); - } - - #[test] - fn unwrap_passes_plain_turtle_through() { - let input = "@prefix ex: .\nex:a ex:p ex:o .\n"; - let unwrapped = unwrap_trig_graph_blocks(input).unwrap(); - assert_eq!(unwrapped.turtle, input); - assert!(unwrapped.graph_iris.is_empty()); - } } diff --git a/fluree-graph-turtle/src/lib.rs b/fluree-graph-turtle/src/lib.rs index 5130864fe6..e8f10d40ce 100644 --- a/fluree-graph-turtle/src/lib.rs +++ b/fluree-graph-turtle/src/lib.rs @@ -57,6 +57,14 @@ use serde_json::Value as JsonValue; /// /// The resulting JSON is in expanded JSON-LD format, suitable for /// `fluree_db_transact::parse_transaction()`. +/// +/// The conversion is lossy. A collection becomes its entries as plain +/// values, without order or repeats; an `rdf:type` object that is not an IRI +/// is refused or read as one; and expanding the JSON applies JSON-LD's +/// compact-IRI rules, which refuse an absolute IRI whose scheme reads as an +/// undefined prefix (`tag:`, `kb:`). Transactions stage RDF text from one +/// parse (`fluree_db_transact::parse_rdf_text`); the crates that stage them +/// disallow this function in their `clippy.toml`. pub fn parse_to_json(input: &str) -> Result { let mut sink = GraphCollectorSink::new(); parse(input, &mut sink)?; diff --git a/testsuite-sparql/src/rdf_handlers.rs b/testsuite-sparql/src/rdf_handlers.rs index f547c2cb40..dc041706b4 100644 --- a/testsuite-sparql/src/rdf_handlers.rs +++ b/testsuite-sparql/src/rdf_handlers.rs @@ -5,10 +5,10 @@ //! positive-syntax actions must parse, negative-syntax actions must not, //! and evaluation tests parse an action `.ttl` and compare the resulting //! graph with an expected `.nt` up to blank-node isomorphism. Everything -//! runs through [`GraphCollectorSink`] — the same sink behind -//! `parse_to_json`, i.e. the Turtle → JSON-LD conversion used by upsert, -//! graph sync and memory import — so a suite regression here is a -//! regression on a shipped ingest path. +//! runs through [`GraphCollectorSink`], the sink behind `parse_to_json` +//! (the Turtle → JSON-LD conversion memory import uses). The parser is the +//! one every RDF-text write path reads with (insert, upsert, graph sync), +//! so a suite regression here is a regression on a shipped ingest path. //! //! N-Triples tests are dispatched to the same parser: N-Triples is a //! syntactic subset of Turtle, so every positive N-Triples document is a diff --git a/testsuite-sparql/tests/w3c_rdf.rs b/testsuite-sparql/tests/w3c_rdf.rs index 640d28a301..ce9095417f 100644 --- a/testsuite-sparql/tests/w3c_rdf.rs +++ b/testsuite-sparql/tests/w3c_rdf.rs @@ -1,10 +1,10 @@ //! W3C RDF (Turtle) test suite registration. //! //! Runs the vendored `rdf-tests` RDF 1.1 and RDF 1.2 Turtle manifests -//! through the Turtle parser on the `GraphCollectorSink` path — the parser -//! behind `parse_to_json`, i.e. the conversion every JSON-LD write path -//! (upsert, graph sync, memory import) uses for Turtle input. Same contract -//! as `w3c_sparql.rs`: a suite is green when every test passes or appears in +//! through the Turtle parser on the `GraphCollectorSink` path (the sink +//! behind `parse_to_json`, which memory import uses). Insert, upsert and +//! graph sync read RDF text with the same parser. Same contract as +//! `w3c_sparql.rs`: a suite is green when every test passes or appears in //! its register, and `check_testsuite` polices the register both ways. use anyhow::Result; From 442a2577c2b5c7714f2a576bde9e11725ff45594 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 07:00:06 -0400 Subject: [PATCH 23/36] fix(cli): sync sends RDF text as written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `fluree sync` converted Turtle to JSON-LD client-side, so a Turtle export lost collection order and refused `tag:`/`kb:` IRIs before it reached the server. RDF text is now sent as written: locally as a `GraphPayload::Rdf`, remotely as `text/turtle` (or `application/trig` for a body with graph blocks), and parsed once where it is staged. Only a server from before `/sync` read RDF bodies answers 415, before staging anything; for that one a Turtle body is converted to JSON-LD, as older CLIs did, and sent again. TriG has no JSON-LD form for its blocks and is not converted. TriG detection (`is_trig_body`, used by sync, `validate` and `--shacl`) is the locator's: a token pass, so a block holding `[ … ]` or a collection is TriG, not "not TriG" as the old content parse made it. --- fluree-db-cli/clippy.toml | 8 + fluree-db-cli/src/commands/graph_sync.rs | 287 +++++++++++++++++------ fluree-db-cli/src/detect.rs | 17 +- fluree-db-cli/src/remote_client.rs | 39 ++- 4 files changed, 276 insertions(+), 75 deletions(-) create mode 100644 fluree-db-cli/clippy.toml diff --git a/fluree-db-cli/clippy.toml b/fluree-db-cli/clippy.toml new file mode 100644 index 0000000000..d30e20b501 --- /dev/null +++ b/fluree-db-cli/clippy.toml @@ -0,0 +1,8 @@ +# RDF text is sent and staged as written, and parsed once where it is +# staged (`fluree_db_transact::parse_rdf_text`). Converting it to JSON-LD +# first loses collection order, refuses IRIs whose scheme reads as an +# undefined prefix (`tag:`, `kb:`), and reads literals differently from +# insert; the one fallback that needs it says so where it calls it. +disallowed-methods = [ + { path = "fluree_graph_turtle::parse_to_json", reason = "lossy JSON-LD round trip: send RDF text as written" }, +] diff --git a/fluree-db-cli/src/commands/graph_sync.rs b/fluree-db-cli/src/commands/graph_sync.rs index 1936cd3d96..e31ea2b58d 100644 --- a/fluree-db-cli/src/commands/graph_sync.rs +++ b/fluree-db-cli/src/commands/graph_sync.rs @@ -14,6 +14,7 @@ use crate::context::{self, LedgerMode}; use crate::detect; use crate::error::{CliError, CliResult}; use crate::input; +use crate::remote_client::{RemoteLedgerClient, RemoteLedgerError}; use fluree_db_api::server_defaults::FlureeDir; use fluree_db_api::{GraphPayload, GraphSel, SyncGraphOpts, SyncGraphReport, TxnOpts}; use std::path::Path; @@ -38,13 +39,14 @@ pub struct SyncArgs<'a> { /// Where the graph's desired contents come from. /// -/// Today: RDF text. Designed as the seam for mapped sources — an R2RML -/// mapping applied to an Iceberg table, CSV, or spreadsheet would be a new -/// variant whose [`SyncSource::into_payload`] materializes the mapping's +/// Today: RDF text or JSON-LD. Designed as the seam for mapped sources — an +/// R2RML mapping applied to an Iceberg table, CSV, or spreadsheet would be a +/// new variant whose [`SyncSource::into_payload`] materializes the mapping's /// output (locally, or via a server-side materialization when running -/// `--remote`) into the same JSON-LD payload. +/// `--remote`) into the same payload. pub enum SyncSource { - /// Turtle or JSON-LD text, already read from a file / expression / stdin. + /// Turtle, TriG or JSON-LD text, already read from a file / expression / + /// stdin. RdfText { content: String, format: detect::DataFormat, @@ -54,36 +56,87 @@ pub enum SyncSource { /// The desired contents, as submitted. pub enum SyncPayload { JsonLd(serde_json::Value), - /// Sent as TriG: the Turtle-to-JSON-LD conversion has no graph blocks. - /// The API or server requires every block to name the target graph. - Trig(String), + /// RDF text as written: Turtle, N-Triples, or TriG (`trig`) whose graph + /// blocks must name the target graph. It is parsed once where it is + /// staged, as every write lane parses RDF text. + Rdf { + text: String, + trig: bool, + }, } impl SyncSource { - /// Materialize the desired contents as one payload. - /// - /// Turtle is converted to JSON-LD client-side, so a Turtle export (the - /// common ontology-editor case) works against a server from before - /// `/sync` read RDF bodies. TriG is sent as-is. + /// Materialize the desired contents as one payload. RDF text is sent as + /// written; nothing converts it to JSON-LD on the way. pub fn into_payload(self) -> CliResult { match self { SyncSource::RdfText { content, format } => match format { detect::DataFormat::JsonLd => { Ok(SyncPayload::JsonLd(serde_json::from_str(&content)?)) } - // TriG always fails the Turtle parse, so a TriG body sniffed - // or named as Turtle is only looked for once that happens. - detect::DataFormat::Turtle => match fluree_graph_turtle::parse_to_json(&content) { - Ok(json) => Ok(SyncPayload::JsonLd(json)), - Err(_) if detect::is_trig_body(&content) => Ok(SyncPayload::Trig(content)), - Err(e) => Err(CliError::Usage(format!("failed to parse Turtle: {e}"))), - }, - detect::DataFormat::Trig => Ok(SyncPayload::Trig(content)), + detect::DataFormat::Turtle => { + let trig = detect::is_trig_body(&content); + Ok(SyncPayload::Rdf { + text: content, + trig, + }) + } + detect::DataFormat::Trig => Ok(SyncPayload::Rdf { + text: content, + trig: true, + }), }, } } } +/// Send `payload` to a remote `/sync`. RDF text goes as written; only a +/// server from before `/sync` read RDF bodies answers 415, and for that one +/// Turtle is converted to JSON-LD here and sent again (TriG has no JSON-LD +/// form for its blocks, so it gets the 415). +async fn sync_remote( + client: &RemoteLedgerClient, + ledger: &str, + graph: Option<&str>, + payload: &SyncPayload, + dry_run: bool, + allow_empty: bool, +) -> CliResult { + match payload { + SyncPayload::JsonLd(json) => Ok(client + .sync_jsonld(ledger, graph, json, dry_run, allow_empty) + .await?), + SyncPayload::Rdf { text, trig } => { + let content_type = if *trig { + "application/trig" + } else { + "text/turtle" + }; + match client + .sync_rdf(ledger, graph, text, content_type, dry_run, allow_empty) + .await + { + Err(RemoteLedgerError::UnsupportedMediaType(_)) if !*trig => { + let json = turtle_as_json_ld(text)?; + Ok(client + .sync_jsonld(ledger, graph, &json, dry_run, allow_empty) + .await?) + } + other => Ok(other?), + } + } + } +} + +/// The client-side conversion a server without RDF bodies on `/sync` needs. +/// It is lossy (collection order, `rdf:type` objects that are not IRIs, IRIs +/// whose scheme reads as a prefix); upgrading the server avoids it. +#[allow(clippy::disallowed_methods)] // the one fallback that must convert +fn turtle_as_json_ld(text: &str) -> CliResult { + fluree_graph_turtle::parse_to_json(text) + .map_err(|e| CliError::Usage(format!("failed to parse Turtle: {e}"))) +} + pub async fn run(a: SyncArgs<'_>) -> CliResult<()> { if a.graph == Some("") { return Err(CliError::Usage( @@ -110,7 +163,7 @@ pub async fn run(a: SyncArgs<'_>) -> CliResult<()> { .get("@graph") .and_then(serde_json::Value::as_array) .is_some_and(Vec::is_empty), - SyncPayload::Trig(_) => false, + SyncPayload::Rdf { .. } => false, }; if explicitly_empty && !a.allow_empty { return Err(CliError::Usage( @@ -139,18 +192,15 @@ pub async fn run(a: SyncArgs<'_>) -> CliResult<()> { .. } => { let client = client.with_policy(a.policy.clone()); - let response = match &payload { - SyncPayload::JsonLd(json) => { - client - .sync_jsonld(&remote_alias, a.graph, json, a.dry_run, a.allow_empty) - .await? - } - SyncPayload::Trig(trig) => { - client - .sync_trig(&remote_alias, a.graph, trig, a.dry_run, a.allow_empty) - .await? - } - }; + let response = sync_remote( + &client, + &remote_alias, + a.graph, + &payload, + a.dry_run, + a.allow_empty, + ) + .await?; context::persist_refreshed_tokens(&client, &remote_name, a.dirs).await; if a.json { println!( @@ -170,7 +220,7 @@ pub async fn run(a: SyncArgs<'_>) -> CliResult<()> { }; let graph_payload = match &payload { SyncPayload::JsonLd(json) => GraphPayload::JsonLd(json), - SyncPayload::Trig(trig) => GraphPayload::Rdf(trig), + SyncPayload::Rdf { text, .. } => GraphPayload::Rdf(text), }; let report = fluree .sync_graph_with( @@ -271,22 +321,19 @@ fn print_remote_response(graph: Option<&str>, value: &serde_json::Value, dry_run #[cfg(test)] mod tests { use super::*; + use std::sync::{Arc, Mutex}; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + /// The fallback conversion for a server without RDF bodies on `/sync` + /// keeps RDF 1.2 annotations, as `@annotation` blocks. #[test] - fn turtle_star_sync_source_converts_to_annotation_blocks() { - // `fluree graph sync` converts Turtle client-side; an ontology - // export carrying RDF 1.2 annotations must reach the sync endpoint - // as `@annotation` blocks instead of failing the conversion. - let source = SyncSource::RdfText { - content: "@prefix ex: .\n\ - ex:alice ex:knows ex:bob ~ ex:claim1 {| ex:confidence 0.9 |} .\n" - .to_string(), - format: detect::DataFormat::Turtle, - }; - let SyncPayload::JsonLd(payload) = source.into_payload().expect("Turtle-star converts") - else { - panic!("Turtle converts to JSON-LD"); - }; + fn the_fallback_conversion_keeps_annotations() { + let payload = turtle_as_json_ld( + "@prefix ex: .\n\ + ex:alice ex:knows ex:bob ~ ex:claim1 {| ex:confidence 0.9 |} .\n", + ) + .expect("Turtle-star converts"); let alice = payload .as_array() .expect("node array") @@ -307,33 +354,139 @@ mod tests { .into_payload() } - /// TriG goes out as TriG whether it was named as TriG or detected in a - /// body that sniffed as Turtle; Turtle with a `{` in a literal stays - /// Turtle, converted to JSON-LD. + /// RDF text goes out as written. TriG is marked whether it was named as + /// TriG or detected in a body that sniffed as Turtle; Turtle with a `{` + /// in a literal stays Turtle; malformed Turtle is left for the parser + /// that stages it to report. #[test] - fn trig_is_sent_as_trig_and_turtle_as_json_ld() { + fn rdf_text_is_sent_as_written() { let trig = "GRAPH { 1 . }"; for format in [detect::DataFormat::Trig, detect::DataFormat::Turtle] { assert!(matches!( payload_of(trig, format), - Ok(SyncPayload::Trig(t)) if t == trig + Ok(SyncPayload::Rdf { text, trig: true }) if text == trig )); } - assert!(matches!( - payload_of( - " \"{v}\" .", - detect::DataFormat::Turtle - ), - Ok(SyncPayload::JsonLd(_)) - )); - let err = payload_of( + for turtle in [ + " \"{v}\" .", " ", - detect::DataFormat::Turtle, + ] { + assert!(matches!( + payload_of(turtle, detect::DataFormat::Turtle), + Ok(SyncPayload::Rdf { text, trig: false }) if text == turtle + )); + } + } + + /// A stub `/sync` answering its requests with `statuses` in turn, + /// recording each request's content type. + async fn stub(statuses: Vec) -> (RemoteLedgerClient, Arc>>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let seen: Arc>> = Arc::default(); + let recorded = Arc::clone(&seen); + tokio::spawn(async move { + for status in statuses { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut buf = Vec::new(); + let mut tmp = [0u8; 4096]; + let header_end = loop { + let n = sock.read(&mut tmp).await.unwrap(); + if n == 0 { + break buf.len(); + } + buf.extend_from_slice(&tmp[..n]); + if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + break pos + 4; + } + }; + let headers = String::from_utf8_lossy(&buf[..header_end]).to_lowercase(); + let header = |name: &str| { + headers + .lines() + .find_map(|l| l.strip_prefix(name)) + .map(|v| v.trim().to_string()) + }; + let length: usize = header("content-length:") + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + while buf.len() < header_end + length { + let n = sock.read(&mut tmp).await.unwrap(); + if n == 0 { + break; + } + buf.extend_from_slice(&tmp[..n]); + } + recorded + .lock() + .unwrap() + .push(header("content-type:").unwrap_or_default()); + let body = if status == 200 { + r#"{"ledger":"db:main","t":2}"# + } else { + r#"{"error":"unsupported media type"}"# + }; + let reply = format!( + "HTTP/1.1 {status} X\r\nContent-Type: application/json\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + sock.write_all(reply.as_bytes()).await.unwrap(); + sock.flush().await.unwrap(); + } + }); + ( + RemoteLedgerClient::new(&format!("http://{addr}/v1/fluree"), None), + seen, + ) + } + + fn turtle() -> SyncPayload { + SyncPayload::Rdf { + text: " ( 1 2 ) .".to_string(), + trig: false, + } + } + + #[tokio::test] + async fn a_server_that_reads_rdf_gets_the_text() { + let (client, seen) = stub(vec![200]).await; + sync_remote(&client, "db:main", None, &turtle(), false, false) + .await + .expect("sync"); + assert_eq!(*seen.lock().unwrap(), ["text/turtle"]); + } + + #[tokio::test] + async fn turtle_falls_back_to_json_ld_only_on_a_415() { + let (client, seen) = stub(vec![415, 200]).await; + sync_remote(&client, "db:main", None, &turtle(), false, false) + .await + .expect("sync after the fallback"); + assert_eq!(*seen.lock().unwrap(), ["text/turtle", "application/json"]); + } + + #[tokio::test] + async fn trig_is_not_converted() { + let (client, seen) = stub(vec![415]).await; + let trig = SyncPayload::Rdf { + text: + "GRAPH { 1 . }" + .to_string(), + trig: true, + }; + let err = sync_remote( + &client, + "db:main", + Some("http://example.org/g"), + &trig, + false, + false, ) - .err() - .expect("malformed Turtle") - .to_string(); - assert!(err.contains("failed to parse Turtle"), "{err}"); + .await + .expect_err("no JSON-LD form for graph blocks"); + assert!(err.to_string().contains("415"), "{err}"); + assert_eq!(*seen.lock().unwrap(), ["application/trig"]); } } diff --git a/fluree-db-cli/src/detect.rs b/fluree-db-cli/src/detect.rs index 0e8ac8aaf4..3c3efe8aa8 100644 --- a/fluree-db-cli/src/detect.rs +++ b/fluree-db-cli/src/detect.rs @@ -115,14 +115,13 @@ pub fn trig_refused(command: &str) -> CliError { )) } -/// Whether a body that sniffed or was named as Turtle is TriG: it has graph -/// blocks or txn-meta. This parses rather than looking for braces, since a -/// `{` inside a Turtle string literal is not a graph block. A body that is -/// not well-formed TriG either is left to the Turtle parser to report. +/// Whether a body that sniffed or was named as Turtle is TriG: it has a +/// graph block (`<#txn-meta>` included). Read from tokens, so a `{` inside a +/// Turtle string literal is not a graph block and a block's contents are +/// not parsed at all. A body the locator cannot read is left to the parser +/// that stages it to report. pub fn is_trig_body(content: &str) -> bool { - fluree_db_transact::might_contain_graph_block(content) - && fluree_db_transact::parse_trig_phase1(content) - .is_ok_and(|p| !p.named_graphs.is_empty() || p.raw_meta.is_some()) + fluree_db_transact::has_graph_blocks(content).unwrap_or(false) } /// [`trig_refused`] for a TriG body that reached a one-graph command as @@ -320,6 +319,10 @@ mod tests { " { 1 . }", "@prefix fluree: .\n\ GRAPH <#txn-meta> { fluree:commit:this \"m\" . }", + // Block contents the TriG reader used to refuse (`[ ]`, a + // collection) made the body read as "not TriG". + "GRAPH { \ + [ ( 1 2 ) ] . }", ] { assert_eq!( super::detect_data_format(None, trig, None).unwrap(), diff --git a/fluree-db-cli/src/remote_client.rs b/fluree-db-cli/src/remote_client.rs index 11b1445734..524905c7f0 100644 --- a/fluree-db-cli/src/remote_client.rs +++ b/fluree-db-cli/src/remote_client.rs @@ -300,6 +300,8 @@ pub enum RemoteLedgerError { Conflict(String), /// 422 Unprocessable Entity / validation error ValidationError(String), + /// 415 Unsupported Media Type: the server does not read this body format + UnsupportedMediaType(String), /// 5xx Server Error (includes server error message) ServerError(String), /// Request could not be serialized (client-side bug) @@ -323,6 +325,9 @@ impl fmt::Display for RemoteLedgerError { RemoteLedgerError::BadRequest(msg) => write!(f, "bad request: {msg}"), RemoteLedgerError::Conflict(msg) => write!(f, "conflict (409): {msg}"), RemoteLedgerError::ValidationError(msg) => write!(f, "validation error (422): {msg}"), + RemoteLedgerError::UnsupportedMediaType(msg) => { + write!(f, "unsupported media type (415): {msg}") + } RemoteLedgerError::ServerError(msg) => write!(f, "server error: {msg}"), RemoteLedgerError::InvalidRequest(msg) => write!(f, "invalid request: {msg}"), RemoteLedgerError::InvalidResponse(msg) => write!(f, "invalid response: {msg}"), @@ -446,6 +451,13 @@ impl RemoteLedgerClient { message }) } + StatusCode::UNSUPPORTED_MEDIA_TYPE => { + RemoteLedgerError::UnsupportedMediaType(if message.is_empty() { + "unsupported media type".to_string() + } else { + message + }) + } s if s.is_server_error() => RemoteLedgerError::ServerError(if message.is_empty() { format!("status {s}") } else { @@ -1283,12 +1295,37 @@ impl RemoteLedgerClient { body: &str, dry_run: bool, allow_empty: bool, + ) -> Result { + self.sync_rdf( + ledger, + graph, + body, + "application/trig", + dry_run, + allow_empty, + ) + .await + } + + /// [`sync_jsonld`](Self::sync_jsonld) with an RDF text body sent as + /// `content_type` (`text/turtle`, `application/n-triples`, + /// `application/trig`). A server that does not read RDF bodies on + /// `/sync` answers 415 ([`RemoteLedgerError::UnsupportedMediaType`]) + /// before staging anything, so the caller may retry in another format. + pub async fn sync_rdf( + &self, + ledger: &str, + graph: Option<&str>, + body: &str, + content_type: &str, + dry_run: bool, + allow_empty: bool, ) -> Result { let url = self.sync_url(ledger, graph, dry_run, allow_empty); self.send_json( reqwest::Method::POST, &url, - "application/trig", + content_type, Some(RequestBody::Text(body)), ) .await From 6ac5c48b9090e3a415a3aeebf3b9cbc36a6e00cb Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 07:11:35 -0400 Subject: [PATCH 24/36] test(sparql): TriG named graphs load in the W3C triple-term tests The harness loads named-graph data as TriG through `upsert_turtle`, and the five eval-triple-terms tests registered as "blocked on TriG GRAPH-block parsing" now load their data. None of them passes yet; each is re-attributed to the failure it now reaches: graphs-1/2 to the non- isomorphic results of the reifier model, expr-1 to the unlowered triple-term functions, update-1 to quoted triples in SPARQL UPDATE lowering, and update-2 to an annotation valued by a `GRAPH ?g` binding. The harness notes no longer describe the phase-1 reader. --- testsuite-sparql/Cargo.lock | 1 + testsuite-sparql/src/query_handler.rs | 14 +++++++------- testsuite-sparql/tests/registers/mod.rs | 24 +++++++++++++++--------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/testsuite-sparql/Cargo.lock b/testsuite-sparql/Cargo.lock index 194921ec35..fc5c6d5a7a 100644 --- a/testsuite-sparql/Cargo.lock +++ b/testsuite-sparql/Cargo.lock @@ -1125,6 +1125,7 @@ dependencies = [ "tokio", "tracing", "ulid", + "xxhash-rust", "zstd", ] diff --git a/testsuite-sparql/src/query_handler.rs b/testsuite-sparql/src/query_handler.rs index acb62898c1..396c4457af 100644 --- a/testsuite-sparql/src/query_handler.rs +++ b/testsuite-sparql/src/query_handler.rs @@ -144,11 +144,11 @@ pub fn evaluate_update_evaluation_test(test: &Test) -> Result<()> { /// named graphs. Returns the resulting ledger state. /// /// Named graphs are loaded as TriG `GRAPH { ... }` blocks through the -/// alias-based transact builder (`upsert_turtle`), which routes through -/// `parse_trig_phase1`. Each file's `@prefix`/`@base` directives are hoisted -/// above the GRAPH block (TriG directives are document-scoped), and each -/// graph loads in its own transaction so prefix declarations can't collide -/// across files. +/// alias-based transact builder (`upsert_turtle`), which parses the document +/// once with the Turtle parser. Each file's `@prefix`/`@base` directives are +/// hoisted above the GRAPH block (a directive cannot sit inside a block), +/// and each graph loads in its own transaction so prefix declarations can't +/// collide across files. async fn setup_graph_store( fluree: &Fluree, data_url: Option<&str>, @@ -191,8 +191,8 @@ async fn setup_graph_store( continue; } let trig = format!("{directives}GRAPH <{graph_name}> {{\n{body}}}\n"); - // upsert_turtle (not insert_turtle): the builder's insert_turtle - // fast path bypasses TriG GRAPH-block extraction. + // Either verb reads a TriG body; upsert keeps a repeated load of the + // same graph from accumulating copies. match fluree .graph(TEST_LEDGER) .transact() diff --git a/testsuite-sparql/tests/registers/mod.rs b/testsuite-sparql/tests/registers/mod.rs index 8a798a545a..0ac94e2808 100644 --- a/testsuite-sparql/tests/registers/mod.rs +++ b/testsuite-sparql/tests/registers/mod.rs @@ -408,18 +408,14 @@ pub const SPARQL12_EVAL_TRIPLE_TERMS: &[&str] = &[ "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#op-2", "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#order-1", "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#order-2", - // data load: blocked on TriG GRAPH-block parsing, orthogonal to star - // (D-8) — "expected subject, found 'GRAPH'" (5) - "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#expr-1", - "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#graphs-1", - "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#graphs-2", - "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#update-1", - "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#update-2", // results not isomorphic: the star data now LOADS (the collector sink - // accepts `<< >>` / `{| |}` / `~`), but Fluree's model asserts the base + // accepts `<< >>` / `{| |}` / `~`; TriG graph blocks load since RDF text + // is parsed once on every write lane), but Fluree's model asserts the base // triple of every `<< s p o >>` and binds reifiers, not triple terms — // RDF 1.2's non-asserting reified triples and `?t` triple-term bindings - // are the Option-1 epic (8) + // are the Option-1 epic (10) + "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#graphs-1", + "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#graphs-2", "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#basic-2", "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#basic-3", "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#basic-7", @@ -441,6 +437,16 @@ pub const SPARQL12_EVAL_TRIPLE_TERMS: &[&str] = &[ "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#construct-5", // SPARQL lowering: triple-term values in VALUES data not implemented (1) "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#expr-2", + // SPARQL lowering: the TRIPLE/SUBJECT/PREDICATE/OBJECT/isTRIPLE + // functions are accepted but not lowered — Option-1 epic (1) + "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#expr-1", + // SPARQL UPDATE lowering: "RDF-star quoted triple is not yet supported in + // SPARQL UPDATE lowering" (1) + "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#update-1", + // SPARQL UPDATE: an annotation whose value is the `GRAPH ?g` binding — + // "Raw IRI from graph source cannot be used as object for flake + // generation" (1) + "https://w3c.github.io/rdf-tests/sparql/sparql12/eval-triple-terms/manifest#update-2", // update-3: the `{| |}` INSERT DATA executes, but the expected post-update // TriG-star graph carries reifier semantics the harness comparison does // not model (expected 2 triples, got 0 in the default graph) (1) From ddb898a58c17f915f6cbc5739538d5f175405392 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 07:13:37 -0400 Subject: [PATCH 25/36] docs(transactions): RDF text is read once, the same way on every lane turtle.md describes how every transaction endpoint now reads Turtle and TriG: directives in document order, relative IRIs against the base in force (an error without one), literals kept as written, any IRI scheme, collections in order, document-scoped blank-node labels, errors at the document's offset. It narrows the #1930 limitation to bulk import, and documents the upsert blank-node scope with a query that lists blank nodes nothing references, pinned by a test. The ledger-config recipe upserts as written again (tested); `fluree sync` docs describe RDF bodies and the 415 fallback; edge-annotations no longer lists upsert and sync among the paths that refuse an annotated `rdf:type` edge; the compact-IRI troubleshooting entry notes that Turtle and TriG never raise it. --- docs/cli/server-integration.md | 8 ++- docs/cli/sync.md | 22 +++++--- docs/concepts/edge-annotations.md | 2 +- docs/ledger-config/README.md | 5 +- docs/ledger-config/writing-config.md | 2 +- docs/transactions/turtle.md | 30 +++++++++- docs/transactions/upsert.md | 5 +- docs/troubleshooting/common-errors.md | 2 + fluree-db-api/tests/it_rdf_text_parity.rs | 59 ++++++++++++++++++++ fluree-db-transact/src/parse/rdf_text/mod.rs | 3 + 10 files changed, 117 insertions(+), 21 deletions(-) diff --git a/docs/cli/server-integration.md b/docs/cli/server-integration.md index e8f84ef053..969036ad53 100644 --- a/docs/cli/server-integration.md +++ b/docs/cli/server-integration.md @@ -349,9 +349,11 @@ MATCH (n:Person {id: 7}) RETURN n Makes one named graph's contents exactly the payload, committing only the delta. Data-bearer auth (same bracket as `/insert` / `/upsert`), not admin. -The CLI converts Turtle to JSON-LD client-side and sends it as -`application/json`; TriG, which that conversion cannot carry, is sent as -`application/trig`. A dry run answers with a delta report and +The CLI sends RDF text as written: Turtle as `text/turtle`, TriG as +`application/trig`, JSON-LD as `application/json`. A server that answers +`415 Unsupported Media Type` to a Turtle body (one from before `/sync` read +RDF bodies) gets it again converted to JSON-LD client-side; TriG is never +converted. A dry run answers with a delta report and must commit nothing; a real run answers with the standard transact response. Designed so the CLI's source of desired contents (today RDF text; later R2RML-mapped Iceberg / CSV / spreadsheet data) is invisible to the server — diff --git a/docs/cli/sync.md b/docs/cli/sync.md index c8059b1663..1d7e92000a 100644 --- a/docs/cli/sync.md +++ b/docs/cli/sync.md @@ -42,15 +42,19 @@ produce no flakes, and an identical payload produces **no commit**. History is preserved. See [Sync (graph synchronization)](../transactions/sync.md) for the full semantics, safety rails, and blank-node behavior. -Turtle input is converted to JSON-LD client-side before submission, so a -Turtle export works against any server that implements the `/sync` -endpoint, including servers from before it accepted Turtle bodies. A `.nt` -file is read as Turtle, of which N-Triples is a subset. - -TriG is sent as TriG rather than converted. It is recognized by a `.trig` -name or `--format trig`, and also when a body read as Turtle turns out to -hold graph blocks, as when it is piped in. Its blocks must name the -`--graph` target, and cannot sit beside default-graph triples (see +Turtle input is sent as written and parsed where it is staged, the way +every write reads RDF text (see +[How RDF text is read](../transactions/turtle.md#how-rdf-text-is-read)). +A `.nt` file is read as Turtle, of which N-Triples is a subset. With +`--remote`, a server from before `/sync` accepted Turtle bodies answers +415; for that server the CLI converts the Turtle to JSON-LD and sends it +again. The conversion loses collection order and refuses some IRI schemes +(`tag:`, `kb:`), so upgrading the server is the better fix. + +TriG is sent as TriG. It is recognized by a `.trig` name or +`--format trig`, and also when a body read as Turtle turns out to hold +graph blocks, as when it is piped in. Its blocks must name the `--graph` +target, and cannot sit beside default-graph triples (see [payload formats](../transactions/sync.md#payload-formats)): ```bash diff --git a/docs/concepts/edge-annotations.md b/docs/concepts/edge-annotations.md index 1db70fb81d..2b59c7a203 100644 --- a/docs/concepts/edge-annotations.md +++ b/docs/concepts/edge-annotations.md @@ -27,7 +27,7 @@ If a fact is naturally about a *node* (Alice's birthdate, Acme's industry), put |---|---|---| | **JSON-LD insert / upsert / update** | `@annotation` (or alias `@edge`) on a value object | Most ergonomic. Covers literal-valued edges (with explicit `@type` / `@language`), parallel annotations, named reifiers, body cascades, and **named-graph edges** (an annotation on an edge inside a named graph is written into that same graph, keeping the edge's graph identity). `@reifies` is a query-side construct, **not** an insert form (user-authored `@reifies` on a write is rejected). | | **SPARQL 1.2 UPDATE** | `INSERT DATA { :s :p :o {\| ... \|} }`, `~ `, optional `INSERT { } WHERE { }` templates | Use this when integrating with SPARQL pipelines or when porting from RDF 1.2 / SPARQL-star. **Default graph only:** an annotation tail inside an explicit `GRAPH { }` block, or under a `WITH ` template, is rejected — use the JSON-LD surface or TriG-star for named-graph edge annotations. See [SPARQL 1.2 surface](#sparql-12--rdf-12-surface) below for the per-operation rules. | -| **Turtle / N-Triples / TriG / N-Quads ingest** (`insert`, `upsert`, bulk `import`, graph sync (`fluree sync`, `/sync`), memory import; TriG via `insert` / `upsert` / `import` / `fluree sync` / `/sync`) | RDF 1.2 asserting forms: `:s :p :o ~ {\| ... \|}`, `<< :s :p :o ~ :r >>` in subject or object position, and the canonical `:r rdf:reifies <<( :s :p :o )>>` (the only star spelling N-Triples and N-Quads have) — in the default graph and inside TriG `GRAPH { }` blocks alike | Same on-disk bundle as `@annotation`. An annotation inside a `GRAPH { }` block, or on an N-Quads statement with a graph label, is written into that graph with the edge's graph identity, exactly as JSON-LD `@graph` + `@annotation` does. Every form asserts the base edge (Fluree reifies asserted edges, so `<< s p o >>` and `rdf:reifies <<( s p o )>>` are asserting here). Rejected with a specific error: `<<( ... )>>` anywhere other than the object of `rdf:reifies`, nested triple terms, star constructs inside an annotation body, annotations on collections, and annotations in a TriG `<#txn-meta>` block. Paths that convert to JSON-LD first (`upsert`, `graph sync`, memory import) also reject an annotation on an `rdf:type` edge. See [Turtle ingest](../transactions/turtle.md#edge-annotations-rdf-12--turtle-star). | +| **Turtle / N-Triples / TriG / N-Quads ingest** (`insert`, `upsert`, bulk `import`, graph sync (`fluree sync`, `/sync`), memory import; TriG via `insert` / `upsert` / `import` / `fluree sync` / `/sync`) | RDF 1.2 asserting forms: `:s :p :o ~ {\| ... \|}`, `<< :s :p :o ~ :r >>` in subject or object position, and the canonical `:r rdf:reifies <<( :s :p :o )>>` (the only star spelling N-Triples and N-Quads have) — in the default graph and inside TriG `GRAPH { }` blocks alike | Same on-disk bundle as `@annotation`. An annotation inside a `GRAPH { }` block, or on an N-Quads statement with a graph label, is written into that graph with the edge's graph identity, exactly as JSON-LD `@graph` + `@annotation` does. Every form asserts the base edge (Fluree reifies asserted edges, so `<< s p o >>` and `rdf:reifies <<( s p o )>>` are asserting here). Rejected with a specific error: `<<( ... )>>` anywhere other than the object of `rdf:reifies`, nested triple terms, star constructs inside an annotation body, annotations on collections, and annotations in a TriG `<#txn-meta>` block. The memory import converts Turtle to JSON-LD first and also rejects an annotation on an `rdf:type` edge; the transaction endpoints accept one like any other edge. See [Turtle ingest](../transactions/turtle.md#edge-annotations-rdf-12--turtle-star). | Annotations are backed by a set of [reserved system predicates](../reference/vocabulary.md#edge-annotation-predicates-reserved) that application writes can't author by hand — mint annotations through `@annotation` / `@edge` (JSON-LD) or the RDF 1.2 forms (`~`, `{| |}`, `<< >>`, `rdf:reifies <<( )>>`) in SPARQL UPDATE and Turtle. (Bulk import is an administrative bootstrap path that may ingest already-lowered annotation bundles without up-front validation; see the [storage-internals design doc](../design/edge-annotations.md) for how malformed bundles are handled.) diff --git a/docs/ledger-config/README.md b/docs/ledger-config/README.md index 25413090ea..dd7b09066e 100644 --- a/docs/ledger-config/README.md +++ b/docs/ledger-config/README.md @@ -204,9 +204,8 @@ User queries against the config graph go through **policy enforcement**. If `f:d The config graph is written and queried through normal CLI transaction and query commands: ```bash -# Write config via SPARQL UPDATE. TriG through `fluree insert` works too, but -# not with the anonymous blank nodes (`[ … ]`) config settings are usually -# written as (#1930); see "Writing from the CLI" in writing-config.md. +# Write config via SPARQL UPDATE. TriG through `fluree insert` or +# `fluree upsert` works too; see "Writing from the CLI" in writing-config.md. fluree update --ledger mydb:main --format sparql -e ' PREFIX f: INSERT DATA { diff --git a/docs/ledger-config/writing-config.md b/docs/ledger-config/writing-config.md index 7632f23ffd..f9d86ed668 100644 --- a/docs/ledger-config/writing-config.md +++ b/docs/ledger-config/writing-config.md @@ -33,7 +33,7 @@ GRAPH { ## Writing from the CLI -`fluree insert` and `fluree upsert` read TriG, but the recipe above writes its settings as anonymous blank nodes (`[ … ]`) inside the `GRAPH` block, which those commands do not read yet ([#1930](https://github.com/fluree/db/issues/1930)). Write the config graph with SPARQL UPDATE through `fluree update` instead: +`fluree insert` and `fluree upsert` read the TriG recipe above as written, anonymous blank nodes (`[ … ]`) inside the `GRAPH` block included. The same settings can also be written with SPARQL UPDATE through `fluree update`: ```bash fluree update -l mydb:main --format sparql -e ' diff --git a/docs/transactions/turtle.md b/docs/transactions/turtle.md index 5ffe8fd3fe..def3f6156b 100644 --- a/docs/transactions/turtle.md +++ b/docs/transactions/turtle.md @@ -33,7 +33,7 @@ Fluree supports Turtle and TriG on different endpoints with different semantics: | `/data` ([Graph Store](../api/graph-store.md) `PUT` / `POST`) | Supported | Supported (one graph per request) | - **Insert** (`/insert`): Pure insert semantics: triples are added and existing values are kept. Turtle is parsed straight to flakes. A TriG body's graph blocks land in their named graphs and a `<#txn-meta>` block becomes commit metadata; a body with graph blocks is read as TriG under either content type. -- **Upsert** (`/upsert`): For each (subject, predicate) pair, existing values are retracted before new values are asserted. Supports TriG with GRAPH blocks for named graph ingestion. +- **Upsert** (`/upsert`): For each (graph, subject, predicate) the document names, the stored values, every language tag and list position included, are retracted and the document's values asserted (see [Upsert](upsert.md#language-tags-lists-and-blank-nodes)). A TriG body's blocks upsert into their graphs. - **Sync** (`/sync?graph=`): The body becomes the named graph's entire contents, committing only the difference; an unchanged body commits nothing. See [Sync](sync.md#payload-formats) for the TriG rules. N-Triples is a subset of Turtle and is accepted wherever Turtle is: as `application/n-triples` over HTTP, and as a `.nt` file or `--format nt` from the CLI. @@ -621,7 +621,31 @@ curl -X POST "http://localhost:8090/v1/fluree/upsert?ledger=mydb:main" \ From the CLI, `fluree insert -f data.trig` and `fluree upsert -f data.trig` do the same. -**Known limitation ([#1930](https://github.com/fluree/db/issues/1930)):** on `/insert`, `/upsert` and bulk import, a `GRAPH` block's contents are read by a smaller parser that rejects anonymous blank nodes (`[ … ]`) and collections (`( … )`) with `expected object, found '['`. Triples outside blocks are unaffected. `/sync` and the [Graph Store Protocol](../api/graph-store.md) read block contents with the full Turtle parser, so they accept both; for `/insert` and `/upsert`, use labeled blank nodes (`_:b1`) inside blocks. +A `GRAPH` block's contents are Turtle like the rest of the document: anonymous blank nodes (`[ … ]`), collections (`( … )`) and RDF 1.2 annotations work inside blocks on every transaction endpoint. **Known limitation ([#1930](https://github.com/fluree/db/issues/1930)):** bulk import (`fluree create --from data.trig`) still reads block contents with a smaller parser that rejects `[ … ]` and `( … )` with `expected object, found '['`; use labeled blank nodes (`_:b1`) inside blocks there. + +### How RDF Text Is Read + +Every transaction endpoint reads Turtle and TriG with the same parser, once, and stores what it reads; nothing is converted to JSON-LD on the way. So the same document stores the same facts through `/insert`, `/upsert`, `/sync` and the Graph Store routes: + +- **Directives apply in document order.** A `@prefix` or `@base` applies to what follows it, in the default graph and in blocks alike; redefining a prefix later does not change the IRIs before it. A block label is resolved under the declarations in force where the block appears. +- **Relative IRIs need a base.** `` resolves against the `@base` in force; with no base it is a parse error, inside a block or outside one. `<#txn-meta>` with no base in force names the commit-metadata block, as always. +- **Literals are stored as written.** A lexical form that does not fit its datatype (`"abc"^^xsd:integer`) is kept, with its declared datatype, on every endpoint, as insert always did. +- **Any IRI scheme works.** `tag:`, `kb:` and other schemes are stored verbatim, however they are spelled. +- **Collections keep their order and repeats**, and `rdf:type` takes a blank node or a literal like any other predicate. +- **Blank-node labels are document-scoped:** `_:b` in the default graph and in two blocks is one node. Each `[ ]` is a new node. +- **Errors point into the document as sent:** a parse error in a block reports its position in the whole document, and a malformed block (nested, unclosed, a directive inside it, a blank-node label) is named as such. + +An upsert scopes a document's blank nodes to its content, so upserting the same document again addresses the same nodes and commits nothing. The scope ignores statement order (anonymous `[ ]` nodes are numbered in document order, so moving statements that contain them relative to each other gives a new scope); any change to the statements gives a new scope and new nodes, and the previous nodes stay. To find blank-node subjects that nothing references any more: + +```sparql +SELECT DISTINCT ?b WHERE { + ?b ?p ?o . + FILTER(isBlank(?b)) + FILTER NOT EXISTS { ?s ?q ?b } +} +``` + +Review the list before deleting: a document can hold unreferenced blank-node subjects on purpose. Deleting an orphaned node leaves the nodes it referenced orphaned in turn, so repeat until the query returns nothing. ### Querying Named Graphs @@ -672,7 +696,7 @@ Fluree assigns internal graph IDs to named graphs: ### TriG with Transaction Metadata -You can combine named graphs with transaction metadata using the special `#txn-meta` graph fragment. This works on both `insert` and `upsert`: +You can combine named graphs with transaction metadata using the special `#txn-meta` graph fragment (with no `@base` in force). This works on both `insert` and `upsert`: ```trig @prefix ex: . diff --git a/docs/transactions/upsert.md b/docs/transactions/upsert.md index 5191a94a5d..0b4d40db8a 100644 --- a/docs/transactions/upsert.md +++ b/docs/transactions/upsert.md @@ -127,7 +127,10 @@ current value is retracted exactly as it is stored. re-upsert of a document with blank nodes commits nothing. A changed document mints new blank nodes; the old ones stay (see [Stable blank-node ids](update-where-delete-insert.md#editing-blank-node-structures-stable-_fdb--ids) - for editing a stored blank node in place). + for editing a stored blank node in place). For Turtle and TriG the + identity comes from the parsed statements, not their order or spelling; + see [How RDF text is read](turtle.md#how-rdf-text-is-read) for the details + and a query that lists blank nodes nothing references. - **Named graphs.** The graph is part of the unit: an upsert into a named graph replaces that graph's values and leaves the same predicate's values in other graphs alone. diff --git a/docs/troubleshooting/common-errors.md b/docs/troubleshooting/common-errors.md index 71e21b2265..bb61a50e5a 100644 --- a/docs/troubleshooting/common-errors.md +++ b/docs/troubleshooting/common-errors.md @@ -159,6 +159,8 @@ This error fires from the JSON-LD strict compact-IRI guard. A value that *looks* 2. Misspelled or missing prefix in `@context` 3. Intentionally using a bare `prefix:suffix` string as an opaque identifier +Turtle and TriG writes are not read as JSON-LD, so their IRIs, whatever the scheme (`tag:`, `urn:`, `kb:`), never raise this error. + ### Solutions **Add the missing prefix to @context** (most common fix): diff --git a/fluree-db-api/tests/it_rdf_text_parity.rs b/fluree-db-api/tests/it_rdf_text_parity.rs index 9a66cb98dd..d2cce1fb64 100644 --- a/fluree-db-api/tests/it_rdf_text_parity.rs +++ b/fluree-db-api/tests/it_rdf_text_parity.rs @@ -605,6 +605,34 @@ async fn a_turtle_list_upsert_replaces_the_list_in_order() { ); } +/// The ledger-config recipe (`docs/ledger-config/writing-config.md`) +/// upserts as written: its settings are anonymous blank nodes inside the +/// config graph's block, which the TriG block reader refused (#1511). +#[tokio::test] +async fn the_config_recipe_upserts_as_written() { + let fluree = memory(); + let id = "it/rdf-config:main"; + let config = format!("urn:fluree:{id}#config"); + let recipe = format!( + "@prefix f: .\n\ + GRAPH <{config}> {{\n\ + a f:LedgerConfig ;\n\ + f:reasoningDefaults [ f:reasoningModes f:RDFS ] .\n\ + }}\n" + ); + let result = fluree + .upsert_turtle(genesis_ledger(&fluree, id), &recipe) + .await + .expect("the recipe upserts"); + let stored = facts(&result.ledger, Some(&config)).await; + assert!( + stored + .iter() + .any(|f| f.contains("reasoningModes https://ns.flur.ee/db#RDFS")), + "{stored:#?}" + ); +} + // ============================================================================= // Identity: the same document addresses the same blank nodes // ============================================================================= @@ -674,6 +702,37 @@ async fn upserting_the_same_document_again_commits_nothing() { .any(|f| f.contains("note String(\"edited\")"))); } +/// The query `turtle.md` documents for blank nodes nothing references +/// finds the node an edited document left behind, and not the one that +/// replaced it. +#[tokio::test] +async fn the_documented_query_lists_the_blank_nodes_an_edit_left_behind() { + let fluree = memory(); + let v1 = "@prefix ex: .\nex:A ex:part [ ex:k \"v1\" ] .\n"; + let first = fluree + .upsert_turtle(genesis_ledger(&fluree, "it/rdf-orphans:main"), v1) + .await + .expect("v1"); + let v2 = v1.replace("\"v1\"", "\"v2\""); + let second = fluree.upsert_turtle(first.ledger, &v2).await.expect("v2"); + let orphans = select( + &fluree, + &second.ledger, + "SELECT DISTINCT ?b WHERE {\n ?b ?p ?o .\n FILTER(isBlank(?b))\n \ + FILTER NOT EXISTS { ?s ?q ?b }\n}", + ) + .await; + assert_eq!(orphans.len(), 1, "{orphans:?}"); + let values = select( + &fluree, + &second.ledger, + "SELECT ?v WHERE { ?b ?v FILTER(isBlank(?b)) \ + FILTER NOT EXISTS { ?s ?q ?b } }", + ) + .await; + assert_eq!(values, vec![row(&["v1"])], "the orphan is v1's node"); +} + // ============================================================================= // Fuel and the raw transaction // ============================================================================= diff --git a/fluree-db-transact/src/parse/rdf_text/mod.rs b/fluree-db-transact/src/parse/rdf_text/mod.rs index a8e4144a77..5aa7edd5ac 100644 --- a/fluree-db-transact/src/parse/rdf_text/mod.rs +++ b/fluree-db-transact/src/parse/rdf_text/mod.rs @@ -519,6 +519,9 @@ mod tests { Placement::AsWritten, ); assert!(e.contains("graph label"), "{e}"); + // Outside a block, the same rule. + let e = err(" 1 .", Placement::AsWritten); + assert!(e.contains("relative"), "{e}"); } // ---- Block contents: the full grammar ------------------------------- From 999d28cce796b11310a7db9a06a9a9d01e7318d0 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 07:40:54 -0400 Subject: [PATCH 26/36] perf(api): a TriG insert locates its document once The TriG insert fallback asked the locator whether the document has a graph block and then parsed it, which located it again: two token passes over every TriG insert. `parse_trig_txn` locates once and returns `None` for a document with no block, so the streaming parser's Turtle error stands as before. The routing stamp now fires once the document parses as TriG. --- fluree-db-api/src/tx.rs | 23 ++++-- fluree-db-transact/src/lib.rs | 3 +- fluree-db-transact/src/parse/mod.rs | 3 +- .../src/parse/rdf_text/locate.rs | 15 ++-- fluree-db-transact/src/parse/rdf_text/mod.rs | 72 +++++++++++++++---- 5 files changed, 88 insertions(+), 28 deletions(-) diff --git a/fluree-db-api/src/tx.rs b/fluree-db-api/src/tx.rs index 9e5614a536..a158ee1abf 100644 --- a/fluree-db-api/src/tx.rs +++ b/fluree-db-api/src/tx.rs @@ -3598,19 +3598,28 @@ impl crate::Fluree { policy: Option<&crate::PolicyContext>, ) -> Result { // Most rejected documents are plain Turtle with a typo; the locator - // rules TriG out from tokens alone. - if !fluree_db_transact::has_graph_blocks(trig).map_err(rdf_text_error)? { + // rules TriG out from tokens alone, and a TriG document is located + // once. + let mut ns_registry = NamespaceRegistry::from_db(&ledger.snapshot); + let parsed = { + let parse_span = + tracing::debug_span!("txn_parse", txn_type = "insert", rdf_bytes = trig.len()); + let _guard = parse_span.enter(); + fluree_db_transact::parse_trig_txn(trig, TxnType::Insert, txn_opts, &mut ns_registry) + .map_err(rdf_text_error)? + }; + let Some((txn, _)) = parsed else { return Err(turtle_err); - } + }; stamp_fast_path( TURTLE_INSERT_SITE, FastPathOutcome::Fallback(FastPathFallback::GateDeclined), ); - self.stage_rdf_text_tracked( + self.stage_built_txn_tracked( ledger, - TxnType::Insert, - trig, - txn_opts, + txn, + ns_registry, + &JsonValue::Null, index_config, tracker, policy, diff --git a/fluree-db-transact/src/lib.rs b/fluree-db-transact/src/lib.rs index 1e5d18a395..abb246026c 100644 --- a/fluree-db-transact/src/lib.rs +++ b/fluree-db-transact/src/lib.rs @@ -81,7 +81,8 @@ pub use namespace::{ SharedNamespaceAllocator, BLANK_NODE_ID_PREFIX, BLANK_NODE_PREFIX, }; pub use parse::{ - has_graph_blocks, parse_rdf_text, parse_rdf_text_txn, Placement, RdfText, RdfTextSummary, + has_graph_blocks, parse_rdf_text, parse_rdf_text_txn, parse_trig_txn, Placement, RdfText, + RdfTextSummary, }; pub use parse::{ might_contain_graph_block, parse_graph_insert, parse_sync_transaction, parse_transaction, diff --git a/fluree-db-transact/src/parse/mod.rs b/fluree-db-transact/src/parse/mod.rs index 932decae01..04cb9a0192 100644 --- a/fluree-db-transact/src/parse/mod.rs +++ b/fluree-db-transact/src/parse/mod.rs @@ -51,7 +51,8 @@ pub(crate) const CLAUSE_KEYS: &[&str] = &["where", "delete", "insert", "upsert", pub use jsonld::{parse_graph_insert, parse_sync_transaction, parse_transaction}; pub use nquads::nquads_to_trig; pub use rdf_text::{ - has_graph_blocks, parse_rdf_text, parse_rdf_text_txn, Placement, RdfText, RdfTextSummary, + has_graph_blocks, parse_rdf_text, parse_rdf_text_txn, parse_trig_txn, Placement, RdfText, + RdfTextSummary, }; pub use trig_meta::{ might_contain_graph_block, parse_trig_phase1, resolve_trig_meta, NamedGraphBlock, RawObject, diff --git a/fluree-db-transact/src/parse/rdf_text/locate.rs b/fluree-db-transact/src/parse/rdf_text/locate.rs index 72c0bafdb8..8ca930c551 100644 --- a/fluree-db-transact/src/parse/rdf_text/locate.rs +++ b/fluree-db-transact/src/parse/rdf_text/locate.rs @@ -264,10 +264,17 @@ pub fn has_graph_blocks(text: &str) -> Result { if !might_contain_graph_block(text) { return Ok(false); } - Ok(locate(text)? - .segments - .iter() - .any(|s| !matches!(s.kind, SegmentKind::Default))) + Ok(locate(text)?.has_blocks()) +} + +impl Located<'_> { + /// Whether the document holds a graph block (`<#txn-meta>` and + /// anonymous `{ … }` blocks included). + pub(super) fn has_blocks(&self) -> bool { + self.segments + .iter() + .any(|s| !matches!(s.kind, SegmentKind::Default)) + } } #[cfg(test)] diff --git a/fluree-db-transact/src/parse/rdf_text/mod.rs b/fluree-db-transact/src/parse/rdf_text/mod.rs index 5aa7edd5ac..609baf5129 100644 --- a/fluree-db-transact/src/parse/rdf_text/mod.rs +++ b/fluree-db-transact/src/parse/rdf_text/mod.rs @@ -29,7 +29,7 @@ use std::sync::Arc; mod locate; pub use locate::has_graph_blocks; -use locate::{locate, SegmentKind}; +use locate::{locate, Located, SegmentKind}; /// Where a document's statements land. #[derive(Clone, Copy, Debug)] @@ -74,10 +74,43 @@ pub fn parse_rdf_text_txn( txn_type: TxnType, placement: Placement<'_>, sync: bool, - mut opts: TxnOpts, + opts: TxnOpts, ns: &mut NamespaceRegistry, ) -> Result<(Txn, RdfTextSummary)> { - let RdfText { parts, statements } = parse_rdf_text(text, placement, ns)?; + let parsed = parse_rdf_text(text, placement, ns)?; + into_txn(parsed, txn_type, placement, sync, opts) +} + +/// [`parse_rdf_text_txn`] for text the streaming Turtle parser stopped on: +/// `Ok(None)` when the locator finds no graph block (`<#txn-meta>` +/// included), so the text is not TriG and the caller's Turtle error stands. +/// The document is located once. +pub fn parse_trig_txn( + text: &str, + txn_type: TxnType, + opts: TxnOpts, + ns: &mut NamespaceRegistry, +) -> Result> { + if !might_contain_graph_block(text) { + return Ok(None); + } + let located = locate(text)?; + if !located.has_blocks() { + return Ok(None); + } + let parsed = parse_located(&located, Placement::AsWritten, ns)?; + into_txn(parsed, txn_type, Placement::AsWritten, false, opts).map(Some) +} + +/// Build the transaction for a parsed document. +fn into_txn( + parsed: RdfText, + txn_type: TxnType, + placement: Placement<'_>, + sync: bool, + mut opts: TxnOpts, +) -> Result<(Txn, RdfTextSummary)> { + let RdfText { parts, statements } = parsed; let summary = RdfTextSummary { content_id: parts.content_id, statements, @@ -118,18 +151,9 @@ pub fn parse_rdf_text( placement: Placement<'_>, ns: &mut NamespaceRegistry, ) -> Result { - let mut sink = TemplateSink::new(ns); - let target_scope = |sink_target: Option<&str>| match sink_target { - Some(iri) => Scope::Named(Arc::from(iri)), - None => Scope::Default, - }; - let default_scope = match placement { - Placement::AsWritten => Scope::Default, - Placement::Into(target) => target_scope(target), - }; - if !might_contain_graph_block(text) { - sink.set_scope(default_scope); + let mut sink = TemplateSink::new(ns); + sink.set_scope(default_scope(placement)); fluree_graph_turtle::parse(text, &mut sink)?; let statements = sink.statements(); return Ok(RdfText { @@ -137,8 +161,26 @@ pub fn parse_rdf_text( statements, }); } - let located = locate(text)?; + parse_located(&located, placement, ns) +} + +/// The scope a document's default-graph statements land in. +fn default_scope(placement: Placement<'_>) -> Scope { + match placement { + Placement::AsWritten | Placement::Into(None) => Scope::Default, + Placement::Into(Some(iri)) => Scope::Named(Arc::from(iri)), + } +} + +/// Parse a located document segment by segment, in document order. +fn parse_located( + located: &Located<'_>, + placement: Placement<'_>, + ns: &mut NamespaceRegistry, +) -> Result { + let mut sink = TemplateSink::new(ns); + let default_scope = default_scope(placement); let mut prefixes: Vec<(String, String)> = Vec::new(); let mut base: Option = None; let mut default_statements = 0usize; From 8501b564c26e104ef6a4b8849c1ff20d83f48b3e Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 07:41:08 -0400 Subject: [PATCH 27/36] fix(transact): a ledger id in a GRAPH block is stored as written Cross-ledger configuration names its model ledger by id, as an IRI reference in the config graph's block (`f:ledger `, the form docs/security/cross-ledger-policy.md uses), and an id with a `/` before its `:` is a relative reference. The TriG block reader kept such a reference as written when no `@base` was in force; the one parse refused it, which broke every TriG cross-ledger config. A new `ParserOptions::relative_iris` (default `Resolve`, so every other caller stays strict) lets the RDF-text driver keep a relative reference as written (`Verbatim`) in a labeled block's contents when no base is in force: upsert and TriG insert as before, and sync bodies now alike. The default graph and block labels still resolve against the base and are refused without one, and a base in force still resolves a reference in a block. The parity test also syncs its fixture: sync stores what insert stores. --- docs/transactions/turtle.md | 2 +- fluree-db-api/tests/it_rdf_text_parity.rs | 87 ++++++++++++++++++-- fluree-db-transact/src/parse/rdf_text/mod.rs | 51 ++++++++++-- fluree-graph-turtle/src/lib.rs | 2 +- fluree-graph-turtle/src/options.rs | 24 ++++++ fluree-graph-turtle/src/parser.rs | 6 +- 6 files changed, 155 insertions(+), 17 deletions(-) diff --git a/docs/transactions/turtle.md b/docs/transactions/turtle.md index def3f6156b..9d9ab3fd24 100644 --- a/docs/transactions/turtle.md +++ b/docs/transactions/turtle.md @@ -628,7 +628,7 @@ A `GRAPH` block's contents are Turtle like the rest of the document: anonymous b Every transaction endpoint reads Turtle and TriG with the same parser, once, and stores what it reads; nothing is converted to JSON-LD on the way. So the same document stores the same facts through `/insert`, `/upsert`, `/sync` and the Graph Store routes: - **Directives apply in document order.** A `@prefix` or `@base` applies to what follows it, in the default graph and in blocks alike; redefining a prefix later does not change the IRIs before it. A block label is resolved under the declarations in force where the block appears. -- **Relative IRIs need a base.** `` resolves against the `@base` in force; with no base it is a parse error, inside a block or outside one. `<#txn-meta>` with no base in force names the commit-metadata block, as always. +- **Relative IRIs resolve against the base.** `` resolves against the `@base` in force. With no base, a relative reference in the default graph (or as a block label) is a parse error, while one inside a `GRAPH` block's contents is kept exactly as written: ledger configuration names a ledger by its id, and an id such as `org/governance:main` is a relative reference (`f:ledger `). `<#txn-meta>` with no base in force names the commit-metadata block, as always. - **Literals are stored as written.** A lexical form that does not fit its datatype (`"abc"^^xsd:integer`) is kept, with its declared datatype, on every endpoint, as insert always did. - **Any IRI scheme works.** `tag:`, `kb:` and other schemes are stored verbatim, however they are spelled. - **Collections keep their order and repeats**, and `rdf:type` takes a blank node or a literal like any other predicate. diff --git a/fluree-db-api/tests/it_rdf_text_parity.rs b/fluree-db-api/tests/it_rdf_text_parity.rs index d2cce1fb64..c5cab481f7 100644 --- a/fluree-db-api/tests/it_rdf_text_parity.rs +++ b/fluree-db-api/tests/it_rdf_text_parity.rs @@ -537,11 +537,12 @@ ex:s ex:knows ex:o ~ ex:r {| ex:src "wiki" |} . ex:s a ex:T {| ex:conf 0.9 |} . "#; -/// One document, inserted by the streaming parser and upserted into a -/// fresh ledger, stores the same facts: same values, datatypes, language -/// tags, list positions and annotation bundles, blank nodes by label. +/// One document, inserted by the streaming parser, upserted, and synced +/// into the default graph of fresh ledgers, stores the same facts: same +/// values, datatypes, language tags, list positions and annotation +/// bundles, blank nodes by label. #[tokio::test] -async fn upsert_stores_what_insert_stores() { +async fn upsert_and_sync_store_what_insert_stores() { let fluree = memory(); let inserted = fluree .insert_turtle( @@ -559,6 +560,13 @@ async fn upsert_stores_what_insert_stores() { .await .expect("upsert") .ledger; + let synced = fluree + .stage_owned(genesis_ledger(&fluree, "it/rdf-parity-sync:main")) + .sync_graph_payload(GraphSel::Default, GraphPayload::Rdf(PARITY_DOC), false) + .execute() + .await + .expect("sync") + .ledger; let expected = facts(&inserted, None).await; assert!( expected @@ -566,7 +574,8 @@ async fn upsert_stores_what_insert_stores() { .any(|f| f.contains("list") && f.ends_with("#2")), "the fixture must hold a list: {expected:#?}" ); - assert_eq!(facts(&upserted, None).await, expected); + assert_eq!(facts(&upserted, None).await, expected, "upsert"); + assert_eq!(facts(&synced, None).await, expected, "sync"); } /// A collection upserted over another replaces it entry by entry, in order. @@ -633,6 +642,74 @@ async fn the_config_recipe_upserts_as_written() { ); } +/// A ledger id used as an IRI reference inside a GRAPH block, with no `@base` +/// in force, is stored exactly as written on upsert, TriG insert and sync: +/// cross-ledger configuration names its model ledger this way +/// (`f:ledger `, a relative reference). The same +/// statement in the default graph still needs a base, and a base in force +/// still resolves the reference. +#[tokio::test] +async fn a_ledger_id_in_a_graph_block_is_stored_as_written() { + let fluree = memory(); + let g = "http://example.org/cfg"; + let statement = " ."; + let block = format!("GRAPH <{g}> {{ {statement} }}\n"); + for lane in ["upsert", "insert", "sync"] { + let ledger = genesis_ledger(&fluree, &format!("it/rdf-ledger-id-{lane}:main")); + let result = match lane { + "upsert" => fluree.upsert_turtle(ledger, &block).await.map(|r| r.ledger), + "insert" => fluree.insert_turtle(ledger, &block).await.map(|r| r.ledger), + _ => fluree + .stage_owned(ledger) + .sync_graph_payload( + GraphSel::Graph(g.to_string()), + GraphPayload::Rdf(&block), + false, + ) + .execute() + .await + .map(|r| r.ledger), + } + .unwrap_or_else(|e| panic!("{lane}: {e}")); + let stored = facts(&result, Some(g)).await; + assert!( + stored + .iter() + .any(|f| f.contains("#ledger org/governance:main ")), + "{lane}: {stored:#?}" + ); + } + + let err = fluree + .upsert_turtle( + genesis_ledger(&fluree, "it/rdf-ledger-id-default:main"), + statement, + ) + .await + .map(|_| ()) + .expect_err("a default-graph relative reference needs a base"); + assert!( + matches!(err, fluree_db_api::ApiError::Turtle(_)) && err.to_string().contains("relative"), + "{err:?}" + ); + + let based = fluree + .upsert_turtle( + genesis_ledger(&fluree, "it/rdf-ledger-id-based:main"), + &format!("@base .\n{block}"), + ) + .await + .expect("upsert under a base") + .ledger; + let stored = facts(&based, Some(g)).await; + assert!( + stored + .iter() + .any(|f| f.contains("#ledger http://b.org/org/governance:main ")), + "{stored:#?}" + ); +} + // ============================================================================= // Identity: the same document addresses the same blank nodes // ============================================================================= diff --git a/fluree-db-transact/src/parse/rdf_text/mod.rs b/fluree-db-transact/src/parse/rdf_text/mod.rs index 609baf5129..814b5ee994 100644 --- a/fluree-db-transact/src/parse/rdf_text/mod.rs +++ b/fluree-db-transact/src/parse/rdf_text/mod.rs @@ -23,7 +23,7 @@ use crate::namespace::NamespaceRegistry; use crate::parse::trig_meta::{might_contain_graph_block, TXN_META_GRAPH_IRI}; use crate::template_sink::{RdfTextParts, Scope, TemplateSink}; use fluree_graph_ir::{Datatype, GraphSink, LiteralValue, SinkResult, TermId}; -use fluree_graph_turtle::{ParserOptions, TurtleError}; +use fluree_graph_turtle::{ParserOptions, RelativeIris, TurtleError}; use std::sync::Arc; mod locate; @@ -174,6 +174,13 @@ fn default_scope(placement: Placement<'_>) -> Scope { } /// Parse a located document segment by segment, in document order. +/// +/// A labeled block's contents keep a relative IRI reference exactly as +/// written when no `@base` is in force ([`RelativeIris::Verbatim`]): ledger +/// configuration names a ledger by its id, and an id such as +/// `org/governance:main` is a relative reference (`f:ledger +/// `). Default-graph statements resolve against the +/// base as everywhere else, and are refused without one. fn parse_located( located: &Located<'_>, placement: Placement<'_>, @@ -216,6 +223,12 @@ fn parse_located( let slice = &located.text[segment.range.clone()]; let seeded_prefixes = prefixes.clone(); let seeded_base = base.clone(); + let options = match segment.kind { + SegmentKind::Named { .. } => { + ParserOptions::default().with_relative_iris(RelativeIris::Verbatim) + } + SegmentKind::Default | SegmentKind::DefaultBlock => ParserOptions::default(), + }; let mut capture = Capture { sink: &mut sink, prefixes: &mut prefixes, @@ -226,7 +239,7 @@ fn parse_located( &mut capture, &seeded_prefixes, seeded_base.as_deref(), - ParserOptions::default(), + options, ) .map_err(|e| shift(e, segment.range.start))?; if matches!( @@ -550,22 +563,42 @@ mod tests { } #[test] - fn relative_references_need_a_base() { - let e = err( - "GRAPH { 1 . }", - Placement::AsWritten, - ); - assert!(e.contains("relative"), "{e}"); + fn relative_references_need_a_base_outside_block_contents() { + // A block label and a default-graph statement resolve against the + // base, and are refused without one. let e = err( "GRAPH { 1 . }", Placement::AsWritten, ); assert!(e.contains("graph label"), "{e}"); - // Outside a block, the same rule. let e = err(" 1 .", Placement::AsWritten); assert!(e.contains("relative"), "{e}"); } + /// Inside a block, with no base in force, a relative reference is kept + /// as written: ledger configuration names ledgers by id this way. A base, + /// when in force, still resolves it. + #[test] + fn a_ledger_id_in_a_block_is_kept_as_written() { + let doc = "@prefix f: .\n\ + GRAPH { f:ledger . }\n"; + assert!( + rendered(doc) + .iter() + .any(|t| t.ends_with("")), + "{:?}", + rendered(doc) + ); + let based = format!("@base .\n{doc}"); + assert!( + rendered(&based) + .iter() + .any(|t| t.ends_with("")), + "{:?}", + rendered(&based) + ); + } + // ---- Block contents: the full grammar ------------------------------- #[test] diff --git a/fluree-graph-turtle/src/lib.rs b/fluree-graph-turtle/src/lib.rs index e8f10d40ce..a37a19da43 100644 --- a/fluree-graph-turtle/src/lib.rs +++ b/fluree-graph-turtle/src/lib.rs @@ -41,7 +41,7 @@ pub mod splitter; pub use adapter::graph_to_transaction_json; pub use error::{Result, TurtleError}; pub use lex::{tokenize, Lexer, StreamingLexer, Token, TokenKind}; -pub use options::{CollectionStyle, NumericStyle, ParserOptions}; +pub use options::{CollectionStyle, NumericStyle, ParserOptions, RelativeIris}; pub use parser::{ parse, parse_with_options, parse_with_prefixes_base, parse_with_prefixes_base_options, }; diff --git a/fluree-graph-turtle/src/options.rs b/fluree-graph-turtle/src/options.rs index e57968b5ce..62bb9bee8a 100644 --- a/fluree-graph-turtle/src/options.rs +++ b/fluree-graph-turtle/src/options.rs @@ -61,6 +61,21 @@ pub enum NumericStyle { PreserveLexical, } +/// What a relative IRI reference means when no `@base` is in force. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum RelativeIris { + /// Resolve against the base; with no base, a relative reference is a + /// parse error (RFC 3986 §5.1): it cannot identify a node on its own. + #[default] + Resolve, + /// With no base in force, keep a relative reference exactly as written. + /// Fluree names a ledger by its id, and an id such as + /// `org/governance:main` is a relative reference (the `/` comes before + /// the `:`), so ledger configuration (`f:ledger `) + /// writes one as an IRI. A base, when in force, still resolves. + Verbatim, +} + /// Conformance knobs for the Turtle parser. /// /// The default is today's ingest behavior in every field; opting in is @@ -72,6 +87,8 @@ pub struct ParserOptions { pub collections: CollectionStyle, /// How numeric literals reach the sink. pub numerics: NumericStyle, + /// What a relative IRI reference means when no base is in force. + pub relative_iris: RelativeIris, } impl ParserOptions { @@ -87,6 +104,7 @@ impl ParserOptions { Self { collections: CollectionStyle::Spine, numerics: NumericStyle::PreserveLexical, + relative_iris: RelativeIris::Resolve, } } @@ -101,6 +119,12 @@ impl ParserOptions { self.numerics = numerics; self } + + /// Set what a relative IRI reference means with no base in force. + pub fn with_relative_iris(mut self, relative_iris: RelativeIris) -> Self { + self.relative_iris = relative_iris; + self + } } #[cfg(test)] diff --git a/fluree-graph-turtle/src/parser.rs b/fluree-graph-turtle/src/parser.rs index 54537693af..4c8a027570 100644 --- a/fluree-graph-turtle/src/parser.rs +++ b/fluree-graph-turtle/src/parser.rs @@ -13,7 +13,7 @@ use rustc_hash::FxHashMap; use crate::error::{Result, TurtleError}; use crate::lex::{StreamingLexer, Token, TokenKind}; -use crate::options::{CollectionStyle, NumericStyle, ParserOptions}; +use crate::options::{CollectionStyle, NumericStyle, ParserOptions, RelativeIris}; /// RDF well-known IRIs (imported from vocab crate) const RDF_TYPE: &str = rdf::TYPE; @@ -1644,6 +1644,10 @@ impl<'a, 'input, S: GraphSink> Parser<'a, 'input, S> { let base = match &self.base { Some(b) => b, + // A ledger id used as an IRI reference (see `RelativeIris`). + None if self.options.relative_iris == RelativeIris::Verbatim => { + return Ok(reference.to_string()); + } None => { // RFC 3986 §5.1: a relative reference needs a base. Keeping it // verbatim, as this used to, writes a relative IRI into the From 7226c7806aa64ee43293a71e7862de5b84c24f62 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 08:09:20 -0400 Subject: [PATCH 28/36] fix(transact): a relative GRAPH block label is kept as written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With no `@base` in force, a block's label keeps a relative reference exactly as written, as the block's contents do and as the TriG reader always did: `GRAPH { … }` names the graph `cfg/local`. A ledger id is a relative reference, so a label that names a ledger reaches staging as written, and what it denotes is decided there with every other write graph name. The default graph still refuses a relative reference without a base, and a base in force still resolves label and contents. --- docs/transactions/turtle.md | 2 +- fluree-db-api/tests/it_rdf_text_parity.rs | 66 +++++++++--------- fluree-db-transact/src/parse/rdf_text/mod.rs | 70 ++++++++++---------- 3 files changed, 72 insertions(+), 66 deletions(-) diff --git a/docs/transactions/turtle.md b/docs/transactions/turtle.md index 9d9ab3fd24..c87753433b 100644 --- a/docs/transactions/turtle.md +++ b/docs/transactions/turtle.md @@ -628,7 +628,7 @@ A `GRAPH` block's contents are Turtle like the rest of the document: anonymous b Every transaction endpoint reads Turtle and TriG with the same parser, once, and stores what it reads; nothing is converted to JSON-LD on the way. So the same document stores the same facts through `/insert`, `/upsert`, `/sync` and the Graph Store routes: - **Directives apply in document order.** A `@prefix` or `@base` applies to what follows it, in the default graph and in blocks alike; redefining a prefix later does not change the IRIs before it. A block label is resolved under the declarations in force where the block appears. -- **Relative IRIs resolve against the base.** `` resolves against the `@base` in force. With no base, a relative reference in the default graph (or as a block label) is a parse error, while one inside a `GRAPH` block's contents is kept exactly as written: ledger configuration names a ledger by its id, and an id such as `org/governance:main` is a relative reference (`f:ledger `). `<#txn-meta>` with no base in force names the commit-metadata block, as always. +- **Relative IRIs resolve against the base.** `` resolves against the `@base` in force. With no base, a relative reference in the default graph is a parse error, while one in a `GRAPH` block, as its label or in its contents, is kept exactly as written: ledger configuration names a ledger by its id, and an id such as `org/governance:main` is a relative reference (`f:ledger `). `<#txn-meta>` with no base in force names the commit-metadata block, as always. - **Literals are stored as written.** A lexical form that does not fit its datatype (`"abc"^^xsd:integer`) is kept, with its declared datatype, on every endpoint, as insert always did. - **Any IRI scheme works.** `tag:`, `kb:` and other schemes are stored verbatim, however they are spelled. - **Collections keep their order and repeats**, and `rdf:type` takes a blank node or a literal like any other predicate. diff --git a/fluree-db-api/tests/it_rdf_text_parity.rs b/fluree-db-api/tests/it_rdf_text_parity.rs index c5cab481f7..caadfb14b1 100644 --- a/fluree-db-api/tests/it_rdf_text_parity.rs +++ b/fluree-db-api/tests/it_rdf_text_parity.rs @@ -645,39 +645,45 @@ async fn the_config_recipe_upserts_as_written() { /// A ledger id used as an IRI reference inside a GRAPH block, with no `@base` /// in force, is stored exactly as written on upsert, TriG insert and sync: /// cross-ledger configuration names its model ledger this way -/// (`f:ledger `, a relative reference). The same -/// statement in the default graph still needs a base, and a base in force -/// still resolves the reference. +/// (`f:ledger `, a relative reference). A relative block +/// label is kept as written too (a sync target must be an absolute IRI, so +/// that case runs on upsert and insert). The same statement in the default +/// graph still needs a base, and a base in force still resolves both. #[tokio::test] async fn a_ledger_id_in_a_graph_block_is_stored_as_written() { let fluree = memory(); - let g = "http://example.org/cfg"; let statement = " ."; - let block = format!("GRAPH <{g}> {{ {statement} }}\n"); - for lane in ["upsert", "insert", "sync"] { - let ledger = genesis_ledger(&fluree, &format!("it/rdf-ledger-id-{lane}:main")); - let result = match lane { - "upsert" => fluree.upsert_turtle(ledger, &block).await.map(|r| r.ledger), - "insert" => fluree.insert_turtle(ledger, &block).await.map(|r| r.ledger), - _ => fluree - .stage_owned(ledger) - .sync_graph_payload( - GraphSel::Graph(g.to_string()), - GraphPayload::Rdf(&block), - false, - ) - .execute() - .await - .map(|r| r.ledger), + let cases: [(&str, &[&str]); 2] = [ + ("http://example.org/cfg", &["upsert", "insert", "sync"]), + ("cfg/local", &["upsert", "insert"]), + ]; + for (i, (g, lanes)) in cases.into_iter().enumerate() { + let block = format!("GRAPH <{g}> {{ {statement} }}\n"); + for &lane in lanes { + let ledger = genesis_ledger(&fluree, &format!("it/rdf-ledger-id-{lane}-{i}:main")); + let result = match lane { + "upsert" => fluree.upsert_turtle(ledger, &block).await.map(|r| r.ledger), + "insert" => fluree.insert_turtle(ledger, &block).await.map(|r| r.ledger), + _ => fluree + .stage_owned(ledger) + .sync_graph_payload( + GraphSel::Graph(g.to_string()), + GraphPayload::Rdf(&block), + false, + ) + .execute() + .await + .map(|r| r.ledger), + } + .unwrap_or_else(|e| panic!("{lane} <{g}>: {e}")); + let stored = facts(&result, Some(g)).await; + assert!( + stored + .iter() + .any(|f| f.contains("#ledger org/governance:main ")), + "{lane} <{g}>: {stored:#?}" + ); } - .unwrap_or_else(|e| panic!("{lane}: {e}")); - let stored = facts(&result, Some(g)).await; - assert!( - stored - .iter() - .any(|f| f.contains("#ledger org/governance:main ")), - "{lane}: {stored:#?}" - ); } let err = fluree @@ -696,12 +702,12 @@ async fn a_ledger_id_in_a_graph_block_is_stored_as_written() { let based = fluree .upsert_turtle( genesis_ledger(&fluree, "it/rdf-ledger-id-based:main"), - &format!("@base .\n{block}"), + &format!("@base .\nGRAPH {{ {statement} }}\n"), ) .await .expect("upsert under a base") .ledger; - let stored = facts(&based, Some(g)).await; + let stored = facts(&based, Some("http://b.org/cfg/local")).await; assert!( stored .iter() diff --git a/fluree-db-transact/src/parse/rdf_text/mod.rs b/fluree-db-transact/src/parse/rdf_text/mod.rs index 814b5ee994..0402642239 100644 --- a/fluree-db-transact/src/parse/rdf_text/mod.rs +++ b/fluree-db-transact/src/parse/rdf_text/mod.rs @@ -175,12 +175,13 @@ fn default_scope(placement: Placement<'_>) -> Scope { /// Parse a located document segment by segment, in document order. /// -/// A labeled block's contents keep a relative IRI reference exactly as -/// written when no `@base` is in force ([`RelativeIris::Verbatim`]): ledger -/// configuration names a ledger by its id, and an id such as -/// `org/governance:main` is a relative reference (`f:ledger -/// `). Default-graph statements resolve against the -/// base as everywhere else, and are refused without one. +/// A labeled block keeps a relative IRI reference exactly as written when no +/// `@base` is in force ([`RelativeIris::Verbatim`]), in its label and in its +/// contents: ledger configuration names a ledger by its id, and an id such +/// as `org/governance:main` is a relative reference (`f:ledger +/// `). What a graph name denotes is decided where +/// staging resolves write graphs, not here. Default-graph statements resolve +/// against the base as everywhere else, and are refused without one. fn parse_located( located: &Located<'_>, placement: Placement<'_>, @@ -288,8 +289,9 @@ fn shift(e: TurtleError, offset: usize) -> TransactError { } /// Resolve a block label with the parser, under the declarations in force -/// where the block appears: exactly the expansion a subject IRI gets there -/// (prefixed-name unescaping, relative references against the base). +/// where the block appears: exactly the expansion an IRI in the block's +/// contents gets (prefixed-name unescaping, relative references against the +/// base, kept as written with no base). /// /// `<#txn-meta>` with no base in force is the txn-meta block, as it always /// was; under a base it resolves like any relative reference. @@ -309,7 +311,7 @@ fn resolve_label( &mut sink, prefixes, base, - ParserOptions::default(), + ParserOptions::default().with_relative_iris(RelativeIris::Verbatim), ) .map_err(|e| { let message = match e { @@ -563,39 +565,37 @@ mod tests { } #[test] - fn relative_references_need_a_base_outside_block_contents() { - // A block label and a default-graph statement resolve against the - // base, and are refused without one. - let e = err( - "GRAPH { 1 . }", - Placement::AsWritten, - ); - assert!(e.contains("graph label"), "{e}"); - let e = err(" 1 .", Placement::AsWritten); - assert!(e.contains("relative"), "{e}"); + fn relative_references_need_a_base_in_the_default_graph() { + // Before a block, after one, and in an anonymous default block. + for doc in [ + " 1 .", + "GRAPH { 1 . }\n 1 .", + "{ 1 . }", + ] { + let e = err(doc, Placement::AsWritten); + assert!(e.contains("relative"), "{doc}: {e}"); + } } - /// Inside a block, with no base in force, a relative reference is kept - /// as written: ledger configuration names ledgers by id this way. A base, - /// when in force, still resolves it. + /// In a block, with no base in force, a relative reference is kept as + /// written, as the block's label and in its contents: ledger + /// configuration names ledgers by id this way. A base, when in force, + /// still resolves both. #[test] fn a_ledger_id_in_a_block_is_kept_as_written() { let doc = "@prefix f: .\n\ - GRAPH { f:ledger . }\n"; - assert!( - rendered(doc) - .iter() - .any(|t| t.ends_with("")), - "{:?}", - rendered(doc) + GRAPH { f:ledger . }\n"; + assert_eq!( + rendered(doc), + ["[cfg/local] "] ); let based = format!("@base .\n{doc}"); - assert!( - rendered(&based) - .iter() - .any(|t| t.ends_with("")), - "{:?}", - rendered(&based) + assert_eq!( + rendered(&based), + [ + "[http://b.org/cfg/local] \ + " + ] ); } From aa45a4aa6b37fda05f96d183dc9e1523ebe36e70 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 13:57:18 -0400 Subject: [PATCH 29/36] perf(transact): a TriG document is lexed once, and its blocks share one parser The driver used to start a fresh parser for every segment of a TriG document. Each one was seeded from a copy of every prefix declared so far, and each block label was resolved by its own probe parse. Each segment was also lexed a second time, after the locator had lexed the whole document to find the blocks. A document of many small blocks paid all of it per block: 100k one-triple blocks under 20 prefixes ingested 1.6 to 1.9 times slower than the phase-1 reader. fluree-graph-turtle gains SegmentParser, one parser over the pieces of a document, fed the tokens the locator lexed. The parser is generic over its token source, a streaming lexer by default, so the lexer path compiles as before. Declarations, the base, the term caches and the sink carry from piece to piece as they do between statements, the relative-IRI mode is set per piece, and error positions are the document's. The locator hands over each segment's tokens instead of an edited copy of the text, so nothing is lexed twice or copied per block. The driver resolves each distinct label once per declaration epoch, which a prefix or base declaration advances, so a label under a redefined prefix still names the new graph. --- .../src/parse/rdf_text/locate.rs | 88 ++++----- fluree-db-transact/src/parse/rdf_text/mod.rs | 173 ++++++++++-------- fluree-graph-turtle/src/lib.rs | 1 + fluree-graph-turtle/src/parser.rs | 167 ++++++++++++++++- 4 files changed, 310 insertions(+), 119 deletions(-) diff --git a/fluree-db-transact/src/parse/rdf_text/locate.rs b/fluree-db-transact/src/parse/rdf_text/locate.rs index 8ca930c551..ee4d853d92 100644 --- a/fluree-db-transact/src/parse/rdf_text/locate.rs +++ b/fluree-db-transact/src/parse/rdf_text/locate.rs @@ -2,15 +2,13 @@ //! //! It never interprets a term, so it cannot disagree with the parser about //! what the text means. It finds each block's label (as written) and extent, -//! blanks the block syntax so every segment is plain Turtle at its original -//! byte offsets, and names the construct it refuses: a nested block, a -//! directive inside a block, a blank-node label, an unclosed block, a stray -//! `}`. +//! hands the parser each segment's tokens (the document is lexed once, here), +//! and names the construct it refuses: a nested block, a directive inside a +//! block, a blank-node label, an unclosed block, a stray `}`. use crate::error::{Result, TransactError}; use crate::parse::trig_meta::might_contain_graph_block; use fluree_graph_turtle::{tokenize, Token, TokenKind, TurtleError}; -use std::borrow::Cow; use std::ops::Range; /// A stretch of the document the parser reads in one call. @@ -28,17 +26,22 @@ pub(super) enum SegmentKind { #[derive(Clone, Debug)] pub(super) struct Segment { pub(super) kind: SegmentKind, + /// The segment's bytes. A statement the segment cuts short is reported + /// at its end. pub(super) range: Range, + /// The segment's tokens, in [`Located::tokens`]. + pub(super) tokens: Range, } /// A located document. #[derive(Debug)] pub(super) struct Located<'a> { - /// The document with each block's header (`GRAPH {`) blanked and its - /// `}` blanked, or turned into the `.` a block's last statement may - /// omit. Every edit keeps the length, so byte offsets are the - /// document's. - pub(super) text: Cow<'a, str>, + /// The document. + pub(super) text: &'a str, + /// The document's tokens. A block's header (`GRAPH {`) and its `}` + /// are in no segment, except that the `}` of a block whose last statement + /// omits its `.` is that `.`. + pub(super) tokens: Vec, pub(super) segments: Vec, } @@ -87,11 +90,13 @@ fn is_directive(kind: &TokenKind) -> bool { /// Find a TriG document's segments by reading tokens only. pub(super) fn locate(input: &str) -> Result> { - let tokens = tokenize(input)?; + let mut tokens = tokenize(input)?; let kind_at = |i: usize| tokens.get(i).map(|t: &Token| &t.kind); let mut segments = Vec::new(); - let mut edits: Vec<(Range, Option)> = Vec::new(); + // The `}` of each block whose last statement omits its `.`. + let mut dots: Vec = Vec::new(); let mut default_start = 0usize; + let mut default_tokens = 0usize; let mut i = 0usize; while i < tokens.len() { @@ -154,12 +159,17 @@ pub(super) fn locate(input: &str) -> Result> { segments.push(Segment { kind: SegmentKind::Default, range: default_start..start, + tokens: default_tokens..i, }); } let content_start = tokens[brace].end as usize; - let (close, needs_dot, next) = scan_block(&tokens, brace + 1)?; - edits.push((start..content_start, None)); - edits.push((close..close + 1, needs_dot.then_some(b'.'))); + let (close_at, needs_dot) = scan_block(&tokens, brace + 1)?; + let close = tokens[close_at].start as usize; + let mut block_tokens = brace + 1..close_at; + if needs_dot { + dots.push(close_at); + block_tokens.end += 1; + } let kind = match label_at { Some(l) => SegmentKind::Named { label: input[tokens[l].start as usize..tokens[l].end as usize].to_string(), @@ -170,39 +180,34 @@ pub(super) fn locate(input: &str) -> Result> { segments.push(Segment { kind, range: content_start..close + 1, + tokens: block_tokens, }); default_start = close + 1; - i = next; + default_tokens = close_at + 1; + i = close_at + 1; } } } if default_start < input.len() { + let end = tokens + .iter() + .rposition(|t| !matches!(t.kind, TokenKind::Eof)) + .map_or(0, |last| last + 1); segments.push(Segment { kind: SegmentKind::Default, range: default_start..input.len(), + tokens: default_tokens..end.max(default_tokens), }); } - - let text = if edits.is_empty() { - Cow::Borrowed(input) - } else { - let mut bytes = input.as_bytes().to_vec(); - for (range, replacement) in edits { - match replacement { - Some(b) => bytes[range.start] = b, - None => { - for b in &mut bytes[range] { - if !matches!(*b, b'\n' | b'\r') { - *b = b' '; - } - } - } - } - } - // Only ASCII bytes were replaced, by ASCII bytes. - Cow::Owned(String::from_utf8(bytes).expect("ASCII-for-ASCII edits keep UTF-8")) - }; - Ok(Located { text, segments }) + for at in dots { + let start = tokens[at].start; + tokens[at] = Token::new(TokenKind::Dot, start, start + 1); + } + Ok(Located { + text: input, + tokens, + segments, + }) } /// Skip a top-level statement starting at `i`; returns the index after its @@ -222,10 +227,9 @@ fn skip_statement(tokens: &[Token], mut i: usize) -> usize { i } -/// Scan a block's contents from `i` (just past its `{`). Returns the byte -/// offset of its `}`, whether its last statement omits the terminating dot, -/// and the token index after the `}`. -fn scan_block(tokens: &[Token], mut i: usize) -> Result<(usize, bool, usize)> { +/// Scan a block's contents from `i` (just past its `{`). Returns the index +/// of its `}` and whether its last statement omits the terminating dot. +fn scan_block(tokens: &[Token], mut i: usize) -> Result<(usize, bool)> { let mut depth = 0i32; let mut last: Option<&TokenKind> = None; while i < tokens.len() { @@ -234,7 +238,7 @@ fn scan_block(tokens: &[Token], mut i: usize) -> Result<(usize, bool, usize)> { TokenKind::Eof => break, TokenKind::RBrace if depth == 0 => { let needs_dot = last.is_some_and(|k| !matches!(k, TokenKind::Dot)); - return Ok((tok.start as usize, needs_dot, i + 1)); + return Ok((i, needs_dot)); } TokenKind::LBrace | TokenKind::KwGraph => { return Err(locate_error( diff --git a/fluree-db-transact/src/parse/rdf_text/mod.rs b/fluree-db-transact/src/parse/rdf_text/mod.rs index 0402642239..76edb0f46f 100644 --- a/fluree-db-transact/src/parse/rdf_text/mod.rs +++ b/fluree-db-transact/src/parse/rdf_text/mod.rs @@ -23,7 +23,8 @@ use crate::namespace::NamespaceRegistry; use crate::parse::trig_meta::{might_contain_graph_block, TXN_META_GRAPH_IRI}; use crate::template_sink::{RdfTextParts, Scope, TemplateSink}; use fluree_graph_ir::{Datatype, GraphSink, LiteralValue, SinkResult, TermId}; -use fluree_graph_turtle::{ParserOptions, RelativeIris, TurtleError}; +use fluree_graph_turtle::{ParserOptions, RelativeIris, SegmentParser, TurtleError}; +use std::collections::HashMap; use std::sync::Arc; mod locate; @@ -182,6 +183,12 @@ fn default_scope(placement: Placement<'_>) -> Scope { /// `). What a graph name denotes is decided where /// staging resolves write graphs, not here. Default-graph statements resolve /// against the base as everywhere else, and are refused without one. +/// +/// One parser runs over every segment, reading the tokens the locator +/// lexed, so the document is lexed once, and declarations, the base and the +/// parser's term caches carry from segment to segment with nothing copied +/// per block. Each distinct label is resolved once per declaration epoch: a +/// document of many small blocks costs what one parse does. fn parse_located( located: &Located<'_>, placement: Placement<'_>, @@ -189,65 +196,67 @@ fn parse_located( ) -> Result { let mut sink = TemplateSink::new(ns); let default_scope = default_scope(placement); - let mut prefixes: Vec<(String, String)> = Vec::new(); - let mut base: Option = None; let mut default_statements = 0usize; let mut named_blocks = 0usize; - for segment in &located.segments { - let before = sink.statements(); - match &segment.kind { - SegmentKind::Default | SegmentKind::DefaultBlock => { - sink.set_scope(default_scope.clone()); - } - SegmentKind::Named { label, at } => { - let iri = resolve_label(label, *at, &prefixes, base.as_deref())?; - if iri == TXN_META_GRAPH_IRI { - sink.set_scope(Scope::TxnMeta); - } else { - named_blocks += 1; - match placement { - Placement::AsWritten => sink.set_scope(Scope::Named(Arc::from(iri))), - Placement::Into(target) if target == Some(iri.as_str()) => { - sink.set_scope(default_scope.clone()); + // (label as written, declaration epoch) -> graph IRI. + let mut labels: HashMap<(&str, u64), Arc> = HashMap::new(); + { + let mut capture = Capture { + sink: &mut sink, + prefixes: Vec::new(), + base: None, + epoch: 0, + }; + let mut parser = SegmentParser::new(located.text, &mut capture, ParserOptions::default())?; + for segment in &located.segments { + let cap = parser.sink_mut(); + let before = cap.sink.statements(); + match &segment.kind { + SegmentKind::Default | SegmentKind::DefaultBlock => { + cap.sink.set_scope(default_scope.clone()); + } + SegmentKind::Named { label, at } => { + let iri = match labels.get(&(label.as_str(), cap.epoch)) { + Some(iri) => Arc::clone(iri), + None => { + let iri: Arc = + resolve_label(label, *at, &cap.prefixes, cap.base.as_deref())? + .into(); + labels.insert((label.as_str(), cap.epoch), Arc::clone(&iri)); + iri } - Placement::Into(target) => { - return Err(TransactError::PayloadGraphMismatch(format!( - "the request targets one graph, {}; the body also has a \ - GRAPH block for <{iri}>", - describe_target(target) - ))); + }; + if &*iri == TXN_META_GRAPH_IRI { + cap.sink.set_scope(Scope::TxnMeta); + } else { + named_blocks += 1; + match placement { + Placement::AsWritten => cap.sink.set_scope(Scope::Named(iri)), + Placement::Into(target) if target == Some(&*iri) => { + cap.sink.set_scope(default_scope.clone()); + } + Placement::Into(target) => { + return Err(TransactError::PayloadGraphMismatch(format!( + "the request targets one graph, {}; the body also has a \ + GRAPH block for <{iri}>", + describe_target(target) + ))); + } } } } } - } - let slice = &located.text[segment.range.clone()]; - let seeded_prefixes = prefixes.clone(); - let seeded_base = base.clone(); - let options = match segment.kind { - SegmentKind::Named { .. } => { - ParserOptions::default().with_relative_iris(RelativeIris::Verbatim) + parser.set_relative_iris(match segment.kind { + SegmentKind::Named { .. } => RelativeIris::Verbatim, + SegmentKind::Default | SegmentKind::DefaultBlock => RelativeIris::Resolve, + }); + parser.parse(&located.tokens[segment.tokens.clone()], segment.range.end)?; + if matches!( + segment.kind, + SegmentKind::Default | SegmentKind::DefaultBlock + ) { + default_statements += parser.sink_mut().sink.statements() - before; } - SegmentKind::Default | SegmentKind::DefaultBlock => ParserOptions::default(), - }; - let mut capture = Capture { - sink: &mut sink, - prefixes: &mut prefixes, - base: &mut base, - }; - fluree_graph_turtle::parse_with_prefixes_base_options( - slice, - &mut capture, - &seeded_prefixes, - seeded_base.as_deref(), - options, - ) - .map_err(|e| shift(e, segment.range.start))?; - if matches!( - segment.kind, - SegmentKind::Default | SegmentKind::DefaultBlock - ) { - default_statements += sink.statements() - before; } } if let Placement::Into(target) = placement { @@ -273,21 +282,6 @@ fn describe_target(target: Option<&str>) -> String { } } -/// Move a parser error's byte position from a segment into the document. -fn shift(e: TurtleError, offset: usize) -> TransactError { - TransactError::Turtle(match e { - TurtleError::Parse { position, message } => TurtleError::Parse { - position: position + offset, - message, - }, - TurtleError::Lexer { position, message } => TurtleError::Lexer { - position: position + offset, - message, - }, - other => other, - }) -} - /// Resolve a block label with the parser, under the declarations in force /// where the block appears: exactly the expansion an IRI in the block's /// contents gets (prefixed-name unescaping, relative references against the @@ -357,17 +351,21 @@ impl GraphSink for LabelProbe { } } -/// Forwards every event to the sink and records the prefix and base -/// declarations a segment makes, so the next segment is seeded with them. +/// Forwards every event to the sink and records the declarations in force, +/// for resolving block labels. struct Capture<'s, 'n> { sink: &'s mut TemplateSink<'n>, - prefixes: &'s mut Vec<(String, String)>, - base: &'s mut Option, + prefixes: Vec<(String, String)>, + base: Option, + /// Bumped by every declaration, so a label resolved under one set of + /// declarations is not reused under another. + epoch: u64, } impl GraphSink for Capture<'_, '_> { fn on_base(&mut self, base_iri: &str) { - *self.base = Some(base_iri.to_string()); + self.base = Some(base_iri.to_string()); + self.epoch += 1; self.sink.on_base(base_iri); } fn on_prefix(&mut self, prefix: &str, namespace_iri: &str) { @@ -377,6 +375,7 @@ impl GraphSink for Capture<'_, '_> { .prefixes .push((prefix.to_string(), namespace_iri.to_string())), } + self.epoch += 1; self.sink.on_prefix(prefix, namespace_iri); } fn term_iri(&mut self, iri: &str) -> TermId { @@ -564,6 +563,36 @@ mod tests { ); } + /// A label is resolved once per declaration epoch: the same label under + /// a redefined prefix or base names another graph, and repeated blocks + /// under one epoch share one. + #[test] + fn a_block_label_follows_the_declarations_in_force() { + let doc = "@prefix g: .\n\ + g:x { 1 . }\n\ + g:x { 2 . }\n\ + @prefix g: .\n\ + g:x { 3 . }\n\ + @base .\n\ + { 4 . }\n\ + @base .\n\ + { 5 . }\n"; + let graphs: Vec = rendered(doc) + .iter() + .map(|t| t.split(']').next().unwrap_or_default().to_string()) + .collect(); + assert_eq!( + graphs, + [ + "[http://a.org/x", + "[http://a.org/x", + "[http://b.org/x", + "[http://c.org/y", + "[http://d.org/y" + ] + ); + } + #[test] fn relative_references_need_a_base_in_the_default_graph() { // Before a block, after one, and in an anonymous default block. diff --git a/fluree-graph-turtle/src/lib.rs b/fluree-graph-turtle/src/lib.rs index a37a19da43..fb687fe30b 100644 --- a/fluree-graph-turtle/src/lib.rs +++ b/fluree-graph-turtle/src/lib.rs @@ -44,6 +44,7 @@ pub use lex::{tokenize, Lexer, StreamingLexer, Token, TokenKind}; pub use options::{CollectionStyle, NumericStyle, ParserOptions, RelativeIris}; pub use parser::{ parse, parse_with_options, parse_with_prefixes_base, parse_with_prefixes_base_options, + SegmentParser, }; use fluree_graph_ir::GraphCollectorSink; diff --git a/fluree-graph-turtle/src/parser.rs b/fluree-graph-turtle/src/parser.rs index 4c8a027570..dd680bca5c 100644 --- a/fluree-graph-turtle/src/parser.rs +++ b/fluree-graph-turtle/src/parser.rs @@ -21,12 +21,46 @@ const RDF_FIRST: &str = rdf::FIRST; const RDF_REST: &str = rdf::REST; const RDF_NIL: &str = rdf::NIL; +/// Where a [`Parser`] reads its tokens: a [`StreamingLexer`] over its input, +/// or tokens a caller lexed from that input already ([`SegmentParser`]). The +/// parser is generic over it, so the lexer path compiles as it always has. +pub trait TokenStream { + /// The next token, and an end-of-input token once there are none left. + fn next_token(&mut self) -> Result; +} + +impl TokenStream for StreamingLexer<'_> { + #[inline] + fn next_token(&mut self) -> Result { + StreamingLexer::next_token(self) + } +} + +/// One piece of a document's tokens, lexed already, and the byte offset its +/// end of input is reported at. +struct LexedTokens<'t> { + rest: std::slice::Iter<'t, Token>, + eof: u32, +} + +impl TokenStream for LexedTokens<'_> { + #[inline] + fn next_token(&mut self) -> Result { + Ok(self + .rest + .next() + .cloned() + .unwrap_or_else(|| Token::new(TokenKind::Eof, self.eof, self.eof))) + } +} + /// Turtle parser state. -pub struct Parser<'a, 'input, S> { +pub struct Parser<'a, 'input, S, L = StreamingLexer<'input>> { /// Source input for span extraction. input: &'input str, - /// Streaming lexer — produces tokens on demand (no Vec). - lexer: StreamingLexer<'input>, + /// Streaming lexer — produces tokens on demand (no Vec). A + /// [`SegmentParser`] reads tokens lexed already instead. + lexer: L, /// The current token (most recently lexed). current_token: Token, sink: &'a mut S, @@ -108,7 +142,20 @@ impl<'a, 'input, S: GraphSink> Parser<'a, 'input, S> { crate::error::check_input_len(input.len())?; let mut lexer = StreamingLexer::new(input); let current_token = lexer.next_token()?; + Ok(Self::with_lexer(input, sink, options, lexer, current_token)) + } +} +impl<'a, 'input, S: GraphSink, L: TokenStream> Parser<'a, 'input, S, L> { + /// A parser over `input` reading `lexer`'s tokens, positioned at + /// `current_token`. + fn with_lexer( + input: &'input str, + sink: &'a mut S, + options: ParserOptions, + lexer: L, + current_token: Token, + ) -> Self { // Pre-size caches based on input length. ~20 bytes per token on // average in Turtle, ~3 tokens per unique term → ~60 bytes per // unique term. Cap at 2M to avoid reserving hundreds of MB for @@ -119,7 +166,7 @@ impl<'a, 'input, S: GraphSink> Parser<'a, 'input, S> { let mut prefixed_term_cache = FxHashMap::default(); prefixed_term_cache.reserve(est_unique); - Ok(Self { + Self { input, lexer, current_token, @@ -142,7 +189,7 @@ impl<'a, 'input, S: GraphSink> Parser<'a, 'input, S> { options, emit_count: 0, committed_current: false, - }) + } } /// Run `f` one nesting level deeper, erroring past @@ -168,6 +215,11 @@ impl<'a, 'input, S: GraphSink> Parser<'a, 'input, S> { /// Parse the entire Turtle document. pub fn parse(mut self) -> Result<()> { + self.parse_statements() + } + + /// Parse every statement of the current input. + fn parse_statements(&mut self) -> Result<()> { let span = tracing::debug_span!( "turtle_parse_events", statement_count = tracing::field::Empty, @@ -1733,6 +1785,54 @@ pub fn parse_with_prefixes_base( parse_with_prefixes_base_options(input, sink, prefixes, base, ParserOptions::default()) } +/// One parser over the pieces of a document, parsed in order: a TriG +/// document's default-graph runs and graph-block contents, say, when a driver +/// routes each piece to its own graph. The driver lexes the document once +/// and hands each piece's tokens over, so nothing is lexed twice. Prefixes +/// and the base a piece declares apply to the pieces after it, and the term +/// caches and the sink carry over, exactly as between the statements of one +/// document; nothing is re-seeded per piece. Error positions are the +/// document's. +pub struct SegmentParser<'a, 'input, S> { + parser: Parser<'a, 'input, S, LexedTokens<'input>>, +} + +impl<'a, 'input, S: GraphSink> SegmentParser<'a, 'input, S> { + /// A parser over `input` with no prefixes and no base yet. + pub fn new(input: &'input str, sink: &'a mut S, options: ParserOptions) -> Result { + crate::error::check_input_len(input.len())?; + let lexer = LexedTokens { + rest: [].iter(), + eof: 0, + }; + let eof = Token::new(TokenKind::Eof, 0, 0); + Ok(Self { + parser: Parser::with_lexer(input, sink, options, lexer, eof), + }) + } + + /// Parse the next piece: `tokens`, lexed from the input, which end at + /// byte `end` (where a statement the piece cuts short is reported). + pub fn parse(&mut self, tokens: &'input [Token], end: usize) -> Result<()> { + self.parser.lexer = LexedTokens { + rest: tokens.iter(), + eof: end as u32, + }; + self.parser.current_token = self.parser.lexer.next_token()?; + self.parser.parse_statements() + } + + /// How relative IRI references resolve in the pieces after this call. + pub fn set_relative_iris(&mut self, relative_iris: RelativeIris) { + self.parser.options.relative_iris = relative_iris; + } + + /// The sink, between pieces. + pub fn sink_mut(&mut self) -> &mut S { + self.parser.sink + } +} + /// [`parse_with_prefixes_base`] under explicit [`ParserOptions`]. /// /// This is the full entry point — every other `parse*` function is this one @@ -1771,6 +1871,63 @@ mod tests { use super::*; use fluree_graph_ir::{Graph, GraphCollectorSink, Term}; + /// Declarations carry from one piece to the next, the relative-IRI mode + /// applies per piece, and error positions are the document's. + #[test] + fn a_segment_parser_carries_declarations_across_pieces() { + let pieces = [ + "@prefix ex: .\n", + "ex:a ex:p ex:b .\n", + "ex:a ex:p .\n", + "ex:a ex:p .\n", + "ex:a ex:p ;", + ]; + let doc = pieces.concat(); + let tokens = crate::lex::tokenize(&doc).unwrap(); + // Each piece's tokens, and the byte offset it ends at. + let piece = |i: usize| { + let start: usize = pieces[..i].iter().map(|p| p.len()).sum(); + let end = start + pieces[i].len(); + let in_piece = |t: &Token| { + !matches!(t.kind, TokenKind::Eof) && (start..end).contains(&(t.start as usize)) + }; + let first = tokens.iter().position(in_piece).unwrap(); + let count = tokens[first..].iter().take_while(|t| in_piece(t)).count(); + (&tokens[first..first + count], end) + }; + let mut sink = GraphCollectorSink::new(); + { + let mut parser = SegmentParser::new(&doc, &mut sink, ParserOptions::default()).unwrap(); + let (t, end) = piece(0); + parser.parse(t, end).unwrap(); + let (t, end) = piece(1); + parser.parse(t, end).unwrap(); + parser.set_relative_iris(RelativeIris::Verbatim); + let (t, end) = piece(2); + parser.parse(t, end).unwrap(); + parser.set_relative_iris(RelativeIris::Resolve); + let (t, end) = piece(3); + assert!(parser.parse(t, end).is_err()); + let (t, end) = piece(4); + let start = end - pieces[4].len(); + match parser.parse(t, end) { + Err(TurtleError::Parse { position, .. }) => { + assert!((start..end).contains(&position), "{position}"); + } + other => panic!("expected a positioned parse error, got {other:?}"), + } + } + let graph = sink.into_graph(); + let objects: Vec = graph + .iter() + .map(|t| match &t.o { + Term::Iri(iri) => iri.to_string(), + other => format!("{other:?}"), + }) + .collect(); + assert_eq!(objects, ["http://example.org/b", "org/x:main"]); + } + /// Counts for a document whose only prefixed names are two `e:x`. fn prefixed_cache_counts(doc: &str) -> (u64, u64) { let mut sink = GraphCollectorSink::new(); From 7cf46ea5dce99fc734dfca8c1910d9fdec655e7d Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 13:46:04 -0400 Subject: [PATCH 30/36] docs(transactions): the orphaned blank-node query has a per-graph form A TriG upsert leaves orphaned blank nodes in the graphs its blocks write, where the documented default-graph query cannot see them. turtle.md adds the same check over GRAPH ?g, and the documented-query test pins it on an edited block. --- docs/transactions/turtle.md | 12 +++++++++ fluree-db-api/tests/it_rdf_text_parity.rs | 30 ++++++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/docs/transactions/turtle.md b/docs/transactions/turtle.md index c87753433b..21aa87b601 100644 --- a/docs/transactions/turtle.md +++ b/docs/transactions/turtle.md @@ -645,6 +645,18 @@ SELECT DISTINCT ?b WHERE { } ``` +That query reads the default graph. Blank nodes a TriG upsert wrote inside `GRAPH` blocks live in those graphs; the same check, graph by graph: + +```sparql +SELECT DISTINCT ?g ?b WHERE { + GRAPH ?g { + ?b ?p ?o . + FILTER(isBlank(?b)) + FILTER NOT EXISTS { ?s ?q ?b } + } +} +``` + Review the list before deleting: a document can hold unreferenced blank-node subjects on purpose. Deleting an orphaned node leaves the nodes it referenced orphaned in turn, so repeat until the query returns nothing. ### Querying Named Graphs diff --git a/fluree-db-api/tests/it_rdf_text_parity.rs b/fluree-db-api/tests/it_rdf_text_parity.rs index caadfb14b1..bf7e6f79ec 100644 --- a/fluree-db-api/tests/it_rdf_text_parity.rs +++ b/fluree-db-api/tests/it_rdf_text_parity.rs @@ -785,9 +785,9 @@ async fn upserting_the_same_document_again_commits_nothing() { .any(|f| f.contains("note String(\"edited\")"))); } -/// The query `turtle.md` documents for blank nodes nothing references -/// finds the node an edited document left behind, and not the one that -/// replaced it. +/// The queries `turtle.md` documents for blank nodes nothing references +/// find the node an edited document left behind, and not the one that +/// replaced it: in the default graph, and graph by graph. #[tokio::test] async fn the_documented_query_lists_the_blank_nodes_an_edit_left_behind() { let fluree = memory(); @@ -814,6 +814,30 @@ async fn the_documented_query_lists_the_blank_nodes_an_edit_left_behind() { ) .await; assert_eq!(values, vec![row(&["v1"])], "the orphan is v1's node"); + + // The same edit inside a GRAPH block, found by the per-graph query. + let g = "http://example.org/g"; + let v1 = format!( + "@prefix ex: .\nGRAPH <{g}> {{ ex:A ex:part [ ex:k \"v1\" ] . }}\n" + ); + let first = fluree + .upsert_turtle(genesis_ledger(&fluree, "it/rdf-orphans-graph:main"), &v1) + .await + .expect("v1 in a graph"); + let v2 = v1.replace("\"v1\"", "\"v2\""); + let second = fluree + .upsert_turtle(first.ledger, &v2) + .await + .expect("v2 in a graph"); + let orphans = select( + &fluree, + &second.ledger, + "SELECT DISTINCT ?g ?b WHERE {\n GRAPH ?g {\n ?b ?p ?o .\n FILTER(isBlank(?b))\n \ + FILTER NOT EXISTS { ?s ?q ?b }\n }\n}", + ) + .await; + assert_eq!(orphans.len(), 1, "{orphans:?}"); + assert_eq!(orphans[0][0], g, "{orphans:?}"); } // ============================================================================= From 987a9379cbcbf8776509b510af181aa27601631e Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 13:50:29 -0400 Subject: [PATCH 31/36] fix(transact): a DELETE can name a stored ill-typed literal A Turtle write keeps an ill-typed literal ("abc"^^xsd:integer) as its lexical form with the declared datatype, and upsert and sync now store one the way insert always did. SPARQL and JSON-LD DELETE templates refused the same literal at lowering, so no DELETE could name that stored term. DELETE templates now keep an ill-typed literal as that same stored form: DELETE DATA, DELETE and DELETE WHERE templates in SPARQL, and the JSON-LD delete. A DELETE names only what is stored, so a term that is not stored still retracts nothing. INSERT templates, and JSON-LD VALUES, still refuse an ill-typed literal. --- fluree-db-api/tests/it_delete_stored_facts.rs | 86 +++++++++++++++++++ fluree-db-api/tests/it_typed_literal_index.rs | 42 +++++---- fluree-db-transact/src/lower_sparql_update.rs | 39 +++++++-- fluree-db-transact/src/parse/jsonld.rs | 35 +++++++- 4 files changed, 176 insertions(+), 26 deletions(-) diff --git a/fluree-db-api/tests/it_delete_stored_facts.rs b/fluree-db-api/tests/it_delete_stored_facts.rs index f9e70bd07a..b5c7900657 100644 --- a/fluree-db-api/tests/it_delete_stored_facts.rs +++ b/fluree-db-api/tests/it_delete_stored_facts.rs @@ -944,3 +944,89 @@ async fn a_graph_named_by_the_ledger_address_gets_what_its_templates_write() { ); } } + +// ============================================================================= +// A DELETE can name a stored ill-typed literal +// ============================================================================= + +/// A file-backed ledger holding the Turtle `ttl`, optionally indexed. +async fn seeded_turtle( + name: &str, + ttl: &str, + indexed: bool, +) -> (tempfile::TempDir, Fluree, LedgerState) { + let dir = tempfile::tempdir().expect("tempdir"); + let fluree = FlureeBuilder::file(dir.path().to_string_lossy().to_string()) + .build() + .expect("file fluree"); + let id = format!("it/delete-stored-{name}-{indexed}:main"); + let ledger = fluree.create_ledger(&id).await.expect("create"); + let ledger = fluree + .insert_turtle(ledger, ttl) + .await + .expect("seed") + .ledger; + if !indexed { + return (dir, fluree, ledger); + } + drop(ledger); + support::rebuild_and_publish_index(&fluree, &id).await; + let ledger = fluree.ledger(&id).await.expect("reload"); + (dir, fluree, ledger) +} + +/// A Turtle write keeps an ill-typed literal as its lexical form with the +/// declared datatype. A DELETE names that stored term: its template keeps the +/// lexical form as stored instead of refusing it, on SPARQL and JSON-LD. An +/// insert still refuses one. +#[tokio::test] +async fn a_delete_names_a_stored_ill_typed_literal() { + let ttl = format!( + "@prefix ex: <{EX}> .\n@prefix xsd: .\n\ + ex:s ex:n \"abc\"^^xsd:integer .\n" + ); + for indexed in [false, true] { + for surface in ["sparql", "jsonld"] { + let (_d, fluree, ledger) = + seeded_turtle(&format!("ill-typed-{surface}"), &ttl, indexed).await; + assert_eq!(facts(&fluree, &ledger).await, ["n=abc"], "precondition"); + let r = if surface == "sparql" { + sparql_update( + &fluree, + ledger, + "DELETE DATA { ex:s ex:n \"abc\"^^xsd:integer }", + ) + .await + } else { + jsonld_update( + &fluree, + ledger, + json!({"delete": {"@id": "ex:s", "ex:n": {"@value": "abc", "@type": "xsd:integer"}}}), + ) + .await + }; + assert_eq!(r.receipt.retract_count, 1, "{surface}, indexed={indexed}"); + assert!( + facts(&fluree, &r.ledger).await.is_empty(), + "{surface}, indexed={indexed}" + ); + } + } + + // An insert still refuses an ill-typed literal. + let text = format!( + "PREFIX ex: <{EX}>\nPREFIX xsd: \n\ + INSERT DATA {{ ex:s ex:n \"abc\"^^xsd:integer }}" + ); + let parsed = fluree_db_sparql::parse_sparql(&text); + let mut ns = fluree_db_transact::NamespaceRegistry::new(); + assert!( + fluree_db_transact::lower_sparql_update_ast( + &parsed.ast.expect("ast"), + &mut ns, + fluree_db_transact::TxnOpts::default(), + ) + .is_err(), + "INSERT DATA refuses an ill-typed literal" + ); +} diff --git a/fluree-db-api/tests/it_typed_literal_index.rs b/fluree-db-api/tests/it_typed_literal_index.rs index 4faeea1d38..8e6e1a8bda 100644 --- a/fluree-db-api/tests/it_typed_literal_index.rs +++ b/fluree-db-api/tests/it_typed_literal_index.rs @@ -388,25 +388,37 @@ async fn ill_typed_literals_keep_their_lexical_form() { ); } +/// INSERT DATA refuses an ill-typed literal, naming the datatype. DELETE DATA +/// names one as stored (a Turtle write keeps one as its lexical form with the +/// declared datatype), so one that is not stored retracts nothing. #[tokio::test] -async fn sparql_update_rejects_ill_typed_literal() { +async fn sparql_insert_data_rejects_an_ill_typed_literal() { let fluree = memory_fluree(); let ledger_id = "typed-literal-index:reject"; fluree.create_ledger(ledger_id).await.expect("create"); - for op in ["INSERT DATA", "DELETE DATA"] { - let sparql = format!("{op} {{ <{EX}a> <{EX}date> \"1990-00-00\"^^<{XSD}date> }}"); - let err = fluree - .graph(ledger_id) - .transact() - .sparql_update(&sparql) - .commit() - .await - .expect_err("an ill-typed xsd:date must not lower"); - assert!( - err.to_string().contains("xsd:date"), - "{op}: error should name the datatype: {err}" - ); - } + let quad = format!("<{EX}a> <{EX}date> \"1990-00-00\"^^<{XSD}date>"); + let err = fluree + .graph(ledger_id) + .transact() + .sparql_update(&format!("INSERT DATA {{ {quad} }}")) + .commit() + .await + .expect_err("an ill-typed xsd:date must not lower in INSERT DATA"); + assert!( + err.to_string().contains("xsd:date"), + "error should name the datatype: {err}" + ); + let r = fluree + .graph(ledger_id) + .transact() + .sparql_update(&format!("DELETE DATA {{ {quad} }}")) + .commit() + .await + .expect("DELETE DATA names the term as stored"); + assert_eq!( + r.receipt.flake_count, 0, + "nothing is stored, nothing is retracted" + ); } /// Commits written before the fix hold these literals as strings. They read diff --git a/fluree-db-transact/src/lower_sparql_update.rs b/fluree-db-transact/src/lower_sparql_update.rs index a7041dfacc..82d308c03d 100644 --- a/fluree-db-transact/src/lower_sparql_update.rs +++ b/fluree-db-transact/src/lower_sparql_update.rs @@ -1028,6 +1028,7 @@ fn lower_insert_data( bnodes, &mut write_graphs, None, + IllTyped::Refuse, )?; Ok(Txn { @@ -1083,6 +1084,7 @@ fn lower_delete_data( bnodes, &mut write_graphs, None, + IllTyped::AsStored, )?; Ok(Txn { @@ -1252,6 +1254,7 @@ fn lower_delete_where_with_graphs( &mut bnodes, &mut write_graphs, None, + IllTyped::AsStored, )?; Ok(Txn { @@ -1472,6 +1475,7 @@ fn lower_modify( bnodes, &mut write_graphs, default_template_graph.clone(), + IllTyped::AsStored, )? } else { Vec::new() @@ -1496,6 +1500,7 @@ fn lower_modify( bnodes, &mut write_graphs, default_template_graph.clone(), + IllTyped::Refuse, )? } else { Vec::new() @@ -1522,6 +1527,7 @@ fn lower_modify( }) } +#[allow(clippy::too_many_arguments)] fn lower_quad_pattern_to_templates( elements: &[QuadPatternElement], prologue: &Prologue, @@ -1530,12 +1536,13 @@ fn lower_quad_pattern_to_templates( bnodes: &mut BlankNodeCounter, write_graphs: &mut BTreeSet, default_graph: Option>, + ill_typed: IllTyped, ) -> Result, LowerError> { let mut out: Vec = Vec::new(); for el in elements { match el { QuadPatternElement::Triple(tp) => { - let mut t = lower_triple_to_template(tp, prologue, ns, vars, bnodes)?; + let mut t = lower_triple_to_template(tp, prologue, ns, vars, bnodes, ill_typed)?; if let Some(iri) = &default_graph { t = t.in_template_default_graph(Arc::clone(iri)); } @@ -1548,7 +1555,7 @@ fn lower_quad_pattern_to_templates( let iri: Arc = Arc::from(iri); for tp in triples { out.push( - lower_triple_to_template(tp, prologue, ns, vars, bnodes)? + lower_triple_to_template(tp, prologue, ns, vars, bnodes, ill_typed)? .in_graph(Arc::clone(&iri)), ); } @@ -1559,7 +1566,7 @@ fn lower_quad_pattern_to_templates( let graph_var = vars.get_or_insert(&format!("?{}", v.name)); for tp in triples { out.push( - lower_triple_to_template(tp, prologue, ns, vars, bnodes)? + lower_triple_to_template(tp, prologue, ns, vars, bnodes, ill_typed)? .with_graph_var(graph_var), ); } @@ -1577,6 +1584,7 @@ fn lower_triple_to_template( ns: &mut NamespaceRegistry, vars: &mut VarRegistry, bnodes: &mut BlankNodeCounter, + ill_typed: IllTyped, ) -> Result { let subject = subject_to_template(&triple.subject, prologue, ns, vars, bnodes)?; let predicate = predicate_to_template(&triple.predicate, prologue, ns, vars)?; @@ -1584,7 +1592,7 @@ fn lower_triple_to_template( // Object needs special handling for literal metadata let (object, dtc) = match &triple.object { Term::Literal(lit) => { - let result = literal_to_template(lit, prologue, ns)?; + let result = literal_to_template(lit, prologue, ns, ill_typed)?; (result.term, result.dtc) } other => (object_to_template(other, prologue, ns, vars, bnodes)?, None), @@ -1750,7 +1758,7 @@ fn lower_triple_to_delete_template_delete_where( (TemplateTerm::Sid(ns.sid_for_iri(&expanded)), None) } Term::Literal(lit) => { - let r = literal_to_template(lit, prologue, ns)?; + let r = literal_to_template(lit, prologue, ns, IllTyped::AsStored)?; (r.term, r.dtc) } Term::BlankNode(bn) => { @@ -1949,7 +1957,7 @@ fn object_to_template( Ok(TemplateTerm::Sid(ns.sid_for_iri(&expanded))) } // Literals should go through literal_to_template for metadata; this is a fallback - Term::Literal(lit) => Ok(literal_to_template(lit, prologue, ns)?.term), + Term::Literal(lit) => Ok(literal_to_template(lit, prologue, ns, IllTyped::Refuse)?.term), Term::BlankNode(bn) => { let label = match &bn.value { BlankNodeValue::Labeled(l) => { @@ -1975,11 +1983,24 @@ fn object_to_template( } } +/// How a template lowers a typed literal whose lexical form its datatype +/// does not accept (`"abc"^^xsd:integer`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum IllTyped { + /// Refuse it: an insert template writes what it names. + Refuse, + /// Keep the lexical form as a string with the declared datatype, the + /// form a Turtle write stores an ill-typed literal in. A DELETE names + /// what is stored, and a term that is not stored retracts nothing. + AsStored, +} + /// Convert SPARQL Literal to TemplateTerm with datatype/language metadata. fn literal_to_template( lit: &Literal, prologue: &Prologue, ns: &mut NamespaceRegistry, + ill_typed: IllTyped, ) -> Result { match &lit.value { SparqlLiteralValue::Simple(s) => Ok(LiteralResult { @@ -1993,7 +2014,11 @@ fn literal_to_template( SparqlLiteralValue::Typed { value, datatype } => { let dt_iri = expand_iri(datatype, prologue)?; let dt_sid = ns.sid_for_iri(&dt_iri); - let coerced = coerce_typed_flake_value(value, &dt_iri, lit.span)?; + let coerced = match coerce_typed_flake_value(value, &dt_iri, lit.span) { + Ok(coerced) => coerced, + Err(_) if ill_typed == IllTyped::AsStored => FlakeValue::String(value.to_string()), + Err(e) => return Err(e), + }; Ok(LiteralResult { term: TemplateTerm::Value(coerced), dtc: Some(DatatypeConstraint::Explicit(dt_sid)), diff --git a/fluree-db-transact/src/parse/jsonld.rs b/fluree-db-transact/src/parse/jsonld.rs index 145d13cb73..50e12e2fa1 100644 --- a/fluree-db-transact/src/parse/jsonld.rs +++ b/fluree-db-transact/src/parse/jsonld.rs @@ -496,6 +496,7 @@ fn parse_update(json: &Value, opts: TxnOpts, ns_registry: &mut NamespaceRegistry &from_named_aliases, ); ctx.default_graph_is_template_default = template_default_graph.is_some(); + ctx.ill_typed_as_stored = true; let templates = parse_update_templates_with_ctx(delete_val, &mut ctx)?; // Blank nodes are not allowed in delete templates (mirrors SPARQL 1.1 // Update §19.8 note 8 on the JSON-LD surface): a blank node denotes a @@ -865,6 +866,9 @@ struct TemplateParseCtx<'a> { default_graph_is_template_default: bool, from_named_aliases: &'a HashMap, blank_counter: usize, + /// Delete templates keep an ill-typed literal's lexical form as stored + /// (see [`coerce_value_with_datatype`]) instead of refusing it. + ill_typed_as_stored: bool, } impl<'a> TemplateParseCtx<'a> { @@ -890,6 +894,7 @@ impl<'a> TemplateParseCtx<'a> { default_graph_is_template_default: false, from_named_aliases, blank_counter: 0, + ill_typed_as_stored: false, } } @@ -1298,7 +1303,8 @@ fn parse_values_cell( } let expanded_type = expand_datatype_iri(type_val, context, strict)?; - let parsed = coerce_value_with_datatype(value_val, &expanded_type, ns_registry)?; + let parsed = + coerce_value_with_datatype(value_val, &expanded_type, ns_registry, false)?; return Ok(parsed.term); } @@ -1654,6 +1660,7 @@ fn parse_expanded_value_with_ctx( ctx.ns_registry, ctx.object_var_parsing, ctx.strict_compact_iri, + ctx.ill_typed_as_stored, ); } @@ -1792,6 +1799,7 @@ fn parse_literal_value_with_meta( ns_registry: &mut NamespaceRegistry, object_var_parsing: bool, strict: bool, + ill_typed_as_stored: bool, ) -> Result { // Check for @type first - always route through typed coercion when present if let Some(type_val) = obj.get("@type") { @@ -1829,7 +1837,12 @@ fn parse_literal_value_with_meta( }; // Route all @value types through typed coercion - return coerce_value_with_datatype(val, resolved_type, ns_registry); + return coerce_value_with_datatype( + val, + resolved_type, + ns_registry, + ill_typed_as_stored, + ); } } @@ -1893,16 +1906,30 @@ fn parse_literal_value_with_meta( /// - Boolean @value + xsd:string → ERROR /// - Numeric @value + xsd:boolean → ERROR /// - Integer subtypes enforce range bounds (e.g., xsd:byte must be -128 to 127) +/// +/// With `ill_typed_as_stored` (delete templates), a string `@value` the +/// datatype does not accept is kept as a string with the declared datatype, +/// the form a Turtle write stores an ill-typed literal in: a DELETE names +/// what is stored, and a term that is not stored retracts nothing. fn coerce_value_with_datatype( val: &Value, type_iri: &str, ns_registry: &mut NamespaceRegistry, + ill_typed_as_stored: bool, ) -> Result { let datatype_sid = ns_registry.sid_for_iri(type_iri); // Delegate to core coercion module - let flake_value = fluree_db_core::coerce::coerce_json_value(val, type_iri) - .map_err(|e| TransactError::Parse(e.message))?; + let flake_value = match ( + fluree_db_core::coerce::coerce_json_value(val, type_iri), + val, + ) { + (Ok(value), _) => value, + (Err(_), Value::String(lexical)) if ill_typed_as_stored => { + FlakeValue::String(lexical.clone()) + } + (Err(e), _) => return Err(TransactError::Parse(e.message)), + }; Ok(ParsedValue::new(TemplateTerm::Value(flake_value)) .with_dtc(DatatypeConstraint::Explicit(datatype_sid))) From fe6707663cb297783b8b767c5a2f8772a4f6f56c Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 19:03:14 -0400 Subject: [PATCH 32/36] docs(transactions): every idempotence claim names the one exception upsert.md's Idempotency section says that a value stored with less detail than it was written is the one exception to a retry-safe upsert. The page's opening line, its behavior list and its comparison table, the transactions README, and the WHERE/DELETE/INSERT comparison table still called replace mode idempotent without it. They now name the exception and point to that section, and the README's retry checklist says how to write such values. --- docs/transactions/README.md | 3 ++- docs/transactions/update-where-delete-insert.md | 2 +- docs/transactions/upsert.md | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/transactions/README.md b/docs/transactions/README.md index dc89521fe5..b1f649dead 100644 --- a/docs/transactions/README.md +++ b/docs/transactions/README.md @@ -22,7 +22,7 @@ Adding new data to the database: ### [Upsert](upsert.md) -Idempotent transactions that replace values for supplied predicates: +Transactions that replace values for supplied predicates, idempotent except for values stored with less detail than they were written: - Upsert semantics - Use cases for upsert - Idempotent operations @@ -285,6 +285,7 @@ For idempotent transactions: - Include unique identifiers - Design for retry safety - Use deterministic IRIs +- Write values at the precision they are stored with: `xsd:integer` for integers beyond 64 bits, at most six fractional digits for `dateTime` and `time` (see [Upsert](upsert.md#idempotency)) ## Best Practices diff --git a/docs/transactions/update-where-delete-insert.md b/docs/transactions/update-where-delete-insert.md index 4bd0657371..877922092e 100644 --- a/docs/transactions/update-where-delete-insert.md +++ b/docs/transactions/update-where-delete-insert.md @@ -698,7 +698,7 @@ Updates ALL entities with status="pending". | **Other properties** | Preserved | Preserved | | **Conditional** | Yes (with filters) | No | | **Pattern matching** | Yes | No | -| **Idempotent** | Depends on logic | Yes | +| **Idempotent** | Depends on logic | Yes, with one exception (see [Upsert](upsert.md#idempotency)) | | **Use case** | Partial updates | Complete replacement | ## Best Practices diff --git a/docs/transactions/upsert.md b/docs/transactions/upsert.md index 0b4d40db8a..44d9a28ba2 100644 --- a/docs/transactions/upsert.md +++ b/docs/transactions/upsert.md @@ -1,6 +1,6 @@ # Upsert -Upsert operations provide idempotent transactions by **replacing the values of the predicates you supply** for an entity (matched by `@id`). +Upsert operations **replace the values of the predicates you supply** for an entity (matched by `@id`). Repeating an upsert is idempotent, with one exception for values stored with less detail than they were written (see [Idempotency](#idempotency)). ## What is Upsert? @@ -210,7 +210,7 @@ POST /upsert?ledger=mydb:main **Behavior:** - Replaces values **for the predicates you supply** (per subject) - Leaves other predicates unchanged -- Retry-safe/idempotent for the supplied predicates +- Retry-safe/idempotent for the supplied predicates, except for values stored with less detail than they were written (see [Idempotency](#idempotency)) ## Use Cases @@ -551,10 +551,10 @@ For partial updates, use WHERE/DELETE/INSERT: |---------|--------------|--------------| | **Behavior** | Additive | Replaces each named predicate's values | | **Predicates not in the payload** | Preserved | Preserved | -| **Idempotent** | No | Yes | +| **Idempotent** | No | Yes, with one exception (see [Idempotency](#idempotency)) | | **Partial updates** | Yes (with WHERE/DELETE/INSERT) | Per predicate | | **Use case** | Adding data | Synchronization | -| **Retry safety** | Requires care | Safe by default | +| **Retry safety** | Requires care | Safe by default, with the same exception | | **Performance** | Fewer operations | More operations | ## Related Documentation From 895e6f87aea2ce4d5f7cbaa56bb74bcb27392a45 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 19:50:19 -0400 Subject: [PATCH 33/36] test(api): OPTIONAL and UNION DELETE rows take the witnessed lane on SPARQL too The routing test's doc named the SPARQL shapes but ran only JSON-LD. It now runs SPARQL OPTIONAL and UNION twins, and a SPARQL UNION whose other branch binds the object from another predicate, which stays on the resolver. It collects every mismatch before asserting, so one run shows each case. --- .../tests/it_delete_witness_routing.rs | 120 +++++++++++++----- 1 file changed, 87 insertions(+), 33 deletions(-) diff --git a/fluree-db-api/tests/it_delete_witness_routing.rs b/fluree-db-api/tests/it_delete_witness_routing.rs index 5d135a5ad8..59f6fba722 100644 --- a/fluree-db-api/tests/it_delete_witness_routing.rs +++ b/fluree-db-api/tests/it_delete_witness_routing.rs @@ -16,7 +16,7 @@ #[path = "support/span_capture.rs"] mod span_capture; -use fluree_db_api::FlureeBuilder; +use fluree_db_api::{Fluree, FlureeBuilder, LedgerState, TransactResult}; use serde_json::{json, Value as JsonValue}; fn ctx() -> JsonValue { @@ -115,6 +115,36 @@ async fn delete_witnessed_lane_fires_only_on_a_list_free_ledger() { ); } +/// An update on either surface. +enum Update { + JsonLd(JsonValue), + Sparql(&'static str), +} + +async fn apply(fluree: &Fluree, ledger: LedgerState, update: Update) -> TransactResult { + match update { + Update::JsonLd(txn) => fluree.update(ledger, &txn).await.expect("JSON-LD update"), + Update::Sparql(body) => { + let text = format!("PREFIX ex: \n{body}"); + let parsed = fluree_db_sparql::parse_sparql(&text); + assert!(!parsed.has_errors(), "{text}: {:?}", parsed.diagnostics); + let mut ns = fluree_db_transact::NamespaceRegistry::from_db(&ledger.snapshot); + let txn = fluree_db_transact::lower_sparql_update_ast( + &parsed.ast.expect("ast"), + &mut ns, + fluree_db_transact::TxnOpts::default(), + ) + .expect("lower"); + fluree + .stage_owned(ledger) + .txn(txn) + .execute() + .await + .expect("SPARQL update") + } + } +} + /// A value bound by the sole triple of an OPTIONAL, or of one UNION branch, /// is the decode of that triple's fact, so its row takes the witnessed lane: /// the JSON-LD and SPARQL shapes, and Cypher `SET` and `DETACH DELETE`, @@ -147,7 +177,10 @@ async fn optional_and_union_rows_take_the_witnessed_lane() { {"@id": "ex:b", "@type": "ex:Node", "ex:tag": "y"}, {"@id": "ex:c", "@type": "ex:Node", "ex:tag": "z"}, {"@id": "ex:d", "@type": "ex:Node", "ex:tag": "w", "ex:other": "v"}, - {"@id": "ex:e", "@type": "ex:Node", "ex:tag": "u"} + {"@id": "ex:e", "@type": "ex:Node", "ex:tag": "u"}, + {"@id": "ex:f", "@type": "ex:Node", "ex:tag": "t"}, + {"@id": "ex:g", "@type": "ex:Node", "ex:tag": "s"}, + {"@id": "ex:h", "@type": "ex:Node", "ex:tag": "r", "ex:other": "q"} ] }), ) @@ -155,54 +188,75 @@ async fn optional_and_union_rows_take_the_witnessed_lane() { .expect("seed") .ledger; - let cases: Vec<(&str, JsonValue, &str)> = vec![ + // (case, update, witnessed). Each retracts one fact either way; where + // the other UNION branch binds the object from another predicate, the + // row goes through the resolver. + let cases = [ ( "JSON-LD OPTIONAL", - json!({ + Update::JsonLd(json!({ "@context": ctx(), "where": [{"@id": "ex:a", "@type": "ex:Node"}, ["optional", {"@id": "ex:a", "ex:tag": "?o"}]], "delete": {"@id": "ex:a", "ex:tag": "?o"} - }), - "proceed", + })), + true, ), ( "JSON-LD UNION", - json!({ + Update::JsonLd(json!({ "@context": ctx(), "where": [["union", {"@id": "ex:b", "ex:tag": "?o"}, {"@id": "ex:b", "@type": "?t"}]], "delete": {"@id": "ex:b", "ex:tag": "?o"} - }), - "proceed", + })), + true, + ), + ( + "SPARQL OPTIONAL", + Update::Sparql( + "DELETE { ex:f ex:tag ?o } WHERE { ex:f a ex:Node OPTIONAL { ex:f ex:tag ?o } }", + ), + true, + ), + ( + "SPARQL UNION", + Update::Sparql( + "DELETE { ex:g ex:tag ?o } WHERE { { ex:g ex:tag ?o } UNION { ex:g a ?t } }", + ), + true, + ), + ( + "JSON-LD UNION binding the object twice", + Update::JsonLd(json!({ + "@context": ctx(), + "where": [["union", {"@id": "ex:d", "ex:tag": "?o"}, {"@id": "ex:d", "ex:other": "?o"}]], + "delete": {"@id": "ex:d", "ex:tag": "?o"} + })), + false, + ), + ( + "SPARQL UNION binding the object twice", + Update::Sparql( + "DELETE { ex:h ex:tag ?o } WHERE { { ex:h ex:tag ?o } UNION { ex:h ex:other ?o } }", + ), + false, ), ]; - for (what, txn, expected) in cases { + let mut failures = Vec::new(); + for (what, update, witnessed) in cases { let before = store.find_events("fast-path outcome").len(); - let r = fluree.update(ledger, &txn).await.expect(what); - assert_eq!(r.receipt.retract_count, 1, "{what}"); - assert_eq!(outcomes(before), [expected], "{what}"); + let r = apply(&fluree, ledger, update).await; + let stamps = outcomes(before); + let expected: &[&str] = if witnessed { &["proceed"] } else { &[] }; + if r.receipt.retract_count != 1 || stamps != expected { + failures.push(format!( + "{what}: {} retracted, stamps {stamps:?}", + r.receipt.retract_count + )); + } ledger = r.ledger; } - - // The other branch binds the object from another predicate: matched. - let before = store.find_events("fast-path outcome").len(); - let r = fluree - .update( - ledger, - &json!({ - "@context": ctx(), - "where": [["union", {"@id": "ex:d", "ex:tag": "?o"}, {"@id": "ex:d", "ex:other": "?o"}]], - "delete": {"@id": "ex:d", "ex:tag": "?o"} - }), - ) - .await - .expect("UNION binding the object twice"); - assert_eq!(r.receipt.retract_count, 1, "the delete still retracts"); - assert!( - outcomes(before).is_empty(), - "a UNION binding the object twice is not a witness" - ); - ledger = r.ledger; + assert!(failures.is_empty(), "{failures:#?}"); for (what, stmt) in [ ( From 11b5872a6f346f19b76e34f76d0a0e6987dd1dc7 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 30 Sep 2026 19:50:19 -0400 Subject: [PATCH 34/36] docs(transactions): the orphan queries count references from every graph A blank-node label names one node across a TriG document, so a node described in one graph can be referenced from another graph or from the default graph. The per-graph orphan query checked references only within the node's own graph and listed such a node, and the default-graph query missed references from named graphs. Both now look for references in every graph. A test pins a node referenced from another graph and one referenced from a named graph, and shows the own-graph checks listing them. --- docs/transactions/turtle.md | 8 ++- fluree-db-api/tests/it_rdf_text_parity.rs | 63 ++++++++++++++++++----- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/docs/transactions/turtle.md b/docs/transactions/turtle.md index 21aa87b601..c6a54f59b4 100644 --- a/docs/transactions/turtle.md +++ b/docs/transactions/turtle.md @@ -642,21 +642,25 @@ SELECT DISTINCT ?b WHERE { ?b ?p ?o . FILTER(isBlank(?b)) FILTER NOT EXISTS { ?s ?q ?b } + FILTER NOT EXISTS { GRAPH ?h { ?s2 ?q2 ?b } } } ``` -That query reads the default graph. Blank nodes a TriG upsert wrote inside `GRAPH` blocks live in those graphs; the same check, graph by graph: +That query lists subjects in the default graph. Blank nodes a TriG upsert wrote inside `GRAPH` blocks live in those graphs; the same check, graph by graph: ```sparql SELECT DISTINCT ?g ?b WHERE { GRAPH ?g { ?b ?p ?o . FILTER(isBlank(?b)) - FILTER NOT EXISTS { ?s ?q ?b } } + FILTER NOT EXISTS { ?s ?q ?b } + FILTER NOT EXISTS { GRAPH ?h { ?s2 ?q2 ?b } } } ``` +Both look for references in every graph. A blank-node label names one node across a TriG document, so a node described in one graph can be referenced from another, and a check within its own graph would list it. + Review the list before deleting: a document can hold unreferenced blank-node subjects on purpose. Deleting an orphaned node leaves the nodes it referenced orphaned in turn, so repeat until the query returns nothing. ### Querying Named Graphs diff --git a/fluree-db-api/tests/it_rdf_text_parity.rs b/fluree-db-api/tests/it_rdf_text_parity.rs index bf7e6f79ec..feeeb562f0 100644 --- a/fluree-db-api/tests/it_rdf_text_parity.rs +++ b/fluree-db-api/tests/it_rdf_text_parity.rs @@ -785,6 +785,16 @@ async fn upserting_the_same_document_again_commits_nothing() { .any(|f| f.contains("note String(\"edited\")"))); } +/// `turtle.md`'s query for blank-node subjects of the default graph that +/// nothing references, in any graph. +const ORPHANS_DEFAULT: &str = "SELECT DISTINCT ?b WHERE {\n ?b ?p ?o .\n FILTER(isBlank(?b))\n \ + FILTER NOT EXISTS { ?s ?q ?b }\n FILTER NOT EXISTS { GRAPH ?h { ?s2 ?q2 ?b } }\n}"; + +/// `turtle.md`'s query for the same, graph by graph. +const ORPHANS_PER_GRAPH: &str = "SELECT DISTINCT ?g ?b WHERE {\n GRAPH ?g {\n ?b ?p ?o .\n \ + FILTER(isBlank(?b))\n }\n FILTER NOT EXISTS { ?s ?q ?b }\n \ + FILTER NOT EXISTS { GRAPH ?h { ?s2 ?q2 ?b } }\n}"; + /// The queries `turtle.md` documents for blank nodes nothing references /// find the node an edited document left behind, and not the one that /// replaced it: in the default graph, and graph by graph. @@ -798,13 +808,7 @@ async fn the_documented_query_lists_the_blank_nodes_an_edit_left_behind() { .expect("v1"); let v2 = v1.replace("\"v1\"", "\"v2\""); let second = fluree.upsert_turtle(first.ledger, &v2).await.expect("v2"); - let orphans = select( - &fluree, - &second.ledger, - "SELECT DISTINCT ?b WHERE {\n ?b ?p ?o .\n FILTER(isBlank(?b))\n \ - FILTER NOT EXISTS { ?s ?q ?b }\n}", - ) - .await; + let orphans = select(&fluree, &second.ledger, ORPHANS_DEFAULT).await; assert_eq!(orphans.len(), 1, "{orphans:?}"); let values = select( &fluree, @@ -829,15 +833,48 @@ async fn the_documented_query_lists_the_blank_nodes_an_edit_left_behind() { .upsert_turtle(first.ledger, &v2) .await .expect("v2 in a graph"); - let orphans = select( + let orphans = select(&fluree, &second.ledger, ORPHANS_PER_GRAPH).await; + assert_eq!(orphans.len(), 1, "{orphans:?}"); + assert_eq!(orphans[0][0], g, "{orphans:?}"); +} + +/// A blank-node label names one node across a TriG document, so a node +/// described in one graph can be referenced from another graph, or from the +/// default graph. The documented queries do not list it; a check within the +/// node's own graph would. +#[tokio::test] +async fn the_documented_query_counts_references_from_other_graphs() { + let fluree = memory(); + let doc = "@prefix ex: .\n\ + _:d ex:k \"described in the default graph\" .\n\ + GRAPH { _:n ex:k \"described in g1\" . }\n\ + GRAPH { ex:A ex:part _:n , _:d . }\n"; + let r = fluree + .upsert_turtle(genesis_ledger(&fluree, "it/rdf-orphans-across:main"), doc) + .await + .expect("upsert"); + let default_orphans = select(&fluree, &r.ledger, ORPHANS_DEFAULT).await; + assert!(default_orphans.is_empty(), "{default_orphans:?}"); + let graph_orphans = select(&fluree, &r.ledger, ORPHANS_PER_GRAPH).await; + assert!(graph_orphans.is_empty(), "{graph_orphans:?}"); + + // Checked within the node's own graph only, both would be listed. + let own_graph_only = select( &fluree, - &second.ledger, - "SELECT DISTINCT ?g ?b WHERE {\n GRAPH ?g {\n ?b ?p ?o .\n FILTER(isBlank(?b))\n \ - FILTER NOT EXISTS { ?s ?q ?b }\n }\n}", + &r.ledger, + "SELECT DISTINCT ?g ?b WHERE { GRAPH ?g { ?b ?p ?o . FILTER(isBlank(?b)) \ + FILTER NOT EXISTS { ?s ?q ?b } } }", ) .await; - assert_eq!(orphans.len(), 1, "{orphans:?}"); - assert_eq!(orphans[0][0], g, "{orphans:?}"); + assert_eq!(own_graph_only.len(), 1, "{own_graph_only:?}"); + let default_only = select( + &fluree, + &r.ledger, + "SELECT DISTINCT ?b WHERE { ?b ?p ?o . FILTER(isBlank(?b)) \ + FILTER NOT EXISTS { ?s ?q ?b } }", + ) + .await; + assert_eq!(default_only.len(), 1, "{default_only:?}"); } // ============================================================================= From 99d5776c99340714619e150fdc8233b1f739dc9d Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Thu, 1 Oct 2026 06:48:13 -0400 Subject: [PATCH 35/36] fix(transact): modify policy judges a restated value the identity cannot view A write that restates a stored value as it is stages nothing: the retraction of the stored fact and the assertion of the same fact cancel before the modify check, so such a write was not checked against modify policy. Under a non-root policy, the accumulator now reports the pairs that cancel, and both sides of each pair whose stored fact the identity cannot view go to the modify check. Restating a value the identity can view still commits nothing without a check, root is unaffected, and a write with no restated value builds no view check. This covers upsert, a DELETE and INSERT of one value on any surface, and graph sync. The policy docs say how restatements are checked, and that granting modify without view does not keep a value confidential from that identity. --- docs/security/policy-in-transactions.md | 4 + fluree-db-api/tests/it_delete_stored_facts.rs | 256 ++++++++++++++++++ .../src/generate/accumulator.rs | 19 +- fluree-db-transact/src/stage.rs | 78 +++++- 4 files changed, 353 insertions(+), 4 deletions(-) diff --git a/docs/security/policy-in-transactions.md b/docs/security/policy-in-transactions.md index 4f537aa2a3..2d04f8db1e 100644 --- a/docs/security/policy-in-transactions.md +++ b/docs/security/policy-in-transactions.md @@ -81,6 +81,10 @@ Every modification path runs the same `f:modify` policy check on its staged flak Crucially, the policy is checked against the **flakes**, not the operation type. A transaction that retracts a flake the identity can't modify is rejected just like an insert that asserts one. +A write that restates a stored value as it is stages nothing: an upsert of an unchanged value, or a DELETE and INSERT of the same value. When the identity cannot view that value, the restatement is still checked against `f:modify`, so it is refused unless the identity may modify the value, whether or not the value is stored. Restating a value the identity can view commits nothing, with no modify check. + +**Modify without view.** Granting `f:modify` on a value without `f:view` does not keep that value confidential from the identity: an identity that may modify a property can tell whether a given value is stored by writing it. Configure such policies with care. + Enforcement is also independent of the **wire format**: the check runs on the staged flakes, so JSON-LD, SPARQL UPDATE, and Turtle / TriG / N-Triples writes are all governed by the same `f:modify` policy. Sending data as Turtle is not a way to bypass write policy. Class targeting on the write side is **exact**: an `f:onClass` modify policy governs *every* flake whose subject is an instance of the class — including properties the class has never carried before. (Read-side class targeting uses the committed class→property index; write-side targeting matches by the subject's classes directly, since a write may introduce class/property combinations no committed data predicts.) diff --git a/fluree-db-api/tests/it_delete_stored_facts.rs b/fluree-db-api/tests/it_delete_stored_facts.rs index b5c7900657..83d1ca167e 100644 --- a/fluree-db-api/tests/it_delete_stored_facts.rs +++ b/fluree-db-api/tests/it_delete_stored_facts.rs @@ -842,6 +842,262 @@ async fn a_permitted_delete_of_an_absent_value_commits_nothing() { } } +// ============================================================================= +// Modify policy judges a restated value the identity cannot view +// ============================================================================= + +/// `ex:alice` with a plain value, a language-tagged one and a list. +fn restate_seed() -> JsonValue { + json!({ + "@id": "ex:alice", + "ex:name": "Alice", + "ex:salary": 100, + "ex:label": {"@value": "Al", "@language": "en"}, + "ex:scores": {"@list": [1, 2, 3]} + }) +} + +/// The identity may view everything and modify only `ex:name`. +async fn view_all_modify_name(ledger: &LedgerState) -> PolicyContext { + policy( + ledger, + GovernanceOptions { + policy: Some(json!([ + { + "@id": "http://example.org/ns/viewAll", + "@type": "f:AccessPolicy", + "f:action": "f:view", + "f:allow": true + }, + { + "@id": "http://example.org/ns/modifyName", + "@type": "f:AccessPolicy", + "f:onProperty": [{"@id": "http://example.org/ns/name"}], + "f:action": "f:modify", + "f:allow": true + } + ])), + default_allow: Some(false), + ..Default::default() + }, + ) + .await +} + +/// Upserts that restate a stored value, and their twins that write a value +/// that is not stored. +fn restating_upserts() -> Vec<(&'static str, JsonValue)> { + vec![ + ( + "plain, stored", + json!({"@id": "ex:alice", "ex:salary": 100}), + ), + ( + "plain, not stored", + json!({"@id": "ex:alice", "ex:salary": 999}), + ), + ( + "language-tagged, stored", + json!({"@id": "ex:alice", "ex:label": {"@value": "Al", "@language": "en"}}), + ), + ( + "language-tagged, not stored", + json!({"@id": "ex:alice", "ex:label": {"@value": "Bo", "@language": "en"}}), + ), + ( + "list, stored", + json!({"@id": "ex:alice", "ex:scores": {"@list": [1, 2, 3]}}), + ), + ( + "list, not stored", + json!({"@id": "ex:alice", "ex:scores": {"@list": [1, 2, 4]}}), + ), + ] +} + +async fn upsert_as( + fluree: &Fluree, + ledger: LedgerState, + mut doc: JsonValue, + policy: Option, +) -> Result { + doc["@context"] = ctx(); + let builder = fluree.stage_owned(ledger).upsert(&doc); + match policy { + Some(policy) => builder.policy(policy).execute().await, + None => builder.execute().await, + } +} + +/// `None` when `result` is the modify-policy refusal, else what happened. +fn not_refused( + result: Result, + what: &str, +) -> Option { + match result { + Ok(r) => Some(format!( + "{what}: committed {} flakes", + r.receipt.flake_count + )), + Err(e) + if e.to_string() + .contains("Policy enforcement prevents modification") => + { + None + } + Err(e) => Some(format!("{what}: {e}")), + } +} + +/// An upsert the identity may not perform is refused with one answer +/// whether it restates a stored value or writes one that is not stored: +/// plain, language-tagged and list values, on JSON-LD and Turtle. The +/// identity may neither view nor modify them. +#[tokio::test] +async fn upsert_refused_by_policy_regardless_of_stored_value() { + let mut failures = Vec::new(); + for indexed in [false, true] { + for (what, doc) in restating_upserts() { + let (_d, fluree, ledger) = seeded("restate-upsert", restate_seed(), indexed).await; + let policy = name_only(&ledger).await; + let r = upsert_as(&fluree, ledger, doc, Some(policy)).await; + failures.extend(not_refused( + r, + &format!("JSON-LD {what}, indexed={indexed}"), + )); + } + for salary in [100, 999] { + let (_d, fluree, ledger) = seeded("restate-turtle", restate_seed(), indexed).await; + let policy = name_only(&ledger).await; + let ttl = format!("@prefix ex: <{EX}> .\nex:alice ex:salary {salary} .\n"); + let r = fluree + .stage_owned(ledger) + .upsert_turtle(&ttl) + .policy(policy) + .execute() + .await; + failures.extend(not_refused( + r, + &format!("Turtle {salary}, indexed={indexed}"), + )); + } + } + assert!(failures.is_empty(), "{failures:#?}"); +} + +/// A DELETE and INSERT of one value is refused with one answer whether or +/// not the value is stored, on SPARQL and JSON-LD; so is a graph sync whose +/// payload restates a graph the identity cannot view. +#[tokio::test] +async fn delete_insert_of_one_value_refused_by_policy_regardless_of_stored_value() { + let mut failures = Vec::new(); + for indexed in [false, true] { + for salary in [100, 999] { + let (_d, fluree, ledger) = seeded("restate-sparql", restate_seed(), indexed).await; + let policy = name_only(&ledger).await; + let body = format!( + "DELETE {{ ?s ex:salary {salary} }} INSERT {{ ?s ex:salary {salary} }} \ + WHERE {{ ?s ex:name \"Alice\" }}" + ); + let r = sparql_update_as(&fluree, ledger, &body, policy).await; + failures.extend(not_refused( + r, + &format!("SPARQL {salary}, indexed={indexed}"), + )); + + let (_d, fluree, ledger) = seeded("restate-jsonld", restate_seed(), indexed).await; + let policy = name_only(&ledger).await; + let txn = json!({ + "@context": ctx(), + "where": {"@id": "?s", "ex:name": "Alice"}, + "delete": {"@id": "?s", "ex:salary": salary}, + "insert": {"@id": "?s", "ex:salary": salary} + }); + let r = fluree + .stage_owned(ledger) + .update(&txn) + .policy(policy) + .execute() + .await; + failures.extend(not_refused( + r, + &format!("JSON-LD {salary}, indexed={indexed}"), + )); + } + } + for salary in [100, 999] { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger = fluree + .create_ledger("it/delete-stored-restate-sync:main") + .await + .expect("create"); + let payload = |salary: i64| { + json!({ + "@context": ctx(), + "@graph": [{"@id": "ex:alice", "ex:name": "Alice", "ex:salary": salary}] + }) + }; + let seed = payload(100); + let ledger = fluree + .stage_owned(ledger) + .sync_graph(G1, &seed) + .execute() + .await + .expect("seed the graph") + .ledger; + let policy = name_only(&ledger).await; + let next = payload(salary); + let r = fluree + .stage_owned(ledger) + .sync_graph(G1, &next) + .policy(policy) + .execute() + .await; + failures.extend(not_refused(r, &format!("graph sync {salary}"))); + } + assert!(failures.is_empty(), "{failures:#?}"); +} + +/// Restating a value the identity can view commits nothing, without modify +/// permission, as it did before; a changed value is still refused. As root, +/// a restatement commits nothing either. +#[tokio::test] +async fn restating_a_viewable_value_commits_nothing_without_modify() { + for indexed in [false, true] { + for (what, doc) in restating_upserts() { + let stored = what.ends_with(", stored"); + let (_d, fluree, ledger) = seeded("restate-viewable", restate_seed(), indexed).await; + let t = ledger.t(); + let policy = view_all_modify_name(&ledger).await; + let r = upsert_as(&fluree, ledger, doc.clone(), Some(policy)).await; + let what = format!("{what}, indexed={indexed}"); + if stored { + no_commit(&r.expect(&what), t, &what); + } else { + assert_eq!(not_refused(r, &what), None); + } + if stored { + let (_d, fluree, ledger) = seeded("restate-root", restate_seed(), indexed).await; + let t = ledger.t(); + let r = upsert_as(&fluree, ledger, doc, None).await; + no_commit(&r.expect(&what), t, &format!("root, {what}")); + } + } + let (_d, fluree, ledger) = seeded("restate-viewable-sparql", restate_seed(), indexed).await; + let t = ledger.t(); + let policy = view_all_modify_name(&ledger).await; + let r = sparql_update_as( + &fluree, + ledger, + "DELETE { ?s ex:salary 100 } INSERT { ?s ex:salary 100 } WHERE { ?s ex:name \"Alice\" }", + policy, + ) + .await + .expect("restating a viewable value"); + no_commit(&r, t, &format!("SPARQL, indexed={indexed}")); + } +} + // ============================================================================= // A witness reads the graph its template writes // ============================================================================= diff --git a/fluree-db-transact/src/generate/accumulator.rs b/fluree-db-transact/src/generate/accumulator.rs index 446f389987..8c8f584ca1 100644 --- a/fluree-db-transact/src/generate/accumulator.rs +++ b/fluree-db-transact/src/generate/accumulator.rs @@ -186,6 +186,18 @@ impl FlakeAccumulator { /// For mixed: per fact, `cancel = min(assert_count, retract_count)` pairs /// cancel; surplus on either side contributes one survivor of that op. pub fn finalize(self) -> Vec { + self.drain(false).0 + } + + /// [`finalize`](Self::finalize), plus one `(retraction, assertion)` pair + /// for each fact the transaction retracted and asserted again: a stored + /// fact restated as it is, which stages nothing. + pub fn finalize_with_restated(self) -> (Vec, Vec<(Flake, Flake)>) { + self.drain(true) + } + + fn drain(self, keep_restated: bool) -> (Vec, Vec<(Flake, Flake)>) { + let mut restated = Vec::new(); let mut out: Vec = match self.inner { AccInner::PureRetract(graphs) => graphs .into_values() @@ -196,6 +208,11 @@ impl FlakeAccumulator { for inner in graphs.into_values() { for (_key, b) in inner { let cancel = b.assert_count.min(b.retract_count); + if keep_restated && cancel > 0 { + if let (Some(r), Some(a)) = (&b.retraction, &b.assertion) { + restated.push((r.clone(), a.clone())); + } + } if b.assert_count > cancel { if let Some(a) = b.assertion { v.push(a); @@ -212,7 +229,7 @@ impl FlakeAccumulator { } }; out.sort_by(|a, b| IndexType::Spot.compare(a, b)); - out + (out, restated) } } diff --git a/fluree-db-transact/src/stage.rs b/fluree-db-transact/src/stage.rs index d533cba3a5..56f1de4288 100644 --- a/fluree-db-transact/src/stage.rs +++ b/fluree-db-transact/src/stage.rs @@ -918,6 +918,11 @@ pub async fn stage_with_graph_delta( let retraction_count = stream_stats.retraction_count; let assertion_count = stream_stats.assertion_count; let total_inputs = acc.input_count(); + // Under a policy, the stored facts the write restates as they are + // (retracted and asserted again, so they cancel) go to the modify + // check below. Without one, cancellation keeps nothing. + let restricted = options.policy_ctx.is_some_and(|p| !p.wrapper().is_root()); + let mut restated: Vec<(Flake, Flake)> = Vec::new(); let mut flakes = if pure_delete { let _span = tracing::debug_span!("dedup_retractions", retraction_count = retraction_count) @@ -939,7 +944,13 @@ pub async fn stage_with_graph_delta( assertion_count = assertion_count, ) .entered(); - let f = acc.finalize(); + let f = if restricted { + let (f, pairs) = acc.finalize_with_restated(); + restated = pairs; + f + } else { + acc.finalize() + }; if f.len() as u64 != total_inputs { tracing::debug!( before = total_inputs, @@ -1060,15 +1071,22 @@ pub async fn stage_with_graph_delta( // Modify policy judges every delete the transaction asked for, // including DELETE intents that named no stored fact and stage // nothing: a delete the identity may not perform is refused the same - // way whether or not its target is stored. + // way whether or not its target is stored. Likewise a stored value + // the write restates as it is, which stages nothing, is judged on + // both sides when the identity cannot view it, so whether restating + // a value it cannot see succeeds does not depend on what is stored. if let Some(policy) = options.policy_ctx { if !policy.wrapper().is_root() { + let hidden = + hidden_restatements(&ledger, policy, restated, &reverse_graph, options.tracker) + .await?; let judged: std::borrow::Cow<'_, [Flake]> = - if stream_stats.unmatched_intents.is_empty() { + if stream_stats.unmatched_intents.is_empty() && hidden.is_empty() { std::borrow::Cow::Borrowed(&flakes) } else { let mut all = flakes.clone(); all.extend(stream_stats.unmatched_intents.iter().cloned()); + all.extend(hidden); std::borrow::Cow::Owned(all) }; let policy_span = tracing::debug_span!("policy_enforce"); @@ -2170,6 +2188,60 @@ struct SubjectDelta { retracts: Vec<(Sid, FlakeValue, Sid)>, } +/// Both sides of each restated pair (see +/// [`FlakeAccumulator::finalize_with_restated`]) whose stored fact `policy` +/// does not let the identity view, for the modify check. Builds a view +/// enforcer only when there is a pair to look at. +async fn hidden_restatements( + ledger: &LedgerState, + policy: &PolicyContext, + restated: Vec<(Flake, Flake)>, + reverse_graph: &HashMap, + tracker: Option<&Tracker>, +) -> Result> { + if restated.is_empty() { + return Ok(Vec::new()); + } + let enforcer = QueryPolicyEnforcer::new(Arc::new(policy.clone())); + let disabled; + let tracker = match tracker { + Some(tracker) => tracker, + None => { + disabled = Tracker::disabled(); + &disabled + } + }; + let mut by_graph: HashMap> = HashMap::new(); + for pair in restated { + let g_id = resolve_flake_graph_id(&pair.0, reverse_graph)?; + by_graph.entry(g_id).or_default().push(pair); + } + let mut hidden = Vec::new(); + for (g_id, pairs) in by_graph { + let db = ledger.as_graph_db_ref(g_id); + if policy.wrapper().has_class_policies() { + let mut subjects: Vec = pairs.iter().map(|(r, _)| r.s.clone()).collect(); + subjects.sort(); + subjects.dedup(); + enforcer + .populate_class_cache_for_graph(db, &subjects) + .await + .map_err(TransactError::Query)?; + } + for (retraction, assertion) in pairs { + let visible = enforcer + .allow_flake_for_graph(db.snapshot, g_id, db.overlay, db.t, tracker, &retraction) + .await + .map_err(TransactError::Query)?; + if !visible { + hidden.push(retraction); + hidden.push(assertion); + } + } + } + Ok(hidden) +} + /// Enforce modify policies on staged flakes /// /// This function handles the complete policy enforcement flow: From 007aa298390b76fd9578d53ed365fc2c85b7ca12 Mon Sep 17 00:00:00 2001 From: bplatz Date: Sun, 11 Oct 2026 11:34:08 -0400 Subject: [PATCH 36/36] fix: address review feedback on stored-fact retraction and RDF text Each item answers a review comment on #2008; each fix has a test that fails with the fix reverted. - Ghost fact after a revert: resolve merged index rows with novelty ops by fact identity, while the index and queries merge by index key. A revert retracts an indexed big integer under its declared subtype, which the row does not keep, so the row stayed current here after queries stopped seeing it and a later upsert of the value committed nothing. An index row is now dropped when a novelty op has its key. - Deletes over a wide slot were quadratic: every intent scanned its slot. Each slot is indexed once, by term and by index key when needed. At 8,000 values: 232 -> 27 ms (JSON-LD delete), 126 -> 25 ms (DELETE WHERE on a list-bearing ledger). - Identical list entries: a WHERE-driven row retracts every stored fact holding its term (each list entry, and a plain value), each once; each row used to take the first entry. it_join_batched_overlay now retracts one of two entries with a constant delete, keeping its o_i check. - ADD/COPY/MOVE under a non-root policy judge a fact the destination already holds as a restatement, so whether a transfer into a graph the identity cannot view succeeds no longer depends on what it holds. - A delete naming a graph the ledger does not have no longer lands an empty commit that registers the graph. An INSERT template's graph is still registered when its WHERE matches nothing. - fluree sync falls back to client-side conversion on the v4.2.0/4.2.1 400 as well as a 415, and warns that the conversion is lossy. - An unclosed '[', '(', '<<' or '{|' in a graph block is reported where it opens, not as a nested or unclosed block. - Docs: blank-node identity and spelling (upsert.md, turtle.md); a delete names a stored ill-typed literal while WHERE and query constants refuse it (datatypes.md); transfers in the restatement rule; sync's 400. - Tests: the urn:default refusal on TriG insert, graph insert and RDF sync; the multi-write transaction's Turtle lanes in the IRI parity test. - Comments left stale by the main merge. --- docs/cli/server-integration.md | 11 +- docs/cli/sync.md | 10 +- docs/concepts/datatypes.md | 8 +- docs/security/policy-in-transactions.md | 2 +- docs/transactions/turtle.md | 2 +- docs/transactions/upsert.md | 10 +- fluree-db-api/src/tx.rs | 7 +- fluree-db-api/tests/it_delete_stored_facts.rs | 221 ++++++++++++- .../tests/it_join_batched_overlay.rs | 18 +- fluree-db-api/tests/it_rdf_text_parity.rs | 58 +++- fluree-db-api/tests/it_sync_graph.rs | 2 +- fluree-db-cli/src/commands/graph_sync.rs | 90 +++++- fluree-db-cli/src/remote_client.rs | 6 +- fluree-db-query/src/dataset_operator.rs | 2 +- fluree-db-transact/src/current_facts.rs | 296 ++++++++++++------ .../src/parse/rdf_text/locate.rs | 63 +++- fluree-db-transact/src/stage.rs | 73 ++++- 17 files changed, 729 insertions(+), 150 deletions(-) diff --git a/docs/cli/server-integration.md b/docs/cli/server-integration.md index 8ba133799a..a215c9e6f3 100644 --- a/docs/cli/server-integration.md +++ b/docs/cli/server-integration.md @@ -350,9 +350,12 @@ MATCH (n:Person {id: 7}) RETURN n Makes one named graph's contents exactly the payload, committing only the delta. Data-bearer auth (same bracket as `/insert` / `/upsert`), not admin. The CLI sends RDF text as written: Turtle as `text/turtle`, TriG as -`application/trig`, JSON-LD as `application/json`. A server that answers -`415 Unsupported Media Type` to a Turtle body (one from before `/sync` read -RDF bodies) gets it again converted to JSON-LD client-side; TriG is never +`application/trig`, JSON-LD as `application/json`. A server from before +`/sync` read RDF bodies refuses a Turtle body, with `415 Unsupported Media +Type` or (v4.2.0 and v4.2.1) a `400` whose message ends `convert Turtle +payloads client-side`; for that server the CLI converts the Turtle to +JSON-LD client-side, sends it again, and prints a warning. Any other `400` +is reported as is, and TriG is never converted. A dry run answers with a delta report and must commit nothing; a real run answers with the standard transact response. Designed so the CLI's source of desired contents (today RDF text; later @@ -1520,7 +1523,7 @@ Content-Type: application/json | `default` (query) | No | Bare key: sync the default graph, said explicitly. Passing it with `graph` is a `400`. The CLI omits `graph` for the default graph. | | `dryRun` (query) | No | `true` → stage and report the delta; commit nothing. | | `allowEmpty` (query) | No | `true` → accept an explicitly empty payload (`"@graph": []`), which clears the graph. Without it an empty payload is a `400`. | -| body | Yes | The graph's desired full contents: insert-shaped JSON-LD, or Turtle / N-Triples / TriG by `Content-Type` (see [Payload formats](../transactions/sync.md#payload-formats)). The CLI sends JSON-LD for JSON-LD and Turtle (converted client-side), so those work against servers that predate RDF bodies; it sends TriG as `application/trig`. Policy headers / `opts` injection follow the [Policy Enforcement Contract](#policy-enforcement-contract). | +| body | Yes | The graph's desired full contents: insert-shaped JSON-LD, or Turtle / N-Triples / TriG by `Content-Type` (see [Payload formats](../transactions/sync.md#payload-formats)). The CLI sends Turtle as `text/turtle` and TriG as `application/trig`, converting Turtle to JSON-LD client-side only when a server that predates RDF bodies refuses it (a `415`, or v4.2.0 / v4.2.1's `400`). Policy headers / `opts` injection follow the [Policy Enforcement Contract](#policy-enforcement-contract). | ### Auth diff --git a/docs/cli/sync.md b/docs/cli/sync.md index 1d7e92000a..72db3b54bc 100644 --- a/docs/cli/sync.md +++ b/docs/cli/sync.md @@ -46,10 +46,12 @@ Turtle input is sent as written and parsed where it is staged, the way every write reads RDF text (see [How RDF text is read](../transactions/turtle.md#how-rdf-text-is-read)). A `.nt` file is read as Turtle, of which N-Triples is a subset. With -`--remote`, a server from before `/sync` accepted Turtle bodies answers -415; for that server the CLI converts the Turtle to JSON-LD and sends it -again. The conversion loses collection order and refuses some IRI schemes -(`tag:`, `kb:`), so upgrading the server is the better fix. +`--remote`, a server from before `/sync` accepted Turtle bodies refuses +one: v4.2.0 and v4.2.1 answer 400 (`convert Turtle payloads client-side`), +others 415. For that server the CLI converts the Turtle to JSON-LD, sends +it again, and prints a warning. The conversion loses collection order and +refuses some IRI schemes (`tag:`, `kb:`), so upgrading the server is the +better fix. TriG is sent as TriG. It is recognized by a `.trig` name or `--format trig`, and also when a body read as Turtle turns out to hold diff --git a/docs/concepts/datatypes.md b/docs/concepts/datatypes.md index 93875b5c8e..55879f023a 100644 --- a/docs/concepts/datatypes.md +++ b/docs/concepts/datatypes.md @@ -406,11 +406,17 @@ Numeric types (long, double, bigint, decimal) are mutually comparable via automa A literal such as `"2024-02-30"^^xsd:date` or `"300"^^xsd:byte` names a built-in datatype but is not one of its values. JSON-LD transactions and -SPARQL UPDATE reject it, naming the datatype in the error. Turtle +SPARQL UPDATE refuse to insert it, naming the datatype in the error. Turtle transactions and bulk import keep it, as RDF allows: the literal is stored with its text and its datatype and reads back exactly as written, but it is not a date or a number, so it never equals one. +A delete names such a literal as it is stored, so `DELETE DATA` or a JSON-LD +`delete` with the same text and datatype removes it, and one that matches +nothing stored commits nothing. A constant in a WHERE clause or a query is +still refused, so `DELETE WHERE` and `SELECT` reach a stored one only through +a variable (`FILTER(STR(?o) = "2024-02-30")`). + ### Type Casting in Queries SPARQL provides functions for type conversion: diff --git a/docs/security/policy-in-transactions.md b/docs/security/policy-in-transactions.md index 2d04f8db1e..f7fcfd67ee 100644 --- a/docs/security/policy-in-transactions.md +++ b/docs/security/policy-in-transactions.md @@ -81,7 +81,7 @@ Every modification path runs the same `f:modify` policy check on its staged flak Crucially, the policy is checked against the **flakes**, not the operation type. A transaction that retracts a flake the identity can't modify is rejected just like an insert that asserts one. -A write that restates a stored value as it is stages nothing: an upsert of an unchanged value, or a DELETE and INSERT of the same value. When the identity cannot view that value, the restatement is still checked against `f:modify`, so it is refused unless the identity may modify the value, whether or not the value is stored. Restating a value the identity can view commits nothing, with no modify check. +A write that restates a stored value as it is stages nothing: an upsert of an unchanged value, a DELETE and INSERT of the same value, or an `ADD`, `COPY` or `MOVE` of a fact the destination graph already holds. When the identity cannot view that value, the restatement is still checked against `f:modify`, so it is refused unless the identity may modify the value, whether or not the value is stored. Restating a value the identity can view commits nothing, with no modify check. **Modify without view.** Granting `f:modify` on a value without `f:view` does not keep that value confidential from the identity: an identity that may modify a property can tell whether a given value is stored by writing it. Configure such policies with care. diff --git a/docs/transactions/turtle.md b/docs/transactions/turtle.md index d29c8b3adb..beefac23ed 100644 --- a/docs/transactions/turtle.md +++ b/docs/transactions/turtle.md @@ -635,7 +635,7 @@ Every transaction endpoint reads Turtle and TriG with the same parser, once, and - **Blank-node labels are document-scoped:** `_:b` in the default graph and in two blocks is one node. Each `[ ]` is a new node. - **Errors point into the document as sent:** a parse error in a block reports its position in the whole document, and a malformed block (nested, unclosed, a directive inside it, a blank-node label) is named as such. -An upsert scopes a document's blank nodes to its content, so upserting the same document again addresses the same nodes and commits nothing. The scope ignores statement order (anonymous `[ ]` nodes are numbered in document order, so moving statements that contain them relative to each other gives a new scope); any change to the statements gives a new scope and new nodes, and the previous nodes stay. To find blank-node subjects that nothing references any more: +An upsert scopes a document's blank nodes to its content, so upserting the same document again addresses the same nodes and commits nothing. The scope ignores statement order (anonymous `[ ]` nodes are numbered in document order, so moving statements that contain them relative to each other gives a new scope); any change to the statements, or to how they are written beyond IRI and quote style (`1` vs `"1"^^xsd:integer`, a language tag's case, a repeated statement, a renamed blank-node label), gives a new scope and new nodes, and the previous nodes stay. To find blank-node subjects that nothing references any more: ```sparql SELECT DISTINCT ?b WHERE { diff --git a/docs/transactions/upsert.md b/docs/transactions/upsert.md index 44d9a28ba2..8ce8ff4b0d 100644 --- a/docs/transactions/upsert.md +++ b/docs/transactions/upsert.md @@ -128,8 +128,14 @@ current value is retracted exactly as it is stored. document mints new blank nodes; the old ones stay (see [Stable blank-node ids](update-where-delete-insert.md#editing-blank-node-structures-stable-_fdb--ids) for editing a stored blank node in place). For Turtle and TriG the - identity comes from the parsed statements, not their order or spelling; - see [How RDF text is read](turtle.md#how-rdf-text-is-read) for the details + identity comes from the parsed statements, not their order: IRIs written + in full or as prefixed names, and strings in single or double quotes, + give the same identity. Other differences in how the statements are + written give a new one, and so new blank nodes: a number or boolean + written bare or as a typed literal (`1` vs `"1"^^xsd:integer`), a + language tag's case (`@en-US` vs `@en-us`), a repeated statement, or a + renamed blank-node label (`_:b` vs `_:c`). See + [How RDF text is read](turtle.md#how-rdf-text-is-read) for the details and a query that lists blank nodes nothing references. - **Named graphs.** The graph is part of the unit: an upsert into a named graph replaces that graph's values and leaves the same predicate's values diff --git a/fluree-db-api/src/tx.rs b/fluree-db-api/src/tx.rs index 850021b5fd..9da3f82859 100644 --- a/fluree-db-api/src/tx.rs +++ b/fluree-db-api/src/tx.rs @@ -271,9 +271,10 @@ impl SequentialStager { /// the fact, or retracted then re-asserted when it has it. Last-wins /// alone would commit the final op — a retraction of a fact the ledger /// never held, for `INSERT x ; DELETE x`. Only facts whose first and last - /// ops differ are looked up; the base decides because staging does not - /// filter an assert of a fact already present (or a retract of one - /// absent), so the first op does not say what the base held. + /// ops differ are looked up; the base decides because staging still + /// stages an assert of a fact already present, so a first-op assert does + /// not say what the base held. (A first-op retraction does: staging only + /// retracts a stored fact.) async fn drop_reversed_facts(&mut self) -> Result<()> { let mut keep = vec![true; self.folded.len()]; for (i, flake) in self.folded.iter().enumerate() { diff --git a/fluree-db-api/tests/it_delete_stored_facts.rs b/fluree-db-api/tests/it_delete_stored_facts.rs index 83d1ca167e..d56a752097 100644 --- a/fluree-db-api/tests/it_delete_stored_facts.rs +++ b/fluree-db-api/tests/it_delete_stored_facts.rs @@ -22,7 +22,8 @@ use crate::support; use fluree_db_api::policy_builder::build_policy_context_from_opts; use fluree_db_api::{ - Fluree, FlureeBuilder, GovernanceOptions, LedgerState, PolicyContext, TransactResult, + CommitRef, ConflictStrategy, Fluree, FlureeBuilder, GovernanceOptions, LedgerState, + PolicyContext, TransactResult, }; use fluree_db_core::comparator::IndexType; use fluree_db_core::{range_with_overlay, FlakeValue, RangeMatch, RangeOptions, RangeTest}; @@ -186,6 +187,105 @@ async fn deleting_an_absent_fact_commits_nothing() { } } +/// A delete that names a graph the ledger does not have commits nothing and +/// registers no graph, as in the default graph; it used to land an empty +/// commit that registered the graph. A graph an INSERT template names is still +/// registered when the WHERE matches nothing, delete beside it or not +/// (`no_match_update_naming_a_new_graph_commits_its_registration`). +#[tokio::test] +async fn a_delete_in_a_new_graph_commits_nothing() { + const NEW: &str = "http://example.org/new"; + let registered = |r: &TransactResult| r.ledger.snapshot.graph_registry.graph_id_for_iri(NEW); + for indexed in [false, true] { + let seed = json!({"@id": "ex:s", "ex:p": "present"}); + + let (_d, fluree, ledger) = seeded("new-graph-sparql", seed.clone(), indexed).await; + let t = ledger.t(); + let r = sparql_update( + &fluree, + ledger, + &format!("DELETE DATA {{ GRAPH <{NEW}> {{ ex:s ex:p \"present\" }} }}"), + ) + .await; + let what = format!("SPARQL DELETE DATA (indexed={indexed})"); + no_commit(&r, t, &what); + assert!(registered(&r).is_none(), "{what}: registered {NEW}"); + + let (_d, fluree, ledger) = seeded("new-graph-jsonld", seed.clone(), indexed).await; + let t = ledger.t(); + let r = jsonld_update( + &fluree, + ledger, + json!({"graph": NEW, "delete": {"@id": "ex:s", "ex:p": "present"}}), + ) + .await; + let what = format!("JSON-LD delete (indexed={indexed})"); + no_commit(&r, t, &what); + assert!(registered(&r).is_none(), "{what}: registered {NEW}"); + + let (_d, fluree, ledger) = seeded("new-graph-insert", seed.clone(), indexed).await; + let t = ledger.t(); + let r = sparql_update( + &fluree, + ledger, + &format!( + "DELETE {{ GRAPH <{NEW}> {{ ?s ex:q \"x\" }} }} \ + INSERT {{ GRAPH <{NEW}> {{ ?s ex:q \"y\" }} }} WHERE {{ ?s ex:p \"nothing\" }}" + ), + ) + .await; + let what = format!("DELETE and INSERT, no match (indexed={indexed})"); + assert_eq!( + (r.receipt.flake_count, r.ledger.t()), + (0, t + 1), + "{what}: the registration commits" + ); + assert!(registered(&r).is_some(), "{what}: {NEW} not registered"); + } +} + +/// A WHERE-driven DELETE removes every stored fact its rows name, list +/// entries holding the same value included: each identical row used to take +/// the first stored entry, so `DELETE WHERE` over `["v", "v", "w"]` left one +/// `"v"` behind, and over a plain `"v"` beside a list `["v"]` left the plain +/// one. A WHERE that repeats each row (a second pattern matching twice) +/// retracts each stored fact once. +#[tokio::test] +async fn a_where_driven_delete_removes_every_identical_list_entry() { + for indexed in [false, true] { + for (what, seed, delete, retracted) in [ + ( + "repeated list value", + json!({"@id": "ex:s", "ex:list": {"@list": ["v", "v", "w"]}}), + "DELETE WHERE { ex:s ex:list ?o }", + 3, + ), + ( + "plain value beside a list entry", + json!({"@id": "ex:s", "ex:list": ["v", {"@list": ["v"]}]}), + "DELETE WHERE { ex:s ex:list ?o }", + 2, + ), + ( + "rows repeated by another pattern", + json!({"@graph": [ + {"@id": "ex:s", "ex:list": {"@list": ["v", "v"]}}, + {"@id": "ex:x1", "ex:q": 1}, + {"@id": "ex:x2", "ex:q": 2} + ]}), + "DELETE { ex:s ex:list ?o } WHERE { ex:s ex:list ?o . ?x ex:q ?y }", + 2, + ), + ] { + let (_d, fluree, ledger) = seeded("identical-entries", seed, indexed).await; + let r = sparql_update(&fluree, ledger, delete).await; + let what = format!("{what} (indexed={indexed})"); + assert_eq!(r.receipt.retract_count, retracted, "{what}"); + assert!(facts(&fluree, &r.ledger).await.is_empty(), "{what}"); + } + } +} + #[tokio::test] async fn delete_insert_of_an_absent_fact_inserts_it() { for indexed in [false, true] { @@ -485,10 +585,70 @@ async fn constant_deletes_name_big_integers_under_their_subtype() { } } +/// A novelty retraction of an indexed fact under another identity with the +/// same index key leaves no ghost: a revert retracts a big integer under its +/// declared subtype, which the index row does not keep, and queries stop +/// seeing the value. The stored-fact read used to keep reporting it, so a +/// later upsert of the same value cancelled against it and committed +/// nothing, and the value stayed gone until the next reindex. +#[tokio::test] +async fn a_reverted_indexed_big_integer_can_be_written_again() { + for dt in ["integer", "nonNegativeInteger"] { + let dir = tempfile::tempdir().expect("tempdir"); + let fluree = FlureeBuilder::file(dir.path().to_string_lossy().to_string()) + .build() + .expect("file fluree"); + let name = format!("it/delete-stored-revert-{dt}"); + let id = format!("{name}:main"); + let ledger = fluree.create_ledger(&id).await.expect("create"); + let ledger = fluree + .insert( + ledger, + &json!({"@context": ctx(), "@id": "ex:s", "ex:other": 1}), + ) + .await + .expect("seed") + .ledger; + let value = json!({"@context": ctx(), "@id": "ex:s", + "ex:n": {"@value": BIG, "@type": format!("xsd:{dt}")}}); + let inserted = fluree.insert(ledger, &value).await.expect("insert"); + let commit = inserted.receipt.commit_id.clone(); + drop(inserted); + support::rebuild_and_publish_index(&fluree, &id).await; + fluree + .revert_commit( + &name, + "main", + CommitRef::Exact(commit), + ConflictStrategy::TakeSource, + ) + .await + .expect("revert"); + let ledger = fluree.ledger(&id).await.expect("reload"); + assert_eq!( + facts(&fluree, &ledger).await, + ["other=1"], + "xsd:{dt}: reverted" + ); + + let r = fluree.upsert(ledger, &value).await.expect("upsert"); + assert_eq!( + (r.receipt.assert_count, r.receipt.retract_count), + (1, 0), + "xsd:{dt}: the upsert writes the value" + ); + assert_eq!( + facts(&fluree, &r.ledger).await, + [format!("n={BIG}"), "other=1".to_string()], + "xsd:{dt}: written again" + ); + } +} + /// A value bound by OPTIONAL, UNION or BIND is not a witness, so its DELETE /// row is matched against the stored facts of its slot. The row carries the -/// query's decode of the value; on an indexed ledger that decode and the -/// stored fact's differ in datatype, and the row still names the fact. +/// query's decode of the value, which on an indexed ledger has lost a +/// declared subtype, and the row still names the fact. /// /// Not covered: BIND on a novelty-only ledger with a subtype. BIND gives a /// big integer `xsd:integer` whatever its declared subtype, so there the row @@ -1061,6 +1221,61 @@ async fn delete_insert_of_one_value_refused_by_policy_regardless_of_stored_value /// Restating a value the identity can view commits nothing, without modify /// permission, as it did before; a changed value is still refused. As root, /// a restatement commits nothing either. +/// A transfer judges a fact the destination already holds as a restatement: +/// `ADD` or `COPY` of a value the identity can neither view nor modify is +/// refused with one answer whether or not the destination holds it. A held +/// fact used to stage nothing and commit, so the outcome told the identity +/// what the destination held. +#[tokio::test] +async fn transfer_refused_by_policy_regardless_of_destination_contents() { + const G2: &str = "http://example.org/g2"; + for indexed in [false, true] { + for dest_holds in [true, false] { + for verb in ["ADD", "COPY"] { + let dir = tempfile::tempdir().expect("tempdir"); + let fluree = FlureeBuilder::file(dir.path().to_string_lossy().to_string()) + .build() + .expect("file fluree"); + let id = format!("it/delete-stored-transfer-{verb}-{dest_holds}-{indexed}:main"); + let ledger = fluree.create_ledger(&id).await.expect("create"); + let dest = if dest_holds { + "ex:alice ex:salary 100" + } else { + "ex:alice ex:name \"Alice\"" + }; + let ledger = sparql_update( + &fluree, + ledger, + &format!( + "INSERT DATA {{ GRAPH <{G1}> {{ ex:alice ex:salary 100 }} \ + GRAPH <{G2}> {{ {dest} }} }}" + ), + ) + .await + .ledger; + let ledger = if indexed { + drop(ledger); + support::rebuild_and_publish_index(&fluree, &id).await; + fluree.ledger(&id).await.expect("reload") + } else { + ledger + }; + let policy = name_only(&ledger).await; + let body = format!("{verb} GRAPH <{G1}> TO GRAPH <{G2}>"); + let err = sparql_update_as(&fluree, ledger, &body, policy) + .await + .map(|r| r.receipt.flake_count) + .expect_err(&body); + assert!( + err.to_string() + .contains("Policy enforcement prevents modification"), + "{body}, destination holds it={dest_holds}, indexed={indexed}: {err}" + ); + } + } + } +} + #[tokio::test] async fn restating_a_viewable_value_commits_nothing_without_modify() { for indexed in [false, true] { diff --git a/fluree-db-api/tests/it_join_batched_overlay.rs b/fluree-db-api/tests/it_join_batched_overlay.rs index ef9a52b36a..63b36e3499 100644 --- a/fluree-db-api/tests/it_join_batched_overlay.rs +++ b/fluree-db-api/tests/it_join_batched_overlay.rs @@ -483,8 +483,8 @@ async fn batched_object_join_merges_novelty() { .expect("reindex base"); // Novelty tail: b2's author retracted, b3 asserted, grace + b4 exist only - // in novelty, and b5's two-entry author @list is fully retracted (both - // flakes share (s, p, o) and differ only in o_i). + // in novelty, and one entry of b5's two-entry author @list is retracted + // (both flakes share (s, p, o) and differ only in o_i). let receipt = fluree .update( receipt.ledger, @@ -499,14 +499,15 @@ async fn batched_object_join_merges_novelty() { let receipt = fluree .update( receipt.ledger, + // A constant delete names one entry of a value held at several + // list positions; a WHERE-driven delete would name every one. &json!({ "@context": ctx(), - "where": {"@id": "ex:b5", "ex:authors": "?a"}, - "delete": {"@id": "ex:b5", "ex:authors": "?a"} + "delete": {"@id": "ex:b5", "ex:authors": {"@id": "ex:bob"}} }), ) .await - .expect("retract b5 author list"); + .expect("retract one b5 author entry"); let _receipt = fluree .insert( receipt.ledger, @@ -542,10 +543,9 @@ async fn batched_object_join_merges_novelty() { r"PREFIX ex: SELECT ?x WHERE { VALUES ?b { ex:bob ex:dave } ?x ex:authors ?b } ORDER BY ?x", - 1, // the delete's WHERE dedupes the repeated @list value to one - // binding, so exactly ONE of the two o_i entries is retracted: - // the fate check must discriminate the entries by o_i (a - // default-read o_i would either resurrect both or drop both) + 1, // exactly ONE of the two o_i entries is retracted: the fate + // check must discriminate the entries by o_i (a default-read o_i + // would either resurrect both or drop both) true, ), ( diff --git a/fluree-db-api/tests/it_rdf_text_parity.rs b/fluree-db-api/tests/it_rdf_text_parity.rs index feeeb562f0..381b81705c 100644 --- a/fluree-db-api/tests/it_rdf_text_parity.rs +++ b/fluree-db-api/tests/it_rdf_text_parity.rs @@ -11,7 +11,7 @@ use crate::support::{genesis_ledger, rebuild_and_publish_index, MemoryFluree}; use fluree_db_api::policy_builder::build_policy_context_from_opts; use fluree_db_api::{ CommitOpts, Fluree, FlureeBuilder, GovernanceOptions, GraphDb, GraphPayload, LedgerState, - PolicyContext, TrackingOptions, + PolicyContext, TrackingOptions, TransactionOptions, TxnOperation, }; use fluree_db_core::comparator::IndexType; use fluree_db_core::{ @@ -250,11 +250,13 @@ async fn every_rdf_text_lane_stores_iris_the_json_ld_round_trip_refused() { "cached handle", "cached handle + policy", "graph builder", + "multi-write transaction", "insert: Fluree::insert_turtle", "insert: owned builder", "insert: cached handle", "insert: cached handle + policy", "insert: graph builder", + "insert: multi-write transaction", ] { let id = format!( "it/rdf-iri-{}:main", @@ -348,6 +350,21 @@ async fn every_rdf_text_lane_stores_iris_the_json_ld_round_trip_refused() { .commit() .await .map(|_| ()), + "multi-write transaction" | "insert: multi-write transaction" => { + let operation = if lane.starts_with("insert") { + TxnOperation::InsertTurtle(doc.clone()) + } else { + TxnOperation::UpsertTurtle(doc.clone()) + }; + let mut txn = fluree + .begin_transaction(&id, TransactionOptions::default()) + .await + .expect("begin"); + match txn.stage(operation).await { + Ok(()) => txn.commit(CommitOpts::default()).await.map(|_| ()), + Err(e) => Err(e), + } + } other => unreachable!("{other}"), }; if let Err(e) = outcome { @@ -1023,6 +1040,45 @@ async fn a_malformed_document_fails_as_a_turtle_parse_error() { assert!(err.to_string().contains("nested graph block"), "{err}"); } +/// `urn:default` names the default graph, never a graph an RDF text write can +/// create: a TriG insert, an RDF graph insert and an RDF graph sync that name +/// it are refused, as the upsert and JSON-LD sync lanes are +/// (`it_default_graph_name`). +#[tokio::test] +async fn rdf_text_writes_naming_urn_default_as_a_graph_are_refused() { + const REFUSAL: &str = "names the default graph, not a named graph"; + let fluree = memory(); + let statements = " \"zed\" .\n"; + let block = format!("GRAPH {{ {statements} }}\n"); + let urn_default = || GraphSel::Graph("urn:default".to_string()); + let mut failures = Vec::new(); + for (lane, body) in [ + ("TriG insert", &block), + ("graph insert, TriG block", &block), + ("sync, Turtle body", &statements.to_string()), + ("sync, TriG block", &block), + ] { + let id = format!( + "it/rdf-urn-default-{}:main", + lane.replace(|c: char| !c.is_ascii_alphanumeric(), "-") + ); + let builder = fluree.stage_owned(genesis_ledger(&fluree, &id)); + let builder = match lane { + "TriG insert" => builder.insert_turtle(body), + "graph insert, TriG block" => { + builder.insert_graph_payload(urn_default(), GraphPayload::Rdf(body)) + } + _ => builder.sync_graph_payload(urn_default(), GraphPayload::Rdf(body), false), + }; + match builder.execute().await { + Ok(_) => failures.push(format!("{lane}: committed")), + Err(e) if !e.to_string().contains(REFUSAL) => failures.push(format!("{lane}: {e}")), + Err(_) => {} + } + } + assert!(failures.is_empty(), "\n{}", failures.join("\n")); +} + /// What a block's contents are refused for is the user's mistake, reported /// as one: an undefined prefix is a Turtle parse error, a hand-written /// `f:reifies*` statement a transaction error. Neither reads as an engine diff --git a/fluree-db-api/tests/it_sync_graph.rs b/fluree-db-api/tests/it_sync_graph.rs index f3c7cc700e..1fd78b2509 100644 --- a/fluree-db-api/tests/it_sync_graph.rs +++ b/fluree-db-api/tests/it_sync_graph.rs @@ -468,7 +468,7 @@ async fn staging_rejects_malformed_and_system_graph_targets() { /// An identical resync must still be a no-op after the graph has been /// indexed — the case the memory-backed tests cannot reach. /// -/// `scan_graph_flakes` reads through the range provider, and +/// `whole_graph_facts` reads through the range provider, and /// `BinaryRangeProvider` materializes flakes with `g: None` /// (`binary_range.rs:753`, `:1393`). The accumulator buckets on `flake.g`, /// so without the explicit stamp in the sync wave the retractions land in a diff --git a/fluree-db-cli/src/commands/graph_sync.rs b/fluree-db-cli/src/commands/graph_sync.rs index 376edf0f3a..a4e40df59b 100644 --- a/fluree-db-cli/src/commands/graph_sync.rs +++ b/fluree-db-cli/src/commands/graph_sync.rs @@ -15,6 +15,7 @@ use crate::detect; use crate::error::{CliError, CliResult}; use crate::input; use crate::remote_client::{RemoteLedgerClient, RemoteLedgerError}; +use colored::Colorize; use fluree_db_api::server_defaults::FlureeDir; use fluree_db_api::{GraphPayload, GraphSel, SyncGraphOpts, SyncGraphReport, TxnOpts}; use std::path::Path; @@ -91,9 +92,10 @@ impl SyncSource { } /// Send `payload` to a remote `/sync`. RDF text goes as written; only a -/// server from before `/sync` read RDF bodies answers 415, and for that one -/// Turtle is converted to JSON-LD here and sent again (TriG has no JSON-LD -/// form for its blocks, so it gets the 415). +/// server from before `/sync` read RDF bodies refuses it (see +/// [`predates_rdf_sync`]), and for that one Turtle is converted to JSON-LD +/// here and sent again (TriG has no JSON-LD form for its blocks, so it gets +/// the refusal). async fn sync_remote( client: &RemoteLedgerClient, ledger: &str, @@ -116,7 +118,13 @@ async fn sync_remote( .sync_rdf(ledger, graph, text, content_type, dry_run, allow_empty) .await { - Err(RemoteLedgerError::UnsupportedMediaType(_)) if !*trig => { + Err(e) if !*trig && predates_rdf_sync(&e) => { + eprintln!( + " {} the server predates RDF bodies on /sync, so the Turtle was converted \ + to JSON-LD client-side,\n which can lose collection order and mishandle \ + IRIs with schemes like `tag:` or `kb:`. Upgrade the server to avoid this.", + "warning:".yellow().bold() + ); let json = turtle_as_json_ld(text)?; Ok(client .sync_jsonld(ledger, graph, &json, dry_run, allow_empty) @@ -128,6 +136,20 @@ async fn sync_remote( } } +/// Whether `/sync` refused an RDF body because the server cannot read one: +/// v4.2.0 and v4.2.1 answer with this 400, other such servers with a 415. +/// Both come before staging, so resending as JSON-LD is safe. Any other 400 +/// (bad Turtle, policy, ...) is a verdict on the payload and must stand. +fn predates_rdf_sync(err: &RemoteLedgerError) -> bool { + const PRE_RDF_SYNC_400: &str = + "sync accepts application/json (JSON-LD); convert Turtle payloads client-side"; + match err { + RemoteLedgerError::UnsupportedMediaType(_) => true, + RemoteLedgerError::BadRequest(msg) => msg.contains(PRE_RDF_SYNC_400), + _ => false, + } +} + /// The client-side conversion a server without RDF bodies on `/sync` needs. /// It is lossy (collection order, `rdf:type` objects that are not IRIs, IRIs /// whose scheme reads as a prefix); upgrading the server avoids it. @@ -383,12 +405,32 @@ mod tests { /// A stub `/sync` answering its requests with `statuses` in turn, /// recording each request's content type. async fn stub(statuses: Vec) -> (RemoteLedgerClient, Arc>>) { + stub_replying( + statuses + .into_iter() + .map(|status| { + let body = if status == 200 { + r#"{"ledger":"db:main","t":2}"# + } else { + r#"{"error":"unsupported media type"}"# + }; + (status, body) + }) + .collect(), + ) + .await + } + + /// [`stub`] with each reply's body given. + async fn stub_replying( + replies: Vec<(u16, &'static str)>, + ) -> (RemoteLedgerClient, Arc>>) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let seen: Arc>> = Arc::default(); let recorded = Arc::clone(&seen); tokio::spawn(async move { - for status in statuses { + for (status, body) in replies { let (mut sock, _) = listener.accept().await.unwrap(); let mut buf = Vec::new(); let mut tmp = [0u8; 4096]; @@ -423,11 +465,6 @@ mod tests { .lock() .unwrap() .push(header("content-type:").unwrap_or_default()); - let body = if status == 200 { - r#"{"ledger":"db:main","t":2}"# - } else { - r#"{"error":"unsupported media type"}"# - }; let reply = format!( "HTTP/1.1 {status} X\r\nContent-Type: application/json\r\n\ Content-Length: {}\r\nConnection: close\r\n\r\n{body}", @@ -460,7 +497,7 @@ mod tests { } #[tokio::test] - async fn turtle_falls_back_to_json_ld_only_on_a_415() { + async fn turtle_falls_back_to_json_ld_on_a_415() { let (client, seen) = stub(vec![415, 200]).await; sync_remote(&client, "db:main", None, &turtle(), false, false) .await @@ -468,6 +505,37 @@ mod tests { assert_eq!(*seen.lock().unwrap(), ["text/turtle", "application/json"]); } + /// v4.2.0 and v4.2.1 refuse a Turtle body with a 400, not a 415. + #[tokio::test] + async fn turtle_falls_back_on_the_v4_2_0_refusal() { + let (client, seen) = stub_replying(vec![ + ( + 400, + r#"{"error":"Bad request: sync accepts application/json (JSON-LD); convert Turtle payloads client-side","status":400,"@type":"err:api/BadRequest"}"#, + ), + (200, r#"{"ledger":"db:main","t":2}"#), + ]) + .await; + sync_remote(&client, "db:main", None, &turtle(), false, false) + .await + .expect("sync after the fallback"); + assert_eq!(*seen.lock().unwrap(), ["text/turtle", "application/json"]); + } + + #[tokio::test] + async fn any_other_400_is_reported_not_converted() { + let (client, seen) = stub_replying(vec![( + 400, + r#"{"error":"Turtle parse error: Parse error at position 52: expected '.'","status":400,"@type":"err:api/BadRequest"}"#, + )]) + .await; + let err = sync_remote(&client, "db:main", None, &turtle(), false, false) + .await + .expect_err("the server's verdict stands"); + assert!(err.to_string().contains("Turtle parse error"), "{err}"); + assert_eq!(*seen.lock().unwrap(), ["text/turtle"]); + } + #[tokio::test] async fn trig_is_not_converted() { let (client, seen) = stub(vec![415]).await; diff --git a/fluree-db-cli/src/remote_client.rs b/fluree-db-cli/src/remote_client.rs index e401f64e44..2871272f37 100644 --- a/fluree-db-cli/src/remote_client.rs +++ b/fluree-db-cli/src/remote_client.rs @@ -1312,8 +1312,10 @@ impl RemoteLedgerClient { /// [`sync_jsonld`](Self::sync_jsonld) with an RDF text body sent as /// `content_type` (`text/turtle`, `application/n-triples`, /// `application/trig`). A server that does not read RDF bodies on - /// `/sync` answers 415 ([`RemoteLedgerError::UnsupportedMediaType`]) - /// before staging anything, so the caller may retry in another format. + /// `/sync` refuses before staging anything, so the caller may retry in + /// another format: 415 ([`RemoteLedgerError::UnsupportedMediaType`]), or + /// on v4.2.0 and v4.2.1 a 400 ([`RemoteLedgerError::BadRequest`]) asking + /// for JSON-LD. pub async fn sync_rdf( &self, ledger: &str, diff --git a/fluree-db-query/src/dataset_operator.rs b/fluree-db-query/src/dataset_operator.rs index b8026c2b3a..60c01f0554 100644 --- a/fluree-db-query/src/dataset_operator.rs +++ b/fluree-db-query/src/dataset_operator.rs @@ -212,7 +212,7 @@ struct DatasetMember { /// `Eq`/`Hash` exclude history metadata, so this is only armed in current mode /// (see [`DatasetOperator::open`]). /// -/// MEMORY CLIFF (documented like `scan_graph_flakes`'s): unlike a +/// MEMORY CLIFF (documented like `whole_graph_facts`'s): unlike a /// single-graph scan, a `>= 2`-member default union is no longer /// bounded-memory streaming — the seen-set grows O(distinct emitted rows) /// with full normalized `Binding` signatures as keys, exactly like a diff --git a/fluree-db-transact/src/current_facts.rs b/fluree-db-transact/src/current_facts.rs index 6c92220401..f7307e83ca 100644 --- a/fluree-db-transact/src/current_facts.rs +++ b/fluree-db-transact/src/current_facts.rs @@ -22,7 +22,7 @@ //! An intent names a stored fact when it holds the same term: value, exact //! datatype, and language tag compared case-insensitively. A fact read from //! the persisted index is also named by its index key, which keeps less -//! than some terms carry (see [`IndexKeys`]). +//! than some terms carry (see [`SlotMatcher`]). //! //! ## The read //! @@ -50,6 +50,7 @@ use fluree_db_core::{ use fluree_db_ledger::LedgerState; use fluree_db_query::BinaryRangeProvider; use std::collections::{HashMap, HashSet}; +use std::hash::{Hash, Hasher}; /// Seek-versus-walk break-even for a graph's novelty. Slots are answered by /// one filtered walk over the graph's novelty when @@ -62,9 +63,9 @@ pub const NOVELTY_WALK_RATIO: usize = 16; /// Where a stored fact's surviving op was read from. /// /// Index rows are decodes, and some decodes are lossy — a big integer's -/// declared XSD subtype is not stored, so it decodes as `xsd:integer` (and -/// as `xsd:decimal` in a query row). The origin tells the resolver which -/// facts it may name by index key ([`IndexKeys`]). +/// declared XSD subtype is not stored, so it decodes as `xsd:integer`. The +/// origin tells the resolver which facts it may name by index key +/// ([`SlotMatcher`]). #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Origin { /// The persisted index. @@ -170,8 +171,8 @@ impl ListFree { pub(crate) struct ResolveStats { /// Witnessed rows retracted as decoded, with no read. pub(crate) witnessed: usize, - /// Witnessed rows on a list-bearing ledger, given the stored list - /// position of the first matching entry. + /// Witnessed rows on a list-bearing ledger resolved to the stored facts + /// holding their term. pub(crate) enriched: usize, /// Intents matched to a stored fact. pub(crate) matched: usize, @@ -224,36 +225,108 @@ fn same_term(stored: &Flake, intent: &Flake) -> bool { && lang_matches(lang_of(stored), lang_of(intent)) } -/// The stored fact an intent names, if any, where `names(n, fact)` says -/// whether `facts[n]` holds the intent's term. An intent with a list -/// position names that position. One without names the first list entry -/// holding the term, in index order, else the plain value: the contract -/// hydration had, under which a value asserted at N list positions loses -/// exactly one entry per distinct DELETE row. -fn match_stored<'f>( - facts: &'f [StoredFact], +/// Which of `candidates` (positions of facts that hold an intent's term or +/// key, in SPOT order) the intent names. An intent with a list position +/// names that position. One without names the first list entry, else the +/// plain value: the contract hydration had, under which a value asserted at +/// N list positions loses exactly one entry per distinct DELETE row. +fn pick( + facts: &[StoredFact], + candidates: impl Iterator + Clone, intent: &Flake, - names: impl Fn(usize, &StoredFact) -> bool, -) -> Option<&'f StoredFact> { - let first = |position: &dyn Fn(Option) -> bool| { - facts - .iter() - .enumerate() - .find(|(n, f)| position(list_index_of(&f.flake)) && names(*n, f)) - .map(|(_, f)| f) - }; +) -> Option { + let position = |n: &usize| list_index_of(&facts[*n].flake); match list_index_of(intent) { - Some(i) => first(&|p| p == Some(i)), - None => first(&|p| p.is_some()).or_else(|| first(&|p| p.is_none())), + Some(i) => candidates.clone().find(|n| position(n) == Some(i)), + None => candidates + .clone() + .find(|n| position(n).is_some()) + .or_else(|| candidates.clone().find(|n| position(n).is_none())), + } +} + +/// A hash of a flake's term, consistent with [`same_term`]: value, exact +/// datatype and the language tag lowercased. +fn term_hash(f: &Flake) -> u64 { + let mut h = rustc_hash::FxHasher::default(); + f.o.hash(&mut h); + f.dt.hash(&mut h); + match lang_of(f) { + Some(lang) => lang + .bytes() + .for_each(|b| b.to_ascii_lowercase().hash(&mut h)), + None => 0xffu8.hash(&mut h), } + h.finish() +} + +/// The ledger's binary index provider, the source of index keys. +fn binary_provider(ledger: &LedgerState) -> Option<&BinaryRangeProvider> { + ledger + .snapshot + .range_provider + .as_ref() + .and_then(|rp| rp.as_any().downcast_ref::()) } -/// The stored fact an intent names by term identity. -fn match_term<'f>(facts: &'f [StoredFact], intent: &Flake) -> Option<&'f StoredFact> { - match_stored(facts, intent, |_, f| same_term(&f.flake, intent)) +/// One slot's facts, indexed once so an intent finds its candidates by hash +/// rather than by scanning the slot (a delete of every value of a wide slot +/// was quadratic). +struct SlotLookup { + /// Positions of the facts holding each term ([`term_hash`]), SPOT order. + by_term: HashMap>, + /// Some fact of the slot is a list entry. + has_entries: bool, + /// Some list entry was read from the index. + has_indexed_entries: bool, + /// Some fact was read from the index. + has_indexed: bool, + /// Positions of the facts read from the index, by index key, built on + /// first use. + by_key: Option>>, + /// Facts a witnessed row has already retracted. + retracted: Vec, } -/// Names facts read from the persisted index by their index key. +impl SlotLookup { + fn new(facts: &[StoredFact]) -> Self { + let mut by_term: HashMap> = HashMap::with_capacity(facts.len()); + let (mut has_entries, mut has_indexed_entries, mut has_indexed) = (false, false, false); + for (n, f) in facts.iter().enumerate() { + by_term.entry(term_hash(&f.flake)).or_default().push(n); + let entry = list_index_of(&f.flake).is_some(); + let indexed = f.origin == Origin::Index; + has_entries |= entry; + has_indexed_entries |= entry && indexed; + has_indexed |= indexed; + } + Self { + by_term, + has_entries, + has_indexed_entries, + has_indexed, + by_key: None, + retracted: vec![false; facts.len()], + } + } + + /// Positions of the facts holding `intent`'s term, in SPOT order. + fn term_candidates<'a>( + &'a self, + facts: &'a [StoredFact], + intent: &'a Flake, + ) -> impl Iterator + Clone + 'a { + self.by_term + .get(&term_hash(intent)) + .into_iter() + .flatten() + .copied() + .filter(move |&n| same_term(&facts[n].flake, intent)) + } +} + +/// Matches intents against the facts of their slots: by term, and, for a +/// fact read from the persisted index, by index key. /// /// The index keeps less than some terms carry: a big integer keeps no XSD /// subtype, a `dateTime` or `time` keeps microseconds. An indexed fact's @@ -262,54 +335,53 @@ fn match_term<'f>(facts: &'f [StoredFact], intent: &Flake) -> Option<&'f StoredF /// stored fact to the index, which merges every retraction by that key; so /// an intent whose key is an indexed fact's key names that fact, and the /// resolver retracts the fact as read. -struct IndexKeys<'l> { +struct SlotMatcher<'l> { provider: Option<&'l BinaryRangeProvider>, - /// Per slot, computed on first use. - slot_keys: Vec>, + /// Per slot, built on first use. + lookups: Vec>, } -/// The index key `(o_type, o_key)` of each fact of a slot that was read -/// from the index; `None` for a novelty fact, or one the index cannot key. -type SlotKeys = Vec>; - -impl<'l> IndexKeys<'l> { +impl<'l> SlotMatcher<'l> { fn new(ledger: &'l LedgerState, slots: usize) -> Self { Self { - provider: ledger - .snapshot - .range_provider - .as_ref() - .and_then(|rp| rp.as_any().downcast_ref::()), - slot_keys: vec![None; slots], + provider: binary_provider(ledger), + lookups: (0..slots).map(|_| None).collect(), } } - /// Per fact of slot `slot` (in graph `g_id`), whether it was read from - /// the index and has `intent`'s key. `None` when no fact can: no index, - /// no indexed fact in the slot, or no key for the intent (its value is - /// in no persisted dictionary, so no indexed row holds it). - fn hits( + fn lookup(&mut self, slot: usize, facts: &[StoredFact]) -> &mut SlotLookup { + self.lookups[slot].get_or_insert_with(|| SlotLookup::new(facts)) + } + + /// Positions of the facts of slot `slot` (in graph `g_id`) read from the + /// index under `intent`'s key, in SPOT order. `None` when no fact can + /// be: no index, no indexed fact in the slot, or no key for the intent + /// (its value is in no persisted dictionary, so no indexed row holds it). + fn key_candidates( &mut self, slot: usize, g_id: GraphId, facts: &[StoredFact], intent: &Flake, - ) -> Option> { + ) -> Option<&[usize]> { let provider = self.provider?; - if !facts.iter().any(|f| f.origin == Origin::Index) { + let lookup = self.lookups[slot].get_or_insert_with(|| SlotLookup::new(facts)); + if !lookup.has_indexed { return None; } let key = provider.persisted_object_key(g_id, intent)?; - let keys = self.slot_keys[slot].get_or_insert_with(|| { - facts - .iter() - .map(|f| match f.origin { - Origin::Index => provider.persisted_object_key(g_id, &f.flake), - Origin::Novelty => None, - }) - .collect() + let by_key = lookup.by_key.get_or_insert_with(|| { + let mut by_key: HashMap<(u16, u64), Vec> = HashMap::new(); + for (n, f) in facts.iter().enumerate() { + if f.origin == Origin::Index { + if let Some(k) = provider.persisted_object_key(g_id, &f.flake) { + by_key.entry(k).or_default().push(n); + } + } + } + by_key }); - Some(keys.iter().map(|k| *k == Some(key)).collect()) + by_key.get(&key).map(Vec::as_slice) } } @@ -354,7 +426,7 @@ impl<'l> CurrentFacts<'l> { let rm = RangeMatch::subject_predicate(slot.s.clone(), slot.p.clone()); let mut flakes = self.base(slot.g_id, rm).await?; novelty.collect_slot(self.ledger, slot, self.to_t, &mut flakes); - out.push(self.resolve(flakes, slot.g_sid.as_ref())); + out.push(self.resolve(flakes, slot.g_id, slot.g_sid.as_ref())); } Ok(out) } @@ -380,7 +452,7 @@ impl<'l> CurrentFacts<'l> { self.to_t, &mut |f| flakes.push(f.clone()), ); - Ok(self.resolve(flakes, g_sid)) + Ok(self.resolve(flakes, g_id, g_sid)) } /// Every current fact of graph `g_id`, graph Sid stamped. @@ -429,15 +501,16 @@ impl<'l> CurrentFacts<'l> { /// /// - A witnessed row on a list-free ledger (`list_free`) is retracted as /// decoded, with no read: the WHERE just matched that stored fact. - /// - A witnessed row on a list-bearing ledger reads its slot for the one - /// thing a WHERE binding does not carry, the list position: it takes - /// the position of the first stored list entry holding the same term - /// (or, for an indexed entry, the same index key), and otherwise stays - /// as decoded. + /// - A witnessed row on a list-bearing ledger reads its slot, since a + /// WHERE binding does not carry a list position. The witnessing WHERE + /// triple matched every stored fact holding the row's term, so the row + /// retracts each of them as stored (list entries, by term or, for an + /// indexed entry, by index key, and a plain value), each once however + /// many rows name it; a row with no such fact stays as decoded. /// - Every other intent — a template constant, `DELETE DATA`, a value /// bound by another pattern — is matched against the stored facts of - /// its slot ([`match_stored`]): by term, else, for a fact read from - /// the index, by index key ([`IndexKeys`]). The fact is retracted as + /// its slot ([`pick`]): by term, else, for a fact read from + /// the index, by index key ([`SlotMatcher`]). The fact is retracted as /// read. An intent that names no stored fact stages nothing; with /// `unmatched`, it is also appended there as instantiated, for the /// caller's modify-policy check. @@ -486,43 +559,56 @@ impl<'l> CurrentFacts<'l> { } stats.slot_reads = slots.len(); let facts = self.of_slots(&slots).await?; - let mut keys = IndexKeys::new(self.ledger, slots.len()); + let mut matcher = SlotMatcher::new(self.ledger, slots.len()); for (intent, idx) in pending.into_iter().zip(pending_slots) { let slot_facts = &facts[idx]; let g_id = slots[idx].g_id; let t = intent.flake.t; if intent.witnessed { - let mut flake = intent.flake; - if list_index_of(&flake).is_none() { - let is_entry = |f: &StoredFact| list_index_of(&f.flake).is_some(); - let entry = slot_facts - .iter() - .find(|f| is_entry(f) && same_term(&f.flake, &flake)) - .or_else(|| { - // Most witnessed rows are not list entries: no - // key encoding unless an indexed entry could match. - slot_facts - .iter() - .any(|f| is_entry(f) && f.origin == Origin::Index) - .then_some(())?; - let hits = keys.hits(idx, g_id, slot_facts, &flake)?; - let n = (0..slot_facts.len()) - .find(|&n| hits[n] && is_entry(&slot_facts[n]))?; - Some(&slot_facts[n]) - }); - if let Some(entry) = entry { - flake.m = entry.flake.m.clone(); + let flake = intent.flake; + let lookup = matcher.lookup(idx, slot_facts); + if list_index_of(&flake).is_none() && lookup.has_entries { + let is_entry = |n: &usize| list_index_of(&slot_facts[*n].flake).is_some(); + let indexed_entries = lookup.has_indexed_entries; + let mut held: Vec = lookup.term_candidates(slot_facts, &flake).collect(); + // Most witnessed rows are not list entries: no key + // encoding unless an indexed entry could match. + if indexed_entries { + if let Some(keyed) = matcher.key_candidates(idx, g_id, slot_facts, &flake) { + for n in keyed.iter().copied().filter(is_entry) { + if !held.contains(&n) { + held.push(n); + } + } + } + } + if !held.is_empty() { stats.enriched += 1; + let retracted = &mut matcher.lookup(idx, slot_facts).retracted; + for n in held { + if !std::mem::replace(&mut retracted[n], true) { + out.push(slot_facts[n].clone().retract(t)); + } + } + continue; } } out.push(Retraction(flake)); - } else if let Some(stored) = match_term(slot_facts, &intent.flake).or_else(|| { - let hits = keys.hits(idx, g_id, slot_facts, &intent.flake)?; - match_stored(slot_facts, &intent.flake, |n, _| hits[n]) + } else if let Some(n) = { + let lookup = matcher.lookup(idx, slot_facts); + pick( + slot_facts, + lookup.term_candidates(slot_facts, &intent.flake), + &intent.flake, + ) + } + .or_else(|| { + let candidates = matcher.key_candidates(idx, g_id, slot_facts, &intent.flake)?; + pick(slot_facts, candidates.iter().copied(), &intent.flake) }) { stats.matched += 1; - out.push(stored.clone().retract(t)); + out.push(slot_facts[n].clone().retract(t)); } else { stats.phantom_intents += 1; if let Some(unmatched) = unmatched.as_deref_mut() { @@ -551,13 +637,35 @@ impl<'l> CurrentFacts<'l> { /// Resolve base rows plus novelty ops to the current facts, stamping /// the graph and the origin. - fn resolve(&self, flakes: Vec, g_sid: Option<&Sid>) -> Vec { + /// + /// Ops merge by fact identity, but the index and queries merge by index + /// key: an index row is dropped when a novelty op of another identity + /// has its key (a revert of an indexed big integer retracts it under its + /// declared subtype, which the row does not keep). Otherwise the row + /// would stay current here after queries stopped seeing it. + fn resolve(&self, flakes: Vec, g_id: GraphId, g_sid: Option<&Sid>) -> Vec { if flakes.is_empty() { return Vec::new(); } let index_t = self.ledger.snapshot.t; + let provider = binary_provider(self.ledger); + let novelty_keys: HashSet<(u16, u64)> = match provider { + Some(p) if flakes.iter().any(|f| f.t <= index_t) => flakes + .iter() + .filter(|f| f.t > index_t) + .filter_map(|f| p.persisted_object_key(g_id, f)) + .collect(), + _ => HashSet::new(), + }; resolve_current_flakes(flakes, IndexType::Spot) .into_iter() + .filter(|f| { + novelty_keys.is_empty() + || f.t > index_t + || !provider + .and_then(|p| p.persisted_object_key(g_id, f)) + .is_some_and(|k| novelty_keys.contains(&k)) + }) .map(|flake| stored(flake, g_sid, index_t)) .collect() } diff --git a/fluree-db-transact/src/parse/rdf_text/locate.rs b/fluree-db-transact/src/parse/rdf_text/locate.rs index ee4d853d92..1421634df6 100644 --- a/fluree-db-transact/src/parse/rdf_text/locate.rs +++ b/fluree-db-transact/src/parse/rdf_text/locate.rs @@ -230,13 +230,18 @@ fn skip_statement(tokens: &[Token], mut i: usize) -> usize { /// Scan a block's contents from `i` (just past its `{`). Returns the index /// of its `}` and whether its last statement omits the terminating dot. fn scan_block(tokens: &[Token], mut i: usize) -> Result<(usize, bool)> { - let mut depth = 0i32; + // Openers not yet closed, innermost last: a `}` or the end of the text + // reached with one open is reported at that opener, not further on. + let mut open: Vec<&Token> = Vec::new(); let mut last: Option<&TokenKind> = None; while i < tokens.len() { let tok = &tokens[i]; match &tok.kind { TokenKind::Eof => break, - TokenKind::RBrace if depth == 0 => { + TokenKind::RBrace => { + if let Some(opener) = open.last() { + return Err(unclosed(opener, "before the graph block's '}'")); + } let needs_dot = last.is_some_and(|k| !matches!(k, TokenKind::Dot)); return Ok((i, needs_dot)); } @@ -252,15 +257,39 @@ fn scan_block(tokens: &[Token], mut i: usize) -> Result<(usize, bool)> { "a prefix, base or version directive cannot appear inside a graph block", )); } - kind => depth = (depth + nesting(kind)).max(0), + kind => match nesting(kind) { + 1 => open.push(tok), + -1 => { + open.pop(); + } + _ => {} + }, } last = Some(&tok.kind); i += 1; } + if let Some(opener) = open.last() { + return Err(unclosed(opener, "before the end of the document")); + } let at = tokens.last().map_or(0, |t| t.end as usize); Err(locate_error(at, "unclosed graph block: expected '}'")) } +/// An opener inside a graph block that is still open at `context`. +fn unclosed(opener: &Token, context: &str) -> TransactError { + let (open, close) = match opener.kind { + TokenKind::LBracket => ("[", "]"), + TokenKind::LParen => ("(", ")"), + TokenKind::ReifiedTripleStart => ("<<", ">>"), + TokenKind::TripleTermStart => ("<<(", ")>>"), + _ => ("{|", "|}"), + }; + locate_error( + opener.start as usize, + format!("unclosed '{open}' in a graph block: expected '{close}' {context}"), + ) +} + /// Whether `text` holds at least one graph block, read by the locator: a /// token pass, no parse. For callers that must tell TriG from Turtle before /// choosing a lane. @@ -355,4 +384,32 @@ mod tests { assert!(e.contains(needle), "{doc}: {e}"); } } + + /// A bracket left open inside a block is reported where it opens, not at + /// the next block (as a nested block) or at the end (as an unclosed block). + #[test] + fn an_unclosed_bracket_in_a_block_is_reported_where_it_opens() { + for (doc, opener, needle) in [ + ( + "GRAPH { [ 1 } GRAPH { }", + "[", + "unclosed '[' in a graph block: expected ']' before the graph block's '}'", + ), + ( + "GRAPH { ( 1 2 }", + "(", + "unclosed '(' in a graph block: expected ')' before the graph block's '}'", + ), + ( + "GRAPH { [ 1", + "[", + "unclosed '[' in a graph block: expected ']' before the end of the document", + ), + ] { + let e = locate(doc).expect_err("should be refused").to_string(); + let at = doc.find(opener).expect("opener"); + assert!(e.contains(needle), "{doc}: {e}"); + assert!(e.contains(&format!("position {at}")), "{doc}: {e}"); + } + } } diff --git a/fluree-db-transact/src/stage.rs b/fluree-db-transact/src/stage.rs index 17257478e6..15b60abde4 100644 --- a/fluree-db-transact/src/stage.rs +++ b/fluree-db-transact/src/stage.rs @@ -1118,7 +1118,34 @@ pub async fn stage_with_graph_delta( "transaction staging completed" ); - let graph_delta = ledger_graph_delta(&ledger, &txn.write_graphs); + let mut graph_delta = ledger_graph_delta(&ledger, &txn.write_graphs); + // A new graph that only a DELETE names is not registered when no + // staged fact lands in it, so a delete of facts the graph cannot hold + // commits nothing, as it does in the default graph. A graph an INSERT + // template names stays registered even when the WHERE matched + // nothing; CREATE GRAPH and a sync target register one on purpose. + if !txn.delete_templates.is_empty() && txn.sync_graph.is_none() { + let landed: HashSet<&Sid> = flakes.iter().filter_map(|f| f.g.as_ref()).collect(); + let inserted: HashSet<&str> = txn + .insert_templates + .iter() + .filter_map(|t| match &t.graph { + TemplateGraph::Iri(iri) => Some(iri.as_ref()), + TemplateGraph::Default | TemplateGraph::Var(_) => None, + }) + .collect(); + graph_delta.retain(|_, iri| { + ledger + .snapshot + .graph_registry + .graph_id_for_iri(iri) + .is_some() + || inserted.contains(iri.as_str()) + || graph_sids + .get(iri.as_str()) + .is_some_and(|sid| landed.contains(sid)) + }); + } Ok(( StagedLedger::new(ledger, flakes, &reverse_graph)?, ns_registry, @@ -1420,6 +1447,11 @@ async fn stage_graph_mgmt( // Ledger g_id -> graph Sid, for every named graph our flakes touch; // becomes the reverse routing map for novelty application / policy. let mut graph_sids: HashMap = HashMap::new(); + // Under a non-root policy, a re-homed fact the destination already + // holds stages nothing but is judged as a restatement, as in `stage` + // (`hidden_restatements`): (the stored fact's retraction, the assertion). + let policed = options.policy_ctx.is_some_and(|p| !p.wrapper().is_root()); + let mut restated: Vec<(Flake, Flake)> = Vec::new(); match op { GraphMgmtOp::Clear(target) => { @@ -1602,6 +1634,14 @@ async fn stage_graph_mgmt( .iter() .map(|f| flake_content(f.flake())) .collect(); + let dest_held: HashMap = if policed { + dest_facts + .iter() + .map(|f| (flake_content(f.flake()), f.clone().retract(new_t).into_flake())) + .collect() + } else { + HashMap::new() + }; // O5: re-homing rewrites the `f:reifiesGraph` anchor per src/ // dest graph (see the assert loop below), so an @@ -1756,10 +1796,13 @@ async fn stage_graph_mgmt( } // Assert re-homed source facts not already present in the - // destination. + // destination; a present one is a restatement. for a in rehomed { - if !dest_contents.contains(&flake_content(&a)) { + let content = flake_content(&a); + if !dest_contents.contains(&content) { flakes.push(a); + } else if let Some(held) = dest_held.get(&content) { + restated.push((held.clone(), a)); } } @@ -1787,10 +1830,22 @@ async fn stage_graph_mgmt( let reverse_graph = build_reverse_graph_lookup(&graph_sids); - // Policy enforcement (skipped for root), identical to `stage`. + // Policy enforcement (skipped for root), identical to `stage`: so + // whether a transfer into a graph the identity cannot view succeeds + // does not depend on what that graph already holds. if let Some(policy) = options.policy_ctx { if !policy.wrapper().is_root() { - enforce_modify_policies(&flakes, policy, &ledger, options.tracker, &reverse_graph) + let hidden = + hidden_restatements(&ledger, policy, restated, &reverse_graph, options.tracker) + .await?; + let judged: std::borrow::Cow<'_, [Flake]> = if hidden.is_empty() { + std::borrow::Cow::Borrowed(&flakes) + } else { + let mut all = flakes.clone(); + all.extend(hidden); + std::borrow::Cow::Owned(all) + }; + enforce_modify_policies(&judged, policy, &ledger, options.tracker, &reverse_graph) .await?; } } @@ -1917,8 +1972,9 @@ fn subjects_with_prior_rows( /// from, not a field on the flake — while a named-graph transaction's own /// flakes carry `g: Some(sid)`. Any comparison or decode that reads `g` has to /// put it back first, or an indexed named-graph bundle silently fails to line -/// up with the transaction that is editing it. `scan_graph_flakes` has always -/// done this; every other scan of a reifier's own facts needs it too. +/// up with the transaction that is editing it. [`CurrentFacts`] stamps every +/// fact it reads ([`whole_graph_facts`] included); a scan of a reifier's own +/// facts outside it needs the same. fn stamp_graph(flakes: &mut [Flake], g_sid: Option<&Sid>) { for f in flakes { f.g = g_sid.cloned(); @@ -2057,8 +2113,7 @@ async fn enforce_single_target_reifiers( // without this stamp an indexed named-graph bundle never matches // the txn's keys: a re-assert would not collapse, and the net // bundle would mix `None` and `Some` and decode as - // `MixedFlakeGraphs`. `scan_graph_flakes` stamps for the same - // reason. + // `MixedFlakeGraphs`. `CurrentFacts` stamps for the same reason. stamp_graph(&mut current, g_sid.as_ref()); let mut net: HashMap = HashMap::new(); for f in current {