fix(index): every xsd:double value is indexable

docs/concepts/datatypes.md

@@ -72,6 +72,57 @@ Supported numeric types:
 - **xsd:double**: 64-bit floating point
 - **xsd:float**: 32-bit floating point
 
+#### Special Values: INF, -INF and NaN
+
+`xsd:double` and `xsd:float` include three special values, written `INF` (or
+`+INF`), `-INF` and `NaN`. A numeral too large for `xsd:double`, such as
+`"1e400"^^xsd:double`, is `INF` or `-INF`, as XSD 1.1 specifies. Other
+spellings such as `inf`, `Infinity` or `nan` are not values of these datatypes
+(see [Text That Is Not a Value of Its Datatype](#text-that-is-not-a-value-of-its-datatype)).
+
+`xsd:float` values are stored at double precision, so a numeral beyond the
+float range but within the double range, such as `"3.4e39"^^xsd:float`, keeps
+its value when written. The cast `xsd:float("3.4e39")` gives `INF`, so the two
+differ for such numerals.
+
+One exception keeps existing indexes reading as before. Bulk import in earlier
+versions stored `INF`, `-INF` and `NaN` in the index as the text `inf`, `-inf`
+and `NaN`. Under `xsd:double` or `xsd:float`:
+
+- Arithmetic, `SUM`, `AVG`, casts to `xsd:double` and `xsd:float`, and a
+  comparison inside a larger expression, such as `FILTER(?v > 1 || ?v < -1)`
+  or `BIND(?v > 1 AS ?b)`, read `inf` and `-inf` as `INF` and `-INF`, and
+  `NaN` as NaN.
+- To `isNumeric`, `ORDER BY`, `MIN`/`MAX`, and a `FILTER` that is a single
+  comparison, such as `FILTER(?v > 1)`, the text stays text.
+- One answer differs from earlier versions: `SUM` and `AVG` over `NaN` text
+  are `NaN`, as SPARQL specifies, where they were unbound.
+- The text `inf` cannot be told apart from a literal `"inf"^^xsd:double`
+  written today, which is not a value, so arithmetic and these aggregates read
+  that literal as `INF` too. `Infinity` and other spellings stay text.
+
+A full reindex of an index that bulk import built stores those values as
+numbers.
+
+Queries treat the special values as SPARQL specifies:
+
+- `INF` is greater, and `-INF` less, than every other number. Fluree compares
+  numbers exactly, so an integer or decimal beyond the double range is still
+  below `INF`. That is a choice: XPath would convert `10^400` to a double,
+  making it equal to `INF`.
+- No comparison holds for `NaN`: `=`, `<`, `>`, `<=` and `>=` are false and
+  `!=` is true, even between two NaNs. So `FILTER(?v = ?v)` drops NaN values.
+- A `NaN` literal in a pattern, as in `?s ex:v "NaN"^^xsd:double`, matches
+  stored NaN values, because patterns match terms.
+- `SUM` and `AVG` are `NaN` when a member is `NaN`, and `INF + -INF` is `NaN`.
+- `DISTINCT` and `GROUP BY` treat every NaN as one value.
+- `ORDER BY` places `NaN` after `INF` in ascending order. SPARQL leaves that
+  position to the implementation. `MIN` and `MAX` follow `ORDER BY`, so `MAX`
+  is `NaN` when a NaN is present.
+
+JSON-LD query results carry them as the strings `"INF"`, `"-INF"` and
+`"NaN"`, since JSON numbers cannot express them.
+
 ### Boolean Type
 
 ```json

fluree-db-api/Cargo.toml

@@ -364,6 +364,18 @@ name = "it_geo_search_routing"
 path = "tests/it_geo_search_routing.rs"
 required-features = ["native"]
 
+# Own binary: toggles the fast-path kill switch and asserts routing.
+# Own binary: asserts which index build path ran through span capture.
+[[test]]
+name = "it_index_unindexable"
+path = "tests/it_index_unindexable.rs"
+required-features = ["native"]
+
+[[test]]
+name = "it_xsd_double_fast_paths"
+path = "tests/it_xsd_double_fast_paths.rs"
+required-features = ["native"]
+
 # Own binary: toggles the process-global fast-path kill switch.
 [[test]]
 name = "it_datetime_component_presence"
@@ -767,6 +779,10 @@ name = "it_fast_stats_1391_regression"
 path = "tests/it_fast_stats_1391_regression.rs"
 required-features = ["native"]
 
+[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+# Test hooks: halting a ledger, and building incrementally without fallback.
+fluree-db-indexer = { path = "../fluree-db-indexer", features = ["test-utils"] }
+
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 moka = { workspace = true }
 fluree-db-transact = { path = "../fluree-db-transact", features = ["import"] }

fluree-db-api/src/admin.rs

@@ -295,6 +295,13 @@ pub struct IndexStatusResult {
     pub pending_min_t: Option<i64>,
     /// Last error message (if any)
     pub last_error: Option<String>,
+    /// The last build stopped on committed data it cannot index, so commits
+    /// start no build until an explicit index request
+    /// ([`crate::Fluree::trigger_index`]), a reindex, dropping the ledger, a
+    /// restart, or (where catch-up sweeps run) a newer index from another
+    /// process; `last_error` says why. See
+    /// [`fluree_db_indexer::IndexStatusSnapshot::halted`].
+    pub halted: bool,
 }
 
 // =============================================================================
@@ -1887,15 +1894,22 @@ impl crate::Fluree {
             .ok_or_else(|| ApiError::NotFound(format!("Ledger not found: {ledger_id}")))?;
 
         // Get indexer status if available
-        let (indexing_enabled, phase, pending_min_t, last_error) = match &self.indexing_mode {
+        let (indexing_enabled, phase, pending_min_t, last_error, halted) = match &self.indexing_mode
+        {
             IndexingMode::Background(handle) => {
                 if let Some(status) = handle.status(&ledger_id).await {
-                    (true, status.phase, status.pending_min_t, status.last_error)
+                    (
+                        true,
+                        status.phase,
+                        status.pending_min_t,
+                        status.last_error,
+                        status.halted,
+                    )
                 } else {
-                    (true, IndexPhase::Idle, None, None)
+                    (true, IndexPhase::Idle, None, None, false)
                 }
             }
-            IndexingMode::Disabled => (false, IndexPhase::Idle, None, None),
+            IndexingMode::Disabled => (false, IndexPhase::Idle, None, None, false),
         };
 
         Ok(IndexStatusResult {
@@ -1906,6 +1920,7 @@ impl crate::Fluree {
             phase,
             pending_min_t,
             last_error,
+            halted,
         })
     }
 
@@ -1978,7 +1993,8 @@ impl crate::Fluree {
             timeout_ms = ?timeout_ms,
             "Queueing index request"
         );
-        let completion = handle.trigger(&ledger_id, min_t).await;
+        // An explicit request: it also retries a halted ledger.
+        let completion = handle.trigger_explicit(&ledger_id, min_t).await;
 
         if let Some(status) = handle.status(&ledger_id).await {
             info!(
@@ -2371,6 +2387,11 @@ impl crate::Fluree {
         self.publisher()?
             .publish_index_allow_equal(&ledger_id, index_result.index_t, &index_result.root_id)
             .await?;
+        // The committed data indexed, so a halt the background indexer
+        // recorded for this ledger no longer holds.
+        if let IndexingMode::Background(handle) = &self.indexing_mode {
+            handle.clear_halt(&ledger_id).await;
+        }
 
         // Reindex can replace a damaged root at the same index t. A cached
         // handle may otherwise keep its old graph registry indefinitely.

fluree-db-api/src/cross_ledger/shapes_materializer.rs

@@ -294,7 +294,7 @@ fn flake_value_to_lexical(
         FlakeValue::String(s) => s.clone(),
         FlakeValue::Boolean(b) => b.to_string(),
         FlakeValue::Long(n) => n.to_string(),
-        FlakeValue::Double(f) => f.to_string(),
+        FlakeValue::Double(f) => fluree_graph_ir::canonical_xsd_double(*f),
         FlakeValue::BigInt(b) => b.to_string(),
         FlakeValue::Decimal(d) => d.to_string(),
         FlakeValue::DateTime(v) => v.to_string(),

fluree-db-api/src/tx.rs

@@ -2563,6 +2563,22 @@ impl crate::Fluree {
             return;
         }
         if let IndexingMode::Background(handle) = &self.indexing_mode {
+            // A halted ledger starts no build on its own: say why writes are
+            // refused, where an operator sees the refusals. (No background
+            // indexer exists on wasm32.)
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                if let Some(status) = handle.status(ledger_id).await.filter(|s| s.halted) {
+                    tracing::error!(
+                        ledger_id = %ledger_id,
+                        base_t,
+                        error = status.last_error.as_deref().unwrap_or_default(),
+                        "novelty at max: writes are refused because indexing of this ledger \
+                         is halted on committed data it cannot index"
+                    );
+                    return;
+                }
+            }
             if handle.trigger_if_idle(ledger_id, base_t).await {
                 tracing::info!(
                     ledger_id = %ledger_id,

fluree-db-api/tests/fixtures/bulk-import-4.2.3/README.md

@@ -0,0 +1,10 @@
+A ledger bulk-imported by the v4.2.3 CLI from `source.ttl`, in an empty
+directory:
+
+```sh
+fluree --direct init
+fluree --direct create ex:main --from source.ttl
+```
+
+`storage/` is the project's `.fluree/storage`, without its lock files. The
+import indexes as it reads, so the ledger is fully indexed at t=1.

fluree-db-api/tests/fixtures/bulk-import-4.2.3/source.ttl

@@ -0,0 +1,8 @@
+@prefix ex: <http://example.org/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+ex:a ex:v "1.5"^^xsd:double .
+ex:b ex:v "-2.5"^^xsd:double .
+ex:inf ex:v "INF"^^xsd:double .
+ex:ninf ex:v "-INF"^^xsd:double .
+ex:nan ex:v "NaN"^^xsd:double .
+ex:finf ex:f "INF"^^xsd:float .

fluree-db-api/tests/fixtures/bulk-import-4.2.3/storage/ex/main/config/93718e432ebd8ba2f25b37a317b9f3ceb063eba00f3bf7975c2f96473f33d852.json

@@ -0,0 +1 @@
+{"rdf":"http://www.w3.org/1999/02/22-rdf-syntax-ns#","rdfs":"http://www.w3.org/2000/01/rdf-schema#","xsd":"http://www.w3.org/2001/XMLSchema#","owl":"http://www.w3.org/2002/07/owl#","sh":"http://www.w3.org/ns/shacl#","geo":"http://www.opengis.net/ont/geosparql#","ex":"http://example.org/"}

fluree-db-api/tests/fixtures/bulk-import-4.2.3/storage/ns@v2/ex/main.commits.jsonl

@@ -0,0 +1 @@
+{"t":1,"cid":"bagaybqabciqb6axoor5iz7sheyc67et6z3ukafj2p6jxjrohrbjf5uqcjpzendq"}

fluree-db-api/tests/fixtures/bulk-import-4.2.3/storage/ns@v2/ex/main.index.json

@@ -0,0 +1,9 @@
+{
+  "@context": {
+    "f": "https://ns.flur.ee/db#"
+  },
+  "f:ledgerIndex": {
+    "f:cid": "baghybqabciqkbebx4hvh66e55wu3qhhtea6ijj7zuab4kn2ceb7skjpcii7zcka",
+    "f:t": 1
+  }
+}

fluree-db-api/tests/fixtures/bulk-import-4.2.3/storage/ns@v2/ex/main.json

@@ -0,0 +1,19 @@
+{
+  "@context": {
+    "f": "https://ns.flur.ee/db#"
+  },
+  "@id": "ex:main",
+  "@type": [
+    "f:LedgerSource"
+  ],
+  "f:ledger": {
+    "@id": "ex"
+  },
+  "f:branch": "main",
+  "f:commitCid": "bagaybqabciqb6axoor5iz7sheyc67et6z3ukafj2p6jxjrohrbjf5uqcjpzendq",
+  "f:t": 1,
+  "f:status": "ready",
+  "f:defaultContextCid": "bageibqabciqjg4moimxl3c5c6jntpiyxxhz45mdd5oqa6o7xs5oc7fshh4z5quq",
+  "f:statusV": 1,
+  "f:configV": 1
+}

fluree-db-api/tests/fixtures/vector-lengths-4.2.3/README.md

@@ -0,0 +1,12 @@
+A ledger written by the v4.2.3 CLI, in a project made by `fluree --direct init`:
+
+```sh
+fluree --direct create ex
+fluree --direct insert -l ex -e '{"@context":{"ex":"http://example.org/","ex:embedding":{"@type":"@vector"}},"@graph":[{"@id":"ex:a","ex:name":"a","ex:embedding":[0.125,0.25,0.375]}]}'
+fluree --direct index ex
+fluree --direct insert -l ex -e '{"@context":{"ex":"http://example.org/","ex:embedding":{"@type":"@vector"}},"@graph":[{"@id":"ex:b","ex:name":"b","ex:embedding":[0.5,0.625]}]}'
+```
+
+`storage/` is the project's `.fluree/storage`, without its lock files: the
+ledger `ex:main`, indexed at t=1, with a two-element vector on
+`ex:embedding` at t=2 beside the three-element one the index holds.

fluree-db-api/tests/fixtures/vector-lengths-4.2.3/storage/ex/@shared/dicts/f0b21345c75f948f90c7fa7c332f94345181599c0ae64a0f5a0ac81c0326b610.dict

@@ -0,0 +1,14 @@
+{
+  "version": 1,
+  "dims": 3,
+  "dtype": "f32",
+  "normalized": false,
+  "shard_capacity": 3072,
+  "total_count": 1,
+  "shards": [
+    {
+      "cas": "bagdibqabciqhwgrtuluhaxk6iii4dr5kwi4lxgpfiy2trxnyazc32bpkzz534ky",
+      "count": 1
+    }
+  ]
+}

fluree-db-api/tests/fixtures/vector-lengths-4.2.3/storage/ex/main/index/garbage/7f970c2d85a13f6232bd48add8fde5f582650a72976a251ac3f76ff08421880c.json

@@ -0,0 +1 @@
+{"ledger_id":"ex:main","t":1,"garbage":[],"created_at_ms":1791808043284}