Add FTS Sail with for ElasticSearch backend

README.md

@@ -11,6 +11,96 @@ The core idea is to use an RDF knowledge graph for describing the structure of s
 
 <img style="background-color: white; padding: 15px; width: 100%; max-width: 800px" alt="LinkedFactory semantic linking" src="docs/assets/lf-linking.svg">
 
+## Full-text search
+LinkedFactory Pod can index RDF statements for full-text search and expose them to SPARQL via **`SERVICE <fts:...>`**.
+
+Purpose:
+- index literal values for keyword search
+- keep IRIs and named graph context available for exact-match filtering
+- return optional score/snippet bindings
+- support external and internal backends through the same SPARQL shape
+
+Linkedfactory-Pod sends the changes to a search server that must expose a bulk endpoint, by default:
+- `POST {fts:endpoint}{fts:bulkPath}`
+- `Content-Type: application/json`
+
+This endpoint can be configured in linkedfactory-pod via the ttl configuration file with the `fts:endpoint` and `fts:bulkPath` properties of the `fts:FtsSail` configuration.
+
+The Key-Search can be integrated with SPARQL queries using the `SERVICE <fts:...>` syntax.
+
+The following example shows how to query for IRIs that match certain keywords and filter them based on a weight property:
+
+```sparql
+prefix fts: <fts:>
+prefix dtsc: <https://example.org/dtsc/>
+
+select ?iri ?score where {
+  service <fts:> {
+    ?iri fts:keywords "some keywords" ;
+         fts:score ?score .
+  }
+  ?iri dtsc:weight ?weight .
+  filter (?weight > 50)
+}
+```
+
+The federated service is configured through the `fts:FtsSail` .
+
+The following example shows how to configure the federated service and the FtsSail:
+
+```ttl
+@prefix rep: <http://www.openrdf.org/config/repository#>.
+@prefix sr: <http://www.openrdf.org/config/repository/sail#>.
+@prefix sail: <http://www.openrdf.org/config/sail#>.
+@prefix ns: <http://www.openrdf.org/config/sail/native#>.
+@prefix fts: <http://linkedfactory.github.io/config/sail/fts#>.
+
+<urn:enilink:data> a models:RepositoryModelSet ;
+    models:repository <urn:linkedfactory:data-repo> .
+
+<urn:linkedfactory:data-repo> a rep:Repository ;
+   rep:repositoryID "linkedfactory-data" ;
+   rep:repositoryImpl [
+      rep:repositoryType "openrdf:SailRepository" ;
+       sr:sailImpl [
+          sail:sailType "kvin:KvinSail" ;
+          sail:delegate [
+             sail:sailType "fts:FtsSail" ;
+             fts:backend "elastic" ;  # use built-in elasticsearch backend for external Elasticsearch server.
+            fts:endpoint "http://localhost:9200" ; # the endpoint of the external search server
+            fts:bulkPath "/fts/bulk" ; # the bulk endpoint path for indexing (where linkedfactory sends the changes).
+            fts:searchPath "/fts/_search" ; # the search endpoint path for querying .
+            fts:defaultLimit 100 ; # the default result size for queries.
+            fts:failOnError true ; # fail or log on backend errors.
+            sail:delegate [
+               sail:sailType "openrdf:NativeStore" ; # the actual RDF store for the data.
+               ns:tripleIndexes "cspo,cpos,spoc,posc" # the triple indexes to use for the RDF store.
+            ]
+          ]
+       ]
+    ] .
+```
+
+Backend selection:
+- `fts:backend "elastic"` uses `fts:endpoint` and an HTTP search backend.
+- `fts:backend "lucene"` can be added later as a Lucene backend.
+- `fts:backend "internal"` can be added later for in-process indexing and should ignore `fts:endpoint`.
+
+Common TTL options:
+- `fts:endpoint` search endpoint for external backends
+- `fts:searchPath` backend query path
+- `fts:bulkPath "/fts/bulk"` is the bulk endpoint path for indexing (where linkedfactory sends the changes and not related to the federated service)
+- `fts:defaultLimit` default result size
+- `fts:failOnError` fail or log on backend errors
+
+`fts:FtsSail` can also be used with in-memory stores by wrapping `openrdf:MemoryStore` instead of `NativeStore`.
+
+The bulk request format is an `operations` array containing `upsert`, `remove`, `clear`, and `clearContexts` entries. Named graph context is included per statement as `context`.
+
+Query-time endpoint override is also supported with `SERVICE <fts:http://host:9200>`, but internal backends can ignore that value.
+
+_**Note**_: the payload (JSON) of the search request used by the federated service is just now for testing and need to be discussed or agreed on. The same applies to the bulk request payload. see examples in `bundles/io.github.linkedfactory.core/src/test/resources/fts/`
+
 ## Data representation
 Formally, the triple-based data model of RDF _(S, P, O)_ is extended to a quad-based data model _(S, P, T, O)_. If named graphs are used to manage multiple RDF datasets then an additional context **C** can be introduced to extend the data model to _(C, S, P, T, O)_. We call this the **Kvin** data model.
 

bundles/io.github.linkedfactory.core/src/main/java/io/github/linkedfactory/core/rdf4j/FederatedServiceComponent.java

@@ -6,6 +6,9 @@
 import io.github.linkedfactory.core.kvin.http.KvinHttp;
 import io.github.linkedfactory.core.rdf4j.aas.AasFederatedService;
 import io.github.linkedfactory.core.rdf4j.common.BaseFederatedServiceResolver;
+import io.github.linkedfactory.core.rdf4j.fts.FtsFederatedServiceConfig;
+import io.github.linkedfactory.core.rdf4j.fts.FtsFederatedServiceResolver;
+import io.github.linkedfactory.core.rdf4j.fts.FtsSail;
 import io.github.linkedfactory.core.rdf4j.io.SPARQLResultsParquetWriterFactory;
 import io.github.linkedfactory.core.rdf4j.kvin.KvinFederatedService;
 import io.github.linkedfactory.core.rdf4j.kvin.functions.DateTimeFunction;
@@ -17,6 +20,9 @@
 import org.eclipse.rdf4j.query.algebra.evaluation.function.FunctionRegistry;
 import org.eclipse.rdf4j.query.resultio.TupleQueryResultWriterRegistry;
 import org.eclipse.rdf4j.repository.Repository;
+import org.eclipse.rdf4j.repository.sail.SailRepository;
+import org.eclipse.rdf4j.sail.Sail;
+import org.eclipse.rdf4j.sail.StackableSail;
 import org.osgi.service.component.annotations.*;
 
 import java.util.Optional;
@@ -29,6 +35,7 @@ public class FederatedServiceComponent {
 	IModelSet ms;
 	Kvin kvin;
 	AbstractFederatedServiceResolver serviceResolver;
+	AbstractFederatedServiceResolver ftsServiceResolver;
 	@Reference(cardinality = ReferenceCardinality.OPTIONAL)
 	volatile ContextProvider contextProvider;
 
@@ -57,6 +64,7 @@ void activate() {
 		if (repositoryBinding != null) {
 			final Repository repository = repositoryBinding.getProvider().get();
 			if (repository instanceof FederatedServiceResolverClient) {
+				ftsServiceResolver = new FtsFederatedServiceResolver(readFtsFederatedConfig(repository));
 				serviceResolver = new BaseFederatedServiceResolver() {
 					@Override
 					protected FederatedService createService(String serviceUrl)
@@ -73,6 +81,8 @@ protected FederatedService createService(String serviceUrl)
 							return new KvinFederatedService(new KvinHttp(url), this::getExecutorService,
 									() -> contextProvider == null ? Kvin.DEFAULT_CONTEXT : contextProvider.getContext(),
 									true);
+						} else if (serviceUrl.startsWith("fts:")) {
+							return ftsServiceResolver.getService(serviceUrl);
 						}
 						return null;
 					}
@@ -88,6 +98,10 @@ void deactivate() {
 			serviceResolver.shutDown();
 			serviceResolver = null;
 		}
+		if (ftsServiceResolver != null) {
+			ftsServiceResolver.shutDown();
+			ftsServiceResolver = null;
+		}
 	}
 
 	private Optional<String> getKvinServiceUrl(String serviceUrl) {
@@ -98,6 +112,26 @@ private Optional<String> getKvinServiceUrl(String serviceUrl) {
 		return url;
 	}
 
+	private FtsFederatedServiceConfig readFtsFederatedConfig(Repository repository) {
+		if (repository instanceof SailRepository sailRepository) {
+			FtsSail ftsSail = findSailOfType(sailRepository.getSail(), FtsSail.class);
+			if (ftsSail != null) {
+				return ftsSail.getFederatedServiceConfig();
+			}
+		}
+		return FtsFederatedServiceConfig.defaults();
+	}
+
+	private <T> T findSailOfType(Sail sail, Class<T> type) {
+		if (type.isInstance(sail)) {
+			return type.cast(sail);
+		}
+		if (sail instanceof StackableSail stackableSail && stackableSail.getBaseSail() != null) {
+			return findSailOfType(stackableSail.getBaseSail(), type);
+		}
+		return null;
+	}
+
 	@Reference
 	void setModelSet(IModelSet ms) {
 		this.ms = ms;