LLM reference segmenter and citation engine

.github/workflows/benchmark.yml

@@ -164,13 +164,19 @@ jobs:
             --baseline-only
 
       - name: Start sciencebeam-parser
+        env:
+          # An llm profile reads this; every other profile ignores it. Passed as a
+          # bare `-e NAME` so the value never appears in a command line, and so an
+          # unset secret leaves it unset in the container rather than empty.
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
         run: |
           PROFILE_ENV=()
           if [ -n "${{ env.BENCHMARK_PROFILE }}" ]; then
             PROFILE_ENV=(-e "SCIENCEBEAM_PARSER__PROFILE=${{ env.BENCHMARK_PROFILE }}")
           fi
           docker run -d --name sciencebeam-parser -p 8080:8070 \
             -e SCIENCEBEAM_PARSER__PRELOAD_ON_STARTUP=true \
+            -e OPENROUTER_API_KEY \
             "${PROFILE_ENV[@]}" ${{ steps.image_tag.outputs.value }}
 
       - name: Wait for parser
@@ -182,7 +188,10 @@ jobs:
             sleep 5
           done
           docker logs sciencebeam-parser >&2
-          echo "Parser never became healthy" >&2; exit 1
+          echo "Parser never became healthy" >&2
+          echo "For an llm profile, PRELOAD_ON_STARTUP validates the endpoint and" >&2
+          echo "model id at startup, so a missing OPENROUTER_API_KEY fails here." >&2
+          exit 1
 
       - name: Run benchmark
         env:

Makefile

@@ -13,6 +13,12 @@ NOT_SLOW_PYTEST_ARGS = -m 'not slow'
 
 SCIENCEBEAM_PARSER_PORT = 8080
 
+# Phoenix renders the llm engine's OpenInference spans. 6006 serves both its UI
+# and its OTLP collector.
+# Phoenix is the collector used in development; the engine only knows OTLP.
+PHOENIX_PORT = 6006
+OTEL_EXPORTER_OTLP_ENDPOINT = http://localhost:$(PHOENIX_PORT)
+
 # Seconds to wait for the parser API on startup. Cold starts re-download pdfalto
 # + GROBID lexicons, so allow several minutes.
 API_WAIT_TIMEOUT ?= 300
@@ -109,7 +115,8 @@ dev-install:
 		--dev \
 		--extra cpu \
 		--extra delft \
-		--extra cv
+		--extra cv \
+		--extra telemetry
 
 
 dev-venv: venv-create dev-install
@@ -175,6 +182,14 @@ dev-start:
 		$(PYTHON) -m sciencebeam_parser.service.server --port=$(SCIENCEBEAM_PARSER_PORT)
 
 
+# The host parser with tracing pointed at the collector. Which profile is active
+# is separate and set the usual way, with SCIENCEBEAM_PARSER__PROFILE. Start the
+# collector first with docker-start-telemetry.
+dev-start-with-telemetry:
+	OTEL_EXPORTER_OTLP_ENDPOINT=$(OTEL_EXPORTER_OTLP_ENDPOINT) \
+	$(MAKE) dev-start
+
+
 dev-start-debug:
 	FLASK_ENV=development \
 	SCIENCEBEAM_PARSER__LOGGING__HANDLERS__LOG_FILE__LEVEL=DEBUG \
@@ -426,6 +441,24 @@ docker-logs:
 	$(DOCKER_COMPOSE) logs -f
 
 
+# Phoenix renders the llm engine's OpenInference spans. Behind a compose profile,
+# so a plain docker-start does not bring an observability server up with it.
+docker-start-telemetry:
+	$(DOCKER_COMPOSE) --profile telemetry up -d phoenix
+	@echo "Phoenix UI: $(OTEL_EXPORTER_OTLP_ENDPOINT)"
+
+
+# stop + rm rather than `down`, which would take the rest of the stack with it.
+# The named volume survives either way, so traces persist across restarts.
+docker-stop-telemetry:
+	$(DOCKER_COMPOSE) --profile telemetry stop phoenix
+	$(DOCKER_COMPOSE) --profile telemetry rm -f phoenix
+
+
+docker-logs-telemetry:
+	$(DOCKER_COMPOSE) --profile telemetry logs -f phoenix
+
+
 docker-end-to-end-pdfalto: docker-start-and-wait-for-api
 	$(DOCKER_DEV_RUN) curl --fail --show-error --silent \
 		--form "file=@$(EXAMPLE_PDF_DOCUMENT);filename=$(EXAMPLE_PDF_DOCUMENT)" \

benchmarks/run.py

@@ -5,7 +5,7 @@
 import subprocess
 import time
 from pathlib import Path
-from typing import Iterable, List, Optional, Tuple, Union
+from typing import Iterable, List, Optional, Sequence, Tuple, Union
 
 import httpx
 import yaml
@@ -292,6 +292,39 @@ def run_benchmark(  # pylint: disable=too-many-arguments,too-many-positional-arg
         LOGGER.info("Only one summary available; skipping comparison report")
 
 
+# Corpora that must not be sent to a third-party model. Provider zero-retention
+# does not cover the intermediary, and these manuscripts are not redistributable,
+# so an LLM profile and one of these together is refused rather than warned about.
+RESTRICTED_CORPORA_FOR_LLM = frozenset({"plos-manuscripts"})
+
+
+def check_llm_profile_corpora(
+    config: dict, profile: Optional[str], include: Optional[Sequence[str]]
+) -> None:
+    if not profile:
+        return
+    sequence_model_profiles = config.get("sequence_model_profiles")
+    if not isinstance(sequence_model_profiles, dict):
+        return
+    profile_models = sequence_model_profiles.get(profile)
+    if not isinstance(profile_models, dict):
+        return
+    uses_llm = any(
+        isinstance(model_config, dict) and model_config.get("engine") == "llm"
+        for model_config in profile_models.values()
+    )
+    if not uses_llm:
+        return
+    restricted = sorted(RESTRICTED_CORPORA_FOR_LLM.intersection(include or ()))
+    if restricted:
+        raise SystemExit(
+            f"refusing to run profile {profile!r}, which uses the llm engine, over "
+            f"{', '.join(restricted)}: those manuscripts are not redistributable and "
+            "provider zero-retention does not cover the intermediary. Drop the corpus "
+            "or point the engine at a self-hosted endpoint."
+        )
+
+
 def main(argv=None) -> None:
     parser = argparse.ArgumentParser(
         description="Run benchmark: fetch gold, run baselines, predict, score, compare"
@@ -330,6 +363,8 @@ def main(argv=None) -> None:
     with open(args.config, encoding="utf-8") as f:
         config = yaml.safe_load(f)
 
+    check_llm_profile_corpora(config, args.profile, args.include_corpus)
+
     if args.predictions_repo:
         store: PredictionsStore = RepoPredictionsStore(Path(args.predictions_repo))
     else:

benchmarks/tests/llm_guard_test.py

@@ -0,0 +1,44 @@
+import pytest
+
+from benchmarks.run import check_llm_profile_corpora
+
+
+CONFIG = {
+    'sequence_model_profiles': {
+        'grobid_crf_0_9_0': {
+            'citation': {'engine': 'wapiti', 'path': 'x'},
+        },
+        'llm_references': {
+            'citation': {'engine': 'llm', 'task': 'citation'},
+        },
+    },
+}
+
+
+class TestCheckLlmProfileCorpora:
+    def test_should_allow_a_crf_profile_with_the_private_corpus(self):
+        check_llm_profile_corpora(CONFIG, 'grobid_crf_0_9_0', ['plos-manuscripts'])
+
+    def test_should_allow_an_llm_profile_without_the_private_corpus(self):
+        check_llm_profile_corpora(CONFIG, 'llm_references', ['biorxiv'])
+
+    def test_should_allow_an_llm_profile_with_no_opt_in_corpora(self):
+        check_llm_profile_corpora(CONFIG, 'llm_references', None)
+
+    def test_should_refuse_an_llm_profile_with_the_private_corpus(self):
+        with pytest.raises(SystemExit, match='not redistributable'):
+            check_llm_profile_corpora(CONFIG, 'llm_references', ['plos-manuscripts'])
+
+    def test_should_name_the_profile_and_corpus_when_refusing(self):
+        with pytest.raises(SystemExit) as excinfo:
+            check_llm_profile_corpora(
+                CONFIG, 'llm_references', ['biorxiv', 'plos-manuscripts']
+            )
+        assert 'llm_references' in str(excinfo.value)
+        assert 'plos-manuscripts' in str(excinfo.value)
+
+    def test_should_ignore_an_unknown_profile(self):
+        check_llm_profile_corpora(CONFIG, 'nonexistent', ['plos-manuscripts'])
+
+    def test_should_ignore_no_profile(self):
+        check_llm_profile_corpora(CONFIG, None, ['plos-manuscripts'])