diff --git a/.gitignore b/.gitignore index 1f344540..92bed62e 100755 --- a/.gitignore +++ b/.gitignore @@ -10734,3 +10734,13 @@ web-app/appConfig.json web-app/stats.json appConfig.json banality_finder.py.lprof + +# macOS +.DS_Store + +# Backup files +*.bak +in-and-out/ +lib/core/src/compareNgrams/compareNgrams +my_config.ini +text_setup.sh diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..ed7d51a3 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11.15 diff --git a/README.md b/README.md index e62d62ba..77f0439a 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -"Nous ne faisons que nous entregloser" Montaigne wrote famously in his Essais... Since all we do is glose over what's already been written, we may as well build a tool to detect these intertextual relationships... +"Nous ne faisons que nous entregloser" Montaigne wrote famously in his Essais... Since all we do is gloss over what's already been written, we may as well build a tool to detect these intertextual relationships... + # TextPAIR (Pairwise Alignment for Intertextual Relations) @@ -30,6 +31,35 @@ The recommended install is to build your own Docker image and run TextPAIR insid If you do run into the issue where the web server does not respond, restart the web server with the following command: `/var/lib/text-pair/api_server/web_server.sh &` +### macOS bare-metal installation (experimental) + +TextPAIR officially supports 64-bit Linux; the Docker method above is the recommended path for production use. For local development and research runs on a Mac (Apple Silicon or Intel), an opt-in installer sets up the full sequence-alignment pipeline natively, without Docker: + +```console +./install_bare_metal_mac.sh +``` + +**Prerequisite:** [Homebrew](https://brew.sh). The script stops with instructions if it is missing. Everything else is handled automatically: + +- installs `pyenv`, Go, and `lz4` via Homebrew if absent, then installs Python 3.11 via pyenv and creates the environment TextPAIR runs in +- installs the TextPAIR Python package into that environment +- patches the installed PhiloLogic dependency's `line_count.py` (its non-lz4 code path is broken, and BSD `wc` output differs from GNU) +- builds a native `compareNgrams` binary from source with Go — the prebuilt binaries ship as Linux ELF executables and cannot run on macOS — and installs it to `/usr/local/bin` (this step asks for your password) +- seeds `~/.text-pair/global_settings.ini` and a starter `my_config.ini` (copied from `config/sa_config.ini`) if you do not already have them + +Then edit `my_config.ini` (at minimum, set `source_file_path` to your corpus directory) and run: + +```console +textpair --config=my_config.ini --skip_web_app --output_path=/tmp/textpair-out --workers=8 my_run_name +``` + +Known limitations of the macOS path: + +- Avoid corpus and output paths containing spaces (a PhiloLogic limitation; note that iCloud-synced folders live under a path with spaces — copy corpora to `/tmp` or similar first). +- The PostgreSQL credentials in `~/.text-pair/global_settings.ini` are only needed if you drop `--skip_web_app` to build the web application. + +This mode was developed for the corpus-scale alignment runs in Tarah Wheeler's (2026) DPhil thesis at the University of Oxford. + ### Manual installation If you wish to install TextPAIR on a host machine, note that TextPair will only run on 64 bit Linux, see below. diff --git a/config/global_settings.ini b/config/global_settings.ini index 18771883..1ace740a 100755 --- a/config/global_settings.ini +++ b/config/global_settings.ini @@ -1,4 +1,10 @@ -## DATABASE SETTINGS ## +[WEB_APP] +## Directory where generated web apps are written; irrelevant if you always pass --skip_web_app +web_app_path = /var/www/html/text-pair +## Base URL where the TextPAIR API is served +api_server = http://localhost/text-pair-api + +[DATABASE] database_name = textpair database_user = textpair database_password = \ No newline at end of file diff --git a/docs/ubuntu_installation.md b/docs/ubuntu_installation.md index 1fe6aab6..c55422f8 100755 --- a/docs/ubuntu_installation.md +++ b/docs/ubuntu_installation.md @@ -45,7 +45,7 @@ sudo vim /etc/postgresql/10/pg_hba.conf Note that the path the pg_hba.conf may vary based on your postgres version. -Fill in the database info in text-pair config: `sudo vim /etc/text-pair/config/global_settings.ini` +Fill in the database info in text-pair config: `sudo vim /etc/text-pair/global_settings.ini` ### Create webspace with proper permissions diff --git a/install_bare_metal_mac.sh b/install_bare_metal_mac.sh new file mode 100755 index 00000000..db7408ba --- /dev/null +++ b/install_bare_metal_mac.sh @@ -0,0 +1,317 @@ +#!/bin/bash +# TextPAIR Mac Bare-Metal Install Script +# For use without Docker, without web app +# Forked from ARTFL-Project/text-pair +# +# Usage: ./install-mac.sh + +set -e + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo -e "${GREEN}TextPAIR Mac Bare-Metal Installer${NC}" +echo "==================================" +echo "" + +# ============================================================================= +# PATCH PHILOLOGIC WC -L (macOS compatibility) +# ============================================================================= +patch_philologic_wc() { + echo "Patching PhiloLogic line_count.py for macOS..." + + # Find the philologic installation + local philologic_path=$("$PYTHON_BIN" -c "import philologic; print(philologic.__path__[0])" 2>/dev/null) + + if [ -z "$philologic_path" ]; then + echo -e "${YELLOW} PhiloLogic not yet installed, will patch after pip install${NC}" + return 0 + fi + + local line_count_file="${philologic_path}/utils/line_count.py" + + if [ -f "$line_count_file" ]; then + # Check if already patched + if grep -q 'wc -l < {file_path}' "$line_count_file"; then + echo " Already patched" + else + # Rewrite the entire file - the upstream non-lz4 branch is broken + # (runs cut on empty stdin instead of wc -l on the file) + cat > /tmp/_line_count_patch.py << 'PATCH' +#!/usr/bin/env python3 +"""Count number of lines in a file using subprocess module.""" +import subprocess +def count_lines(file_path, lz4=False): + """Count number of lines in a file.""" + if lz4: + cmd = f"lz4 -dc {file_path} | wc -l" + else: + cmd = f"wc -l < {file_path}" + process = subprocess.run(cmd, shell=True, text=True, capture_output=True) + count = int(process.stdout.strip()) + return count +PATCH + sudo cp /tmp/_line_count_patch.py "$line_count_file" + rm /tmp/_line_count_patch.py + echo " Patched: rewrote line_count.py (upstream non-lz4 branch was broken)" + fi + else + echo -e "${YELLOW} line_count.py not found at expected path${NC}" + fi + + echo "" +} + +# ============================================================================= +# ARCHITECTURE CHECK +# ============================================================================= +check_architecture() { + local arch=$(uname -m) + echo "Checking architecture..." + echo " Detected: $arch" + + if [ "$arch" == "arm64" ]; then + BINARY_ARCH="aarch64" + echo " Binary: aarch64 (Apple Silicon)" + elif [ "$arch" == "x86_64" ]; then + BINARY_ARCH="x86_64" + echo " Binary: x86_64 (Intel)" + else + echo -e "${RED}Unsupported architecture: $arch${NC}" + exit 1 + fi + echo "" +} + +# ============================================================================= +# DEPENDENCY CHECK +# ============================================================================= +check_dependencies() { + echo "Checking dependencies..." + + # Homebrew (required to auto-install pyenv/Go below) + if command -v brew &> /dev/null; then + echo " Homebrew: found" + else + echo -e "${RED} ERROR: Homebrew not found. TextPAIR needs it to install pyenv and the Go toolchain.${NC}" + echo -e "${YELLOW} Install Homebrew first, then re-run this script:${NC}" + echo ' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"' + exit 1 + fi + + # pyenv (guarantees a consistent, correct Python 3.11 regardless of whatever + # python3 happens to be on the ambient PATH - relying on system/Homebrew python3 + # directly is what let a Python 3.9 slip past the old version check below) + if command -v pyenv &> /dev/null; then + echo " pyenv: found" + else + echo " pyenv: not found, installing with Homebrew..." + brew install pyenv + echo " pyenv installed" + fi + + # Resolve the latest available Python 3.11.x via pyenv, install it if missing, and + # pin this directory to it (writes .python-version) so python3/pip here always + # resolve to 3.11, regardless of the system's default python3. + TARGET_PY_VERSION=$(pyenv install --list | grep -E '^\s*3\.11\.[0-9]+$' | tail -1 | xargs) + if [ -z "$TARGET_PY_VERSION" ]; then + echo -e "${RED} ERROR: could not find a Python 3.11.x version via pyenv${NC}" + exit 1 + fi + if ! pyenv versions --bare | grep -qx "$TARGET_PY_VERSION"; then + echo " Installing Python $TARGET_PY_VERSION via pyenv (this can take a few minutes)..." + pyenv install "$TARGET_PY_VERSION" + fi + pyenv local "$TARGET_PY_VERSION" + PYTHON_BIN="$(pyenv root)/versions/$TARGET_PY_VERSION/bin/python3" + echo " Python: $("$PYTHON_BIN" --version) (pyenv, pinned to this directory via .python-version)" + + # Confirm pyenv's shims are wired into the user's shell so `textpair`/`python3` keep + # resolving to this pinned version in future terminal sessions, not just this script run. + local shell_rc="" + case "$SHELL" in + */zsh) shell_rc="$HOME/.zshrc" ;; + */bash) shell_rc="$HOME/.bash_profile" ;; + *) shell_rc="$HOME/.profile" ;; + esac + if [ -f "$shell_rc" ] && grep -q 'pyenv init' "$shell_rc"; then + echo " pyenv shell integration: found in $shell_rc" + else + echo -e "${YELLOW} pyenv shell integration not found in $shell_rc${NC}" + echo -e "${YELLOW} Add this line to $shell_rc, then restart your terminal:${NC}" + echo ' eval "$(pyenv init -)"' + fi + + # ripgrep (optional but recommended) + if command -v rg &> /dev/null; then + echo " ripgrep: $(rg --version | head -1)" + else + echo -e "${YELLOW} ripgrep: not found (optional, install with: brew install ripgrep)${NC}" + fi + + # Homebrew (required to auto-install Go below) + if command -v brew &> /dev/null; then + echo " Homebrew: found" + else + echo -e "${RED} ERROR: Homebrew not found. TextPAIR needs it to install the Go toolchain (used to build compareNgrams).${NC}" + echo -e "${YELLOW} Install Homebrew first, then re-run this script:${NC}" + echo ' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"' + exit 1 + fi + + # Go (required to build compareNgrams; the bundled binaries are Linux-only) + if command -v go &> /dev/null; then + echo " Go: $(go version)" + else + echo " Go: not found, installing with Homebrew..." + brew install go + echo " Go installed" + fi + + # lz4 CLI (used to merge alignment result batches) + if command -v lz4 &> /dev/null; then + echo " lz4: found" + else + echo " lz4: not found, installing with Homebrew..." + brew install lz4 + fi + + echo "" +} + +# ============================================================================= +# INSTALL +# ============================================================================= +install_textpair() { + echo "Installing TextPAIR..." + + # Install textpair_llm first (local dependency) + if [ -d "lib/textpair_llm" ]; then + echo " Installing textpair_llm..." + "$PYTHON_BIN" -m pip install -e lib/textpair_llm/. --break-system-packages --quiet + fi + + # Install main package + echo " Installing textpair..." + "$PYTHON_BIN" -m pip install -e lib/. --break-system-packages + + echo "" +} + +# ============================================================================= +# SEED GLOBAL SETTINGS (user-level path, no sudo/root needed on macOS) +# ============================================================================= +setup_global_settings() { + local target="$HOME/.text-pair/global_settings.ini" + echo "Setting up $target..." + + if [ -f "$target" ] || [ -f /etc/text-pair/global_settings.ini ]; then + echo " Already exists, leaving as-is" + else + mkdir -p "$HOME/.text-pair" + cp config/global_settings.ini "$target" + echo " Seeded from config/global_settings.ini" + echo -e "${YELLOW} Edit $target with your actual PostgreSQL credentials (only needed if you drop --skip_web_app)${NC}" + fi + + echo "" +} + +# ============================================================================= +# SCAFFOLD STARTER CORPUS CONFIG +# ============================================================================= +scaffold_config() { + local target="my_config.ini" + echo "Setting up $target..." + + if [ -f "$target" ]; then + echo " Already exists, leaving as-is ($target)" + else + cp config/sa_config.ini "$target" + echo " Seeded from config/sa_config.ini" + echo -e "${YELLOW} Edit $target and set source_file_path to your corpus directory before running textpair${NC}" + fi + + echo "" +} + +# ============================================================================= +# INSTALL BINARY +# ============================================================================= +install_binary() { + echo "Installing compareNgrams binary..." + + # The prebuilt binaries under lib/core/binary are Linux ELF executables (upstream only + # targets Linux) and cannot run on macOS at all, even when the CPU architecture matches. + # Build a native Mach-O binary from source instead. Go is guaranteed present at this point + # (installed by check_dependencies if it was missing). + echo " Building compareNgrams from source with Go..." + (cd lib/core/src/compareNgrams && go build -o /tmp/compareNgrams_build .) + sudo cp /tmp/compareNgrams_build /usr/local/bin/compareNgrams + rm -f /tmp/compareNgrams_build + sudo chmod +x /usr/local/bin/compareNgrams + echo " Built and installed to /usr/local/bin/compareNgrams" + + echo "" +} + +# ============================================================================= +# VERIFY +# ============================================================================= +verify_install() { + echo "Verifying installation..." + + # Check the pyenv-pinned interpreter directly, since `command -v textpair` only + # works if pyenv's shims are already wired into this shell's PATH (see the + # shell-integration note printed by check_dependencies). + local textpair_bin="$(pyenv root)/versions/$TARGET_PY_VERSION/bin/textpair" + if [ -x "$textpair_bin" ]; then + echo -e "${GREEN} textpair command found ($textpair_bin)${NC}" + else + echo -e "${RED} ERROR: textpair command not found at $textpair_bin${NC}" + exit 1 + fi + if ! command -v textpair &> /dev/null; then + echo -e "${YELLOW} Note: textpair isn't on PATH in this shell yet - see the pyenv shell integration note above${NC}" + fi + + if command -v compareNgrams &> /dev/null; then + echo -e "${GREEN} compareNgrams binary found${NC}" + else + echo -e "${RED} ERROR: compareNgrams binary not found${NC}" + exit 1 + fi + + echo "" + echo -e "${GREEN}Installation complete!${NC}" + echo "" + echo "Usage:" + echo " textpair --config=my_config.ini --skip_web_app --output_path=/tmp/textpair-out --workers=4 alignment_name" + echo "" + echo "Notes:" + echo " - Edit my_config.ini first and set source_file_path to your corpus directory" + echo " - ulimit is automatically increased on macOS (no manual fix needed)" + echo " - Use absolute paths in my_config.ini for source_file_path" + echo " - Avoid paths with spaces (copy corpus to /tmp if on iCloud)" + echo " - Input files should be TEI XML format" + echo " - Downloading a Spacy model (python -m spacy download ) is only needed if" + echo " you enable POS/entity filtering or spacy-based lemmatization in my_config.ini" +} + +# ============================================================================= +# MAIN +# ============================================================================= +main() { + check_architecture + check_dependencies + install_textpair + patch_philologic_wc + setup_global_settings + scaffold_config + install_binary + verify_install +} + +main "$@" \ No newline at end of file diff --git a/lib/core/src/compareNgrams/main.go b/lib/core/src/compareNgrams/main.go index fad85e27..f846cbd2 100755 --- a/lib/core/src/compareNgrams/main.go +++ b/lib/core/src/compareNgrams/main.go @@ -603,6 +603,9 @@ func saveAlignmentConfig(config *matchingParams) { os.MkdirAll(config.outputPath, 0755) configOutput, err := os.Create(filepath.Join(config.outputPath, "alignment_config.ini")) configOutput.WriteString("## Alignment Parameters ##\n\n") + // banalNgrams and oneWayMatching were removed from matchingParams; listing + // them here made FieldByName return the zero Value, so every run's + // alignment_config.ini recorded "" for both. matchingParameters := []string{ "matchingWindowSize", "maxGap", @@ -611,11 +614,9 @@ func saveAlignmentConfig(config *matchingParams) { "minimumMatchingNgramsInWindow", "minimumMatchingNgramsInDocs", "contextSize", - "banalNgrams", "mergeOnByteDistance", "mergeOnNgramDistance", "passageDistanceMultiplier", - "oneWayMatching", "duplicateThreshold", "sourceBatch", "targetBatch", diff --git a/lib/pyproject.toml b/lib/pyproject.toml index 2091bd7e..8bdb61a0 100644 --- a/lib/pyproject.toml +++ b/lib/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "orjson", "text_preprocessing @ git+https://github.com/ARTFL-Project/text-preprocessing@v1.1.2#egg=text_preprocessing", "fastapi==0.110.3", - "psycopg2", + "psycopg2-binary", "gunicorn", "uvicorn", "uvloop", diff --git a/lib/textpair/__main__.py b/lib/textpair/__main__.py index fd309e1b..3d2faa0d 100644 --- a/lib/textpair/__main__.py +++ b/lib/textpair/__main__.py @@ -6,10 +6,12 @@ import shutil import subprocess import sys +from shlex import quote import psycopg2 from . import create_web_app, get_config, parse_files, run_vsa +from .parse_config import read_global_config from .passage_classifier import classify_passages from .sequence_alignment import ( Ngrams, @@ -91,8 +93,7 @@ def build_graph_and_labels(alignments_file: str, embedding_model: str, llm_param def delete_database(dbname: str) -> None: - global_config = configparser.ConfigParser() - global_config.read("/etc/text-pair/global_settings.ini") + global_config = read_global_config() conn = psycopg2.connect( user=global_config["DATABASE"]["database_user"], password=global_config["DATABASE"]["database_password"], @@ -112,7 +113,7 @@ def delete_database(dbname: str) -> None: cursor.execute(f"DROP TABLE IF EXISTS {dbname}___groups") print("done") print(f"Deleting {dbname} web app directory...", end="") - os.system(f"rm -rf {global_config['WEB_APP']['web_app_path']}/{dbname}") + shutil.rmtree(os.path.join(global_config['WEB_APP']['web_app_path'], dbname), ignore_errors=True) print("done") print(f"\nDeletion of database {dbname} complete.") @@ -190,14 +191,15 @@ async def run_alignment(params): params.paths["target"]["ngram_output_path"] = params.paths["source"]["ngram_output_path"] result_batch_path = os.path.join(params.output_path, "results/result_batches") if os.path.exists(result_batch_path): - os.system(f"rm -rf {result_batch_path}") + shutil.rmtree(result_batch_path, ignore_errors=True) + # Path-valued arguments are shell-quoted so paths containing spaces work. command = f"""compareNgrams \ - --output_path={params.output_path}/results \ + --output_path={quote(f"{params.output_path}/results")} \ --threads={params.workers} \ - --source_files={params.paths["source"]["ngram_output_path"]}/ngrams \ - --target_files={params.paths["target"]["ngram_output_path"]}/ngrams \ - --source_metadata={params.paths["source"]["metadata_path"]} \ - --target_metadata={params.paths["target"]["metadata_path"]} \ + --source_files={quote(f'{params.paths["source"]["ngram_output_path"]}/ngrams')} \ + --target_files={quote(f'{params.paths["target"]["ngram_output_path"]}/ngrams')} \ + --source_metadata={quote(params.paths["source"]["metadata_path"])} \ + --target_metadata={quote(params.paths["target"]["metadata_path"])} \ --sort_by={params.matching_params["sort_by"]} \ --source_batch={params.matching_params["source_batch"]} \ --target_batch={params.matching_params["target_batch"]} \ @@ -213,24 +215,30 @@ async def run_alignment(params): --merge_passages_on_ngram_distance={params.matching_params["merge_passages_on_ngram_distance"]} \ --passage_distance_multiplier={params.matching_params["passage_distance_multiplier"]} \ --debug={str(params.debug).lower()} \ - --ngram_index={params.matching_params["ngram_index"]}""" + --ngram_index={quote(params.matching_params["ngram_index"])}""" results_file = f"{params.output_path}/results/alignments.jsonl.lz4" if os.path.exists(results_file): - os.system(f"rm -rf {results_file}") + os.remove(results_file) if params.debug: print(f"Running alignment with following arguments:\n{' '.join(command.split())}") os.system(command) if len(os.listdir(result_batch_path)) == 1: filename = os.listdir(result_batch_path)[0] - os.system(f"mv {result_batch_path}/{filename} {results_file} && rm -rf {result_batch_path}") + shutil.move(os.path.join(result_batch_path, filename), results_file) + shutil.rmtree(result_batch_path, ignore_errors=True) else: print( "Merging alignments into one file (this may take a while)... ", end="", flush=True, ) - merge_command = f"find {result_batch_path} -type f | sort -V | xargs lz4cat --rm | lz4 -q > {results_file}; rm -rf {result_batch_path}" + # NUL-delimited so batch paths containing spaces survive the pipeline. + merge_command = ( + f"find {quote(result_batch_path)} -type f -print0 | sort -zV | " + f"xargs -0 lz4cat --rm | lz4 -q > {quote(results_file)}" + ) os.system(merge_command) + shutil.rmtree(result_batch_path, ignore_errors=True) print("done.") count = get_count(os.path.join(params.output_path, "results/count.txt")) @@ -471,12 +479,33 @@ async def main(): elif params.matching_params["matching_algorithm"] == "vsa": await run_vsa_similarity(params) - def run(): """Sync entry point for console_scripts.""" import asyncio + import platform + + # macOS defaults to 256 open file descriptors, which is too low for + # PhiloLogic's sort/merge operations on large corpora; raise it for + # the duration of the run and restore it afterwards. + original_soft = None + hard = None + if platform.system() == "Darwin": + import resource + + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft < 4096: + new_soft = min(hard, 10240) + resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft, hard)) + original_soft = soft + print(f"[macOS] Raised open file limit: {soft} -> {new_soft}") + try: + asyncio.run(main()) + finally: + if original_soft is not None: + import resource - asyncio.run(main()) + resource.setrlimit(resource.RLIMIT_NOFILE, (original_soft, hard)) + print(f"[macOS] Restored open file limit: {original_soft}") if __name__ == "__main__": diff --git a/lib/textpair/parse_config.py b/lib/textpair/parse_config.py index a565aa05..0a99f9bb 100755 --- a/lib/textpair/parse_config.py +++ b/lib/textpair/parse_config.py @@ -7,6 +7,20 @@ from collections import defaultdict, namedtuple from typing import Any +# Checked in order; later paths override earlier ones on a per-key basis. +# The user-level path lets Mac bare-metal installs avoid writing to /etc as root. +GLOBAL_CONFIG_SEARCH_PATHS = [ + "/etc/text-pair/global_settings.ini", + os.path.expanduser("~/.text-pair/global_settings.ini"), +] + + +def read_global_config() -> configparser.ConfigParser: + """Read global_settings.ini from the first location(s) that exist. Missing files are silently skipped.""" + config = configparser.ConfigParser() + config.read(GLOBAL_CONFIG_SEARCH_PATHS) + return config + def _is_philo_db(path: str) -> bool: """Auto-detect whether a path is a PhiloLogic database.""" @@ -44,10 +58,11 @@ def __getattr__(self, attr): def __parse_config(self): """Read config file and store into 4 dicts for each phase of the alignment""" - global_config = configparser.ConfigParser() - global_config.read("/etc/text-pair/global_settings.ini") - self.web_app_config["web_application_directory"] = global_config["WEB_APP"]["web_app_path"] - self.web_app_config["api_server"] = global_config["WEB_APP"]["api_server"] + global_config = read_global_config() + web_app_section = global_config["WEB_APP"] if global_config.has_section("WEB_APP") else {} + web_app_path = web_app_section.get("web_app_path", "") + self.web_app_config["web_application_directory"] = web_app_path + self.web_app_config["api_server"] = web_app_section.get("api_server", "") config = configparser.ConfigParser() config.read(self.__cli_args["config"]) self.web_app_config["source_url"] = config["TEXT_SOURCES"]["source_url"] @@ -60,12 +75,12 @@ def __parse_config(self): self.web_app_config["target_philo_db_path"] = target_file_path or source_file_path else: self.web_app_config["source_philo_db_path"] = os.path.join( - global_config["WEB_APP"]["web_app_path"], + web_app_path, self.__cli_args["dbname"], "source_data", ) self.web_app_config["target_philo_db_path"] = os.path.join( - global_config["WEB_APP"]["web_app_path"], + web_app_path, self.__cli_args["dbname"], "target_data", ) diff --git a/lib/textpair/sequence_alignment/banality_finder.py b/lib/textpair/sequence_alignment/banality_finder.py index a1569bbd..8e3e0ecd 100644 --- a/lib/textpair/sequence_alignment/banality_finder.py +++ b/lib/textpair/sequence_alignment/banality_finder.py @@ -56,8 +56,9 @@ def banality_auto_detect( ): """Detect banalities automatically based on frequent ngram over-representation""" # Count number of ngrams to keep - output = subprocess.check_output(["wc", "-l", common_ngrams_file]).decode("utf-8") - total_ngrams = int(output.split(" ", maxsplit=1)[0]) + # PATCHED_FOR_MACOS - pure Python line count (macOS wc has leading spaces) + with open(common_ngrams_file, "rb") as _f: + total_ngrams = sum(1 for _ in _f) top_ngrams = floor(total_ngrams * proportion / 100) common_ngrams = set() @@ -96,7 +97,7 @@ def banality_auto_detect( alignment["banality"] = False # Always write to main file with banality flag set output_file.write(orjson.dumps(alignment) + b"\n") # type: ignore - os.system(f"rm {filepath} && mv {filepath}.temp.lz4 {filepath}") + os.replace(f"{filepath}.temp.lz4", filepath) return banalities_found @@ -135,7 +136,7 @@ def phrase_matcher(filepath: str, banality_phrases_path: str, count: Optional[in filtered_passages.write(line) # type: ignore if banality is False: output_file.write(line) # type: ignore - os.system(f"rm {filepath} && mv {filepath}.keep.lz4 {filepath}") + os.replace(f"{filepath}.keep.lz4", filepath) print("done") return passages_filtered @@ -168,7 +169,7 @@ def separate_banalities(filepath: str, count: Optional[int]) -> int: else: output_file.write(line) # type: ignore - os.system(f"rm {filepath} && mv {filepath}.keep.lz4 {filepath}") + os.replace(f"{filepath}.keep.lz4", filepath) return banalities_separated diff --git a/lib/textpair/sequence_alignment/generate_ngrams.py b/lib/textpair/sequence_alignment/generate_ngrams.py index a997bddf..c9dcc752 100755 --- a/lib/textpair/sequence_alignment/generate_ngrams.py +++ b/lib/textpair/sequence_alignment/generate_ngrams.py @@ -3,9 +3,12 @@ import configparser import os +import shutil import sqlite3 from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed from glob import glob +from shlex import quote from typing import Any, Dict, List, Tuple import orjson @@ -91,16 +94,18 @@ def generate( files = [file_path] else: files = glob(os.path.join(file_path, "*")) - os.system(f"rm -rf {output_path}/ngrams") - os.system(f"rm -rf {output_path}/ngrams_in_order") - os.system(f"mkdir -p {output_path}/ngrams") + # Use shutil/os rather than shelling out: unquoted paths passed to the + # shell break (dangerously, for rm -rf) on paths containing spaces. + shutil.rmtree(os.path.join(output_path, "ngrams"), ignore_errors=True) + shutil.rmtree(os.path.join(output_path, "ngrams_in_order"), ignore_errors=True) + os.makedirs(os.path.join(output_path, "ngrams"), exist_ok=True) if self.debug: - os.system(f"mkdir {output_path}/debug") - os.system(f"mkdir -p {output_path}/metadata") - os.system(f"mkdir -p {output_path}/index") - os.system(f"mkdir -p {output_path}/config") - os.system(f"mkdir -p {output_path}/temp") - os.system(f"mkdir -p {output_path}/ngrams_in_order") + os.makedirs(os.path.join(output_path, "debug"), exist_ok=True) + os.makedirs(os.path.join(output_path, "metadata"), exist_ok=True) + os.makedirs(os.path.join(output_path, "index"), exist_ok=True) + os.makedirs(os.path.join(output_path, "config"), exist_ok=True) + os.makedirs(os.path.join(output_path, "temp"), exist_ok=True) + os.makedirs(os.path.join(output_path, "ngrams_in_order"), exist_ok=True) self.input_path = os.path.abspath(os.path.join(files[0], "../../../")) self.output_path = output_path combined_metadata: dict[str, Any] = {} @@ -122,23 +127,38 @@ def generate( ascii=self.config["ascii"], post_processing_function=self.text_to_ngram, is_philo_db=True, - workers=workers, + workers=1, progress=False, ) + # NOTE: workers=1 above keeps text_preprocessing on its serial code path (no internal + # multiprocess.Pool). We fan out across files ourselves with a ThreadPoolExecutor instead: + # multiprocess.Pool defaults to fork() on macOS, and forking again right after the + # preceding PhiloLogic parse stage's own Pool tears down reliably deadlocks on modern + # macOS (bpo-33725). Threads sidestep this entirely since they never fork. philo_type_count = self.count_texts(files[0]) with tqdm(total=philo_type_count, leave=False) as pbar: - for local_metadata in preprocessor.process_texts(files, progress=False): - combined_metadata.update(local_metadata) # type: ignore - pbar.update() + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [ + executor.submit(lambda f=f: list(preprocessor.process_texts([f], progress=False))) + for f in files + ] + for future in as_completed(futures): + for local_metadata in future.result(): + combined_metadata.update(local_metadata) # type: ignore + pbar.update() print( "Saving ngram index and most common ngrams (this can take a while)...", flush=True, ) + # The external-sort pipeline stays in the shell on purpose (sort -S does + # the heavy lifting), but every path is shell-quoted so output paths + # containing spaces work. + q_out = quote(output_path) os.system( - rf"""for i in {output_path}/temp/*; do cat $i; done | sort -T {output_path} -S 25% | uniq -c | - sort -rn -T {output_path} -S 25% | awk '{{print $2"\t"$3}}' | tee {output_path}/index/index.tab | - awk '{{print $2}}' > {output_path}/index/most_common_ngrams.txt""" + rf"""for i in {q_out}/temp/*; do cat "$i"; done | sort -T {q_out} -S 25% | uniq -c | + sort -rn -T {q_out} -S 25% | awk '{{print $2"\t"$3}}' | tee {q_out}/index/index.tab | + awk '{{print $2}}' > {q_out}/index/most_common_ngrams.txt""" ) print("Saving metadata...") @@ -147,7 +167,7 @@ def generate( self.__dump_config(output_path) print("Cleaning up...") - os.system(f"rm -r {self.output_path}/temp") + shutil.rmtree(os.path.join(self.output_path, "temp"), ignore_errors=True) def text_to_ngram(self, text_object: Tokens) -> Dict[str, Any]: """Tranform doc to inverted index of ngrams""" @@ -157,6 +177,9 @@ def text_to_ngram(self, text_object: Tokens) -> Dict[str, Any]: for k, v in text_object.metadata.items(): if not isinstance(v, str): text_object.metadata[k] = str(v) + if "philo_id" not in text_object.metadata: + print(f"WARNING: skipping text object with no philo_id: {list(text_object.metadata.keys())}", flush=True) + return {} text_object_id = "_".join( text_object.metadata["philo_id"].split()[: PHILO_TEXT_OBJECT_LEVELS[self.config["text_object_type"]]] ) diff --git a/lib/textpair/web_loader.py b/lib/textpair/web_loader.py index 40df9699..92165bc9 100755 --- a/lib/textpair/web_loader.py +++ b/lib/textpair/web_loader.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """Web loading module""" -import configparser import json import os import re @@ -16,6 +15,8 @@ from psycopg2.extras import execute_values from tqdm import tqdm +from .parse_config import read_global_config + DEFAULT_FIELDS = { "rowid", "group_id", @@ -304,8 +305,7 @@ def load_db( """Load SQL table""" import numpy as np - config = configparser.ConfigParser() - config.read("/etc/text-pair/global_settings.ini") + config = read_global_config() database = psycopg2.connect( user=config["DATABASE"]["database_user"], password=config["DATABASE"]["database_password"], @@ -487,8 +487,7 @@ def load_db( def load_groups_file(groups_file: str, alignments_table: str, searchable_fields: list[str]): """Load the groups file into the database.""" - config = configparser.ConfigParser() - config.read("/etc/text-pair/global_settings.ini") + config = read_global_config() table_name = f"{alignments_table}_groups" with open(groups_file, encoding="utf8") as input_file: @@ -544,8 +543,7 @@ def load_groups_file(groups_file: str, alignments_table: str, searchable_fields: def generate_database_stats(table_name, algorithm): """Generate statistics for the database""" print("Generating database statistics (this could take a while)...") - config = configparser.ConfigParser() - config.read("/etc/text-pair/global_settings.ini") + config = read_global_config() database = psycopg2.connect( user=config["DATABASE"]["database_user"], password=config["DATABASE"]["database_password"],