diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6412438..43c566e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: .venv/bin/pip install ruff pytest - name: Lint Python - run: .venv/bin/ruff check searchy/*.py + run: .venv/bin/ruff check searchy/backend/*.py - name: Test Python - run: .venv/bin/pytest searchy/tests/ -x -q + run: .venv/bin/pytest searchy/backend/tests/ -x -q diff --git a/Makefile b/Makefile index 9791091..b85f2ed 100644 --- a/Makefile +++ b/Makefile @@ -83,19 +83,19 @@ release: dmg sha ## Full release: archive → export → DMG → SHA setup: ## Set up Python venv with dependencies python3 -m venv .venv .venv/bin/pip install --upgrade pip - .venv/bin/pip install -r searchy/requirements.txt + .venv/bin/pip install -r searchy/backend/requirements.txt @echo "✓ Python env ready. Activate with: source .venv/bin/activate" # ─── Lint ───────────────────────────────────────────────── lint: ## Run linters (ruff for Python) @command -v ruff >/dev/null 2>&1 || (echo "Installing ruff..." && pip install ruff) - ruff check searchy/*.py + ruff check searchy/backend/*.py @echo "✓ Lint passed" # ─── Test ───────────────────────────────────────────────── test: ## Run Python unit tests @test -f .venv/bin/pytest || (python3 -m venv .venv && .venv/bin/pip install -q pytest) - .venv/bin/pytest searchy/tests/ -x -q + .venv/bin/pytest searchy/backend/tests/ -x -q # ─── Check (all pre-push checks) ───────────────────────── check: lint test ## Run lint + tests diff --git a/Scripts/pre-push-checks.sh b/Scripts/pre-push-checks.sh index 52c7067..4254753 100755 --- a/Scripts/pre-push-checks.sh +++ b/Scripts/pre-push-checks.sh @@ -20,19 +20,19 @@ echo "═══ Pre-push checks ═══" # 1. Syntax check echo -n "• py_compile... " -for f in searchy/*.py; do +for f in searchy/backend/*.py; do $PYTHON -m py_compile "$f" || FAILED=1 done [ $FAILED -eq 0 ] && echo "ok" || echo "FAIL" # 2. Ruff lint echo -n "• ruff... " -"$VENV/bin/ruff" check searchy/*.py --quiet || FAILED=1 +"$VENV/bin/ruff" check searchy/backend/*.py --quiet || FAILED=1 [ $FAILED -eq 0 ] && echo "ok" || echo "FAIL" # 3. Tests echo -n "• pytest... " -$PYTHON -m pytest searchy/tests/ -x -q || FAILED=1 +$PYTHON -m pytest searchy/backend/tests/ -x -q || FAILED=1 # 4. Swift build (skip with SKIP_SWIFT=1) if [ "${SKIP_SWIFT:-0}" != "1" ]; then diff --git a/pyproject.toml b/pyproject.toml index f3115dd..24f7403 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,4 +6,4 @@ line-length = 120 select = ["F", "E9", "W6"] [tool.pytest.ini_options] -testpaths = ["searchy/tests"] +testpaths = ["searchy/backend/tests"] diff --git a/searchy/ContentView.swift b/searchy/ContentView.swift index 61ee8eb..95be77e 100644 --- a/searchy/ContentView.swift +++ b/searchy/ContentView.swift @@ -1401,11 +1401,7 @@ class DirectoryManager: ObservableObject { let directories = try? JSONDecoder().decode([WatchedDirectory].self, from: data) { self.watchedDirectories = directories } else { - // Default directories - self.watchedDirectories = [ - WatchedDirectory(path: NSString(string: "~/Downloads").expandingTildeInPath), - WatchedDirectory(path: NSString(string: "~/Desktop").expandingTildeInPath, filter: "Screenshot", filterType: .startsWith) - ] + self.watchedDirectories = [] } } diff --git a/searchy/atomic_write.py b/searchy/backend/atomic_write.py similarity index 100% rename from searchy/atomic_write.py rename to searchy/backend/atomic_write.py diff --git a/searchy/clip_model.py b/searchy/backend/clip_model.py similarity index 100% rename from searchy/clip_model.py rename to searchy/backend/clip_model.py diff --git a/searchy/backend/constants.py b/searchy/backend/constants.py new file mode 100644 index 0000000..ca795b3 --- /dev/null +++ b/searchy/backend/constants.py @@ -0,0 +1,46 @@ +"""Centralized constants for the Searchy project.""" + +import os + +# ─── Paths ──────────────────────────────────────────────── +DEFAULT_DATA_DIR = os.path.join( + os.path.expanduser("~/Library/Application Support"), "searchy" +) + +# ─── Image Formats ──────────────────────────────────────── +IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.heic'} + +# ─── Directories to Skip During Indexing ────────────────── +SKIP_DIRS = { + 'site-packages', 'node_modules', 'vendor', '__pycache__', + 'env', 'venv', '.venv', 'virtualenv', + 'Library', 'Caches', 'cache', '.cache', + 'build', 'dist', 'target', '.git', '.svn', + 'DerivedData', 'xcuserdata', 'Pods', + '__MACOSX', '.Trash', '.Spotlight-V100', '.fseventsd', +} + +# ─── Indexing Defaults ──────────────────────────────────── +DEFAULT_BATCH_SIZE = 64 +DEFAULT_MAX_DIMENSION = 384 +LARGE_BATCH_SIZE = 500 # For bulk operations (duplicates, face clustering) + +# ─── Search Defaults ────────────────────────────────────── +DEFAULT_TOP_K = 20 +DEFAULT_OCR_WEIGHT = 0.3 +DEFAULT_SIMILARITY_THRESHOLD = 0.95 # For duplicate detection + +# ─── Model ──────────────────────────────────────────────── +DEFAULT_EMBEDDING_DIM = 512 +DEFAULT_MODEL_NAME = "openai/clip-vit-base-patch32" + +# ─── TTL ────────────────────────────────────────────────── +TTL_CHECK_INTERVAL = 30 # Seconds between TTL checks + +# ─── OCR ────────────────────────────────────────────────── +OCR_TEXT_PREVIEW_LENGTH = 300 + +# ─── Image Watcher ──────────────────────────────────────── +WATCHER_DEBOUNCE_DELAY = 0.5 # Seconds +WATCHER_NOTIFY_TIMEOUT = 5 # Seconds +WATCHER_INDEX_TIMEOUT = 30 # Seconds diff --git a/searchy/embedding_utils.py b/searchy/backend/embedding_utils.py similarity index 100% rename from searchy/embedding_utils.py rename to searchy/backend/embedding_utils.py diff --git a/searchy/face_recognition_service.py b/searchy/backend/face_recognition_service.py similarity index 100% rename from searchy/face_recognition_service.py rename to searchy/backend/face_recognition_service.py diff --git a/searchy/generate_embeddings.py b/searchy/backend/generate_embeddings.py similarity index 92% rename from searchy/generate_embeddings.py rename to searchy/backend/generate_embeddings.py index 6161c3d..fed5deb 100644 --- a/searchy/generate_embeddings.py +++ b/searchy/backend/generate_embeddings.py @@ -14,7 +14,6 @@ import sys import json import argparse -import re import time from PIL import Image @@ -22,6 +21,8 @@ # Import centralized model manager from clip_model import model_manager +from constants import SKIP_DIRS, IMAGE_EXTENSIONS +from utils import matches_filter, is_user_image # OCR support using macOS Vision framework try: @@ -96,45 +97,6 @@ def resize_image_for_fast_indexing(image, max_dimension=384): return image.resize((new_width, new_height), Image.LANCZOS) -def matches_filter(filename, filter_type, filter_value): - """Check if filename matches the filter criteria""" - if not filter_value or filter_type == "all": - return True - - filename_lower = filename.lower() - filter_lower = filter_value.lower() - - if filter_type == "starts-with": - return filename_lower.startswith(filter_lower) - elif filter_type == "ends-with": - return filename_lower.endswith(filter_lower) - elif filter_type == "contains": - return filter_lower in filename_lower - elif filter_type == "regex": - try: - return bool(re.search(filter_value, filename, re.IGNORECASE)) - except re.error: - return False - return True - - -# Directories to skip (system, packages, caches, etc.) -SKIP_DIRS = { - 'site-packages', 'node_modules', 'vendor', '__pycache__', - 'env', 'venv', '.venv', 'virtualenv', - 'Library', 'Caches', 'cache', '.cache', - 'build', 'dist', 'target', '.git', '.svn', - 'DerivedData', 'xcuserdata', 'Pods', - '__MACOSX', '.Trash', '.Spotlight-V100', '.fseventsd' -} - - -def is_user_image(path): - """Check if path is a user image (not system/package file).""" - if os.path.basename(path).startswith('.'): - return False - parts = path.split(os.sep) - return not any(part in SKIP_DIRS for part in parts) def index_images_with_clip(output_dir, incremental=False, new_files=None, @@ -313,7 +275,7 @@ def process_images(image_dirs, output_dir, fast_indexing=True, max_dimension=384 if file.startswith('.'): continue - if file.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp')): + if os.path.splitext(file.lower())[1] in IMAGE_EXTENSIONS: # Apply filter if specified if filter_type and filter_value: if not matches_filter(file, filter_type, filter_value): diff --git a/searchy/image_watcher.py b/searchy/backend/image_watcher.py similarity index 88% rename from searchy/image_watcher.py rename to searchy/backend/image_watcher.py index 893ced2..efa75b8 100644 --- a/searchy/image_watcher.py +++ b/searchy/backend/image_watcher.py @@ -10,7 +10,6 @@ import sys import time import json -import re import argparse import urllib.request import urllib.error @@ -18,35 +17,16 @@ from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler -# Supported image extensions -IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.heic'} +from constants import ( + IMAGE_EXTENSIONS, DEFAULT_BATCH_SIZE, DEFAULT_MAX_DIMENSION, + WATCHER_DEBOUNCE_DELAY, WATCHER_NOTIFY_TIMEOUT, WATCHER_INDEX_TIMEOUT, +) +from utils import matches_filter # Default server URL (can be overridden via --server-url) DEFAULT_SERVER_URL = "http://127.0.0.1:7860" -def matches_filter(filename, filter_type, filter_value): - """Check if filename matches the filter criteria""" - if not filter_value or filter_type == "all": - return True - - filename_lower = filename.lower() - filter_lower = filter_value.lower() - - if filter_type == "starts-with": - return filename_lower.startswith(filter_lower) - elif filter_type == "ends-with": - return filename_lower.endswith(filter_lower) - elif filter_type == "contains": - return filter_lower in filename_lower - elif filter_type == "regex": - try: - return bool(re.search(filter_value, filename, re.IGNORECASE)) - except re.error: - return False - return True - - def call_server_notify(server_url, file_path): """Notify the server that a new image was detected (for immediate display).""" url = f"{server_url}/notify-new-image" @@ -65,7 +45,7 @@ def call_server_notify(server_url, file_path): method='POST' ) - with urllib.request.urlopen(req, timeout=5) as response: + with urllib.request.urlopen(req, timeout=WATCHER_NOTIFY_TIMEOUT) as response: result = json.loads(response.read().decode('utf-8')) return result @@ -76,7 +56,8 @@ def call_server_notify(server_url, file_path): def call_server_index(server_url, data_dir, files, fast_indexing=True, - max_dimension=384, batch_size=64): + max_dimension=DEFAULT_MAX_DIMENSION, + batch_size=DEFAULT_BATCH_SIZE): """Call the server's /index endpoint to index files.""" url = f"{server_url}/index" @@ -97,7 +78,7 @@ def call_server_index(server_url, data_dir, files, fast_indexing=True, method='POST' ) - with urllib.request.urlopen(req, timeout=30) as response: + with urllib.request.urlopen(req, timeout=WATCHER_INDEX_TIMEOUT) as response: result = json.loads(response.read().decode('utf-8')) return result @@ -112,7 +93,8 @@ def call_server_index(server_url, data_dir, files, fast_indexing=True, class ImageEventHandler(FileSystemEventHandler): def __init__(self, data_dir, server_url=DEFAULT_SERVER_URL, filter_type=None, filter_value=None, - fast_indexing=True, max_dimension=384, batch_size=64): + fast_indexing=True, max_dimension=DEFAULT_MAX_DIMENSION, + batch_size=DEFAULT_BATCH_SIZE): self.data_dir = data_dir self.server_url = server_url self.filter_type = filter_type @@ -122,7 +104,7 @@ def __init__(self, data_dir, server_url=DEFAULT_SERVER_URL, self.batch_size = batch_size self.pending_files = set() self.last_index_time = time.time() - self.debounce_delay = 0.5 # Wait 0.5 seconds before indexing (reduced for faster response) + self.debounce_delay = WATCHER_DEBOUNCE_DELAY def on_created(self, event): """Called when a file or directory is created""" @@ -231,7 +213,8 @@ def check_and_index(self): def watch_directory(watch_path, data_dir, server_url=DEFAULT_SERVER_URL, filter_type=None, filter_value=None, - fast_indexing=True, max_dimension=384, batch_size=64): + fast_indexing=True, max_dimension=DEFAULT_MAX_DIMENSION, + batch_size=DEFAULT_BATCH_SIZE): """Watch directory for new images and auto-index them via server.""" print(f"Watching for new images in: {watch_path}", file=sys.stderr) @@ -280,9 +263,9 @@ def watch_directory(watch_path, data_dir, server_url=DEFAULT_SERVER_URL, help="Enable fast indexing (resize images)") parser.add_argument("--no-fast", action="store_false", dest="fast", help="Disable fast indexing") - parser.add_argument("--max-dimension", type=int, default=384, + parser.add_argument("--max-dimension", type=int, default=DEFAULT_MAX_DIMENSION, help="Maximum image dimension for fast indexing") - parser.add_argument("--batch-size", type=int, default=64, + parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE, help="Batch size for processing") args = parser.parse_args() diff --git a/searchy/requirements.txt b/searchy/backend/requirements.txt similarity index 100% rename from searchy/requirements.txt rename to searchy/backend/requirements.txt diff --git a/searchy/server.py b/searchy/backend/server.py similarity index 96% rename from searchy/server.py rename to searchy/backend/server.py index dddfb6b..5ad272f 100644 --- a/searchy/server.py +++ b/searchy/backend/server.py @@ -1,5 +1,4 @@ import os -import re import argparse import logging import logging.handlers @@ -15,13 +14,13 @@ from generate_embeddings import index_images_with_clip from clip_model import model_manager, AVAILABLE_MODELS from atomic_write import atomic_pickle_dump +from constants import DEFAULT_DATA_DIR, SKIP_DIRS, IMAGE_EXTENSIONS, TTL_CHECK_INTERVAL, OCR_TEXT_PREVIEW_LENGTH, LARGE_BATCH_SIZE, DEFAULT_TOP_K, DEFAULT_OCR_WEIGHT, DEFAULT_SIMILARITY_THRESHOLD +from utils import matches_filter, is_user_image import uvicorn import numpy as np from threading import Thread -DEFAULT_DATA_DIR = os.path.join(os.path.expanduser("~/Library/Application Support"), "searchy") - # Set up logging: console + rotating file in app support dir log_dir = os.path.join(DEFAULT_DATA_DIR, "logs") os.makedirs(log_dir, exist_ok=True) @@ -69,25 +68,6 @@ async def lifespan(app): allow_headers=["*"], ) -# Directories to skip (system, packages, caches, etc.) -SKIP_DIRS = { - 'site-packages', 'node_modules', 'vendor', '__pycache__', - 'env', 'venv', '.venv', 'virtualenv', - 'Library', 'Caches', 'cache', '.cache', - 'build', 'dist', 'target', '.git', '.svn', - 'DerivedData', 'xcuserdata', 'Pods', - '__MACOSX', '.Trash', '.Spotlight-V100', '.fseventsd' -} - -def is_user_image(path: str) -> bool: - """Check if path is a user image (not system/package file).""" - if not os.path.exists(path): - return False - if os.path.basename(path).startswith('.'): - return False - parts = path.split(os.sep) - return not any(part in SKIP_DIRS for part in parts) - # Lazy initialization of searcher _searcher = None @@ -130,7 +110,7 @@ def _ttl_checker(): """Background thread that periodically checks if the model should be unloaded.""" global _model_loading_status while True: - time.sleep(30) # Check every 30 seconds + time.sleep(TTL_CHECK_INTERVAL) if model_manager.check_ttl(): _model_loading_status["state"] = "unloaded" _model_loading_status["message"] = f"Model unloaded after {model_manager.ttl_minutes}min idle" @@ -142,11 +122,11 @@ class SearchRequest(BaseModel): query: str top_k: int data_dir: str - ocr_weight: float = 0.3 # Weight for OCR text matching (0-1) + ocr_weight: float = DEFAULT_OCR_WEIGHT class DuplicatesRequest(BaseModel): - threshold: float = 0.95 + threshold: float = DEFAULT_SIMILARITY_THRESHOLD data_dir: str = DEFAULT_DATA_DIR @@ -192,7 +172,7 @@ def union(x, y): parent[px] = py # Find similar pairs (process in batches for memory efficiency) - batch_size = 500 # Process 500 images at a time + batch_size = LARGE_BATCH_SIZE for i in range(0, n, batch_size): end_i = min(i + batch_size, n) # Compare this batch against all images from i onwards @@ -338,13 +318,13 @@ def search(request: SearchRequest): class TextSearchRequest(BaseModel): query: str - top_k: int = 20 + top_k: int = DEFAULT_TOP_K data_dir: str = DEFAULT_DATA_DIR class SimilarRequest(BaseModel): image_path: str - top_k: int = 20 + top_k: int = DEFAULT_TOP_K data_dir: str = DEFAULT_DATA_DIR @@ -398,7 +378,7 @@ def text_search(request: TextSearchRequest): results.append({ "path": path, "similarity": score, - "ocr_text": ocr_text[:300], # Include found text + "ocr_text": ocr_text[:OCR_TEXT_PREVIEW_LENGTH], "size": metadata["size"], "date": metadata["date"], "type": metadata["type"] @@ -685,31 +665,6 @@ def get_indexed_paths(data_dir: str = DEFAULT_DATA_DIR): # ============== Startup Sync Endpoints ============== -# Image extensions supported -IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.heic'} - -def matches_filter(filename: str, filter_type: str, filter_value: str) -> bool: - """Check if filename matches the filter criteria.""" - if not filter_value or filter_type == "all": - return True - - filename_lower = filename.lower() - filter_lower = filter_value.lower() - - if filter_type == "starts-with": - return filename_lower.startswith(filter_lower) - elif filter_type == "ends-with": - return filename_lower.endswith(filter_lower) - elif filter_type == "contains": - return filter_lower in filename_lower - elif filter_type == "regex": - try: - return bool(re.search(filter_value, filename, re.IGNORECASE)) - except re.error: - return False - return True - - class WatchedDirectoryRequest(BaseModel): path: str filter_type: str = "all" @@ -1552,9 +1507,9 @@ class VolumeIndexRequest(BaseModel): class MultiVolumeSearchRequest(BaseModel): """Search across multiple volume indexes.""" query: str - top_k: int = 20 + top_k: int = DEFAULT_TOP_K index_paths: List[str] # List of index file paths to search - ocr_weight: float = 0.3 + ocr_weight: float = DEFAULT_OCR_WEIGHT @app.post("/volume/index") diff --git a/searchy/similarity_search.py b/searchy/backend/similarity_search.py similarity index 96% rename from searchy/similarity_search.py rename to searchy/backend/similarity_search.py index 25825ad..0c206d1 100644 --- a/searchy/similarity_search.py +++ b/searchy/backend/similarity_search.py @@ -14,6 +14,7 @@ # Import centralized model manager from clip_model import model_manager, get_device +from constants import DEFAULT_TOP_K, DEFAULT_OCR_WEIGHT, OCR_TEXT_PREVIEW_LENGTH def text_match_score(query: str, ocr_text: str) -> float: @@ -68,7 +69,7 @@ def generate_image_embedding(self, image_path): print(f"Error generating embedding for {image_path}: {e}", file=sys.stderr) return None - def find_similar(self, image_path, data_dir, top_k=20): + def find_similar(self, image_path, data_dir, top_k=DEFAULT_TOP_K): """Find images similar to the given image.""" try: start_time = time.time() @@ -123,7 +124,7 @@ def find_similar(self, image_path, data_dir, top_k=20): print(f"Error in find_similar: {e}", file=sys.stderr) return {"error": str(e)} - def search(self, query, data_dir, top_k=5, ocr_weight=0.3): + def search(self, query, data_dir, top_k=DEFAULT_TOP_K, ocr_weight=DEFAULT_OCR_WEIGHT): try: start_time = time.time() @@ -197,7 +198,7 @@ def search(self, query, data_dir, top_k=5, ocr_weight=0.3): } # Include OCR text if found if ocr_texts[idx]: - result["ocr_text"] = ocr_texts[idx][:200] # Truncate for response + result["ocr_text"] = ocr_texts[idx][:OCR_TEXT_PREVIEW_LENGTH] results.append(result) total_time = time.time() - start_time diff --git a/searchy/tests/__init__.py b/searchy/backend/tests/__init__.py similarity index 100% rename from searchy/tests/__init__.py rename to searchy/backend/tests/__init__.py diff --git a/searchy/tests/test_atomic_write.py b/searchy/backend/tests/test_atomic_write.py similarity index 100% rename from searchy/tests/test_atomic_write.py rename to searchy/backend/tests/test_atomic_write.py diff --git a/searchy/tests/test_filters.py b/searchy/backend/tests/test_filters.py similarity index 100% rename from searchy/tests/test_filters.py rename to searchy/backend/tests/test_filters.py diff --git a/searchy/tests/test_text_match.py b/searchy/backend/tests/test_text_match.py similarity index 100% rename from searchy/tests/test_text_match.py rename to searchy/backend/tests/test_text_match.py diff --git a/searchy/backend/utils.py b/searchy/backend/utils.py new file mode 100644 index 0000000..0feeafe --- /dev/null +++ b/searchy/backend/utils.py @@ -0,0 +1,38 @@ +"""Shared utility functions for the Searchy project.""" + +import os +import re + +from constants import SKIP_DIRS + + +def matches_filter(filename: str, filter_type: str, filter_value: str) -> bool: + """Check if filename matches the filter criteria.""" + if not filter_value or filter_type == "all": + return True + + filename_lower = filename.lower() + filter_lower = filter_value.lower() + + if filter_type == "starts-with": + return filename_lower.startswith(filter_lower) + elif filter_type == "ends-with": + return filename_lower.endswith(filter_lower) + elif filter_type == "contains": + return filter_lower in filename_lower + elif filter_type == "regex": + try: + return bool(re.search(filter_value, filename, re.IGNORECASE)) + except re.error: + return False + return True + + +def is_user_image(path: str) -> bool: + """Check if path is a user image (not system/package file).""" + if not os.path.exists(path): + return False + if os.path.basename(path).startswith('.'): + return False + parts = path.split(os.sep) + return not any(part in SKIP_DIRS for part in parts)