From 55b7640488f1df571658b42aae6396d2f8e2637f Mon Sep 17 00:00:00 2001 From: CJ Na Date: Sat, 11 Jul 2026 08:24:30 -0700 Subject: [PATCH] fix(watch): require deletion evidence before evicting nodes for a missing source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rebuild reconciliation treated 'source identity not in the collected corpus' as 'source deleted' and evicted its nodes. But absence from the corpus is ambiguous: it also happens when the file still exists and merely stopped being collected — ignore rules or filters changed (e.g. the scanner newly honoring a .gitignore whose patterns cover wanted docs). On a real 27k-node graph, upgrading into gitignore-honoring scan semantics silently evicted all 655 nodes of a gitignored-but-present docs directory on the first incremental rebuild; recovery was only possible because a manual pre-upgrade backup existed. Make eviction fail-closed: before treating a corpus-absent identity as deleted, require the positive evidence — the file actually gone from disk. Excluded-but-alive files keep their nodes and a loud log line says how many were kept and why. True deletions and renames evict exactly as before (the old path is gone from disk in both cases), and a full re-extraction still purges deliberately excluded sources via the existing AST ownership rule. Co-Authored-By: Claude Fable 5 --- graphify/watch.py | 27 ++++++++++++++++++++++ tests/test_watch.py | 56 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/graphify/watch.py b/graphify/watch.py index 45a4a673c..db5d2a86d 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -418,6 +418,17 @@ def _reconcile_existing_graph( # lists can contain only a rename destination, so explicit paths alone # cannot identify the stale source. Keep the comparison scoped to the # watched root so subfolder updates preserve records outside that subtree. + # + # Fail-closed eviction: a source identity missing from the corpus is only + # DELETION evidence when the file is actually gone from disk. A file that + # still exists but stopped being collected was *excluded* (ignore rules or + # filters changed — e.g. a .gitignore the scanner newly honors), and + # treating that as deletion silently mass-evicts good nodes. Preserve + # instead and say so; a full re-extraction still purges deliberately + # excluded sources via the AST ownership rule below. + excluded_alive_files: set[str] = set() + excluded_alive_nodes = 0 + _alive_cache: dict[str, bool] = {} for node in existing.get("nodes", []): source_file = node.get("source_file") if not source_file or _get_extractor(Path(source_file)) is None: @@ -426,6 +437,15 @@ def _reconcile_existing_graph( if not source_paths.in_watch_root(source_file): continue if identity not in current_sources: + if identity: + alive = _alive_cache.get(identity) + if alive is None: + alive = Path(identity).exists() + _alive_cache[identity] = alive + if alive: + excluded_alive_files.add(identity) + excluded_alive_nodes += 1 + continue normalized = source_paths.normalize(source_file) if normalized: deleted_paths.add(normalized) @@ -433,6 +453,13 @@ def _reconcile_existing_graph( node_evicted_source_identities.add(identity) edge_evicted_source_identities.add(identity) hyperedge_evicted_source_identities.add(identity) + if excluded_alive_files: + print( + f"[graphify watch] fail-closed: kept {excluded_alive_nodes} node(s) " + f"from {len(excluded_alive_files)} file(s) that left the scan corpus " + "but still exist on disk (ignore rules or filters changed?). " + "Run a full re-extraction to purge them if the exclusion is intentional." + ) # A full re-extraction owns every AST node under watch_root. Incremental # extraction owns only nodes from rebuilt or deleted sources. Semantic diff --git a/tests/test_watch.py b/tests/test_watch.py index 22021ebd4..24e698169 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -1410,3 +1410,59 @@ def test_merge_changed_paths_dedupes_in_order(): [Path("a.py")], ) assert [p.as_posix() for p in merged] == ["a.py", "b.py", "c.py"] + + +def test_rebuild_code_preserves_nodes_from_excluded_but_alive_file(tmp_path, capsys): + """Fail-closed eviction: a file that leaves the scan corpus (newly ignored) + but still exists on disk was EXCLUDED, not deleted — its nodes must survive + an incremental rebuild, with a loud message, instead of being silently + mass-evicted as stale sources (the docs/brainstorms incident: an upgrade + started honoring .gitignore and evicted 655 nodes whose files were present). + """ + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + (corpus / "notes").mkdir(parents=True) + (corpus / "auth.py").write_text("def login(): pass\n", encoding="utf-8") + (corpus / "notes" / "brainstorm.md").write_text( + "# Brainstorm\n\nA local-only design note.\n", encoding="utf-8" + ) + + assert _rebuild_code(corpus, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "brainstorm.md" in labels + + # The file becomes ignored (leaves the corpus) but stays on disk. + (corpus / ".graphifyignore").write_text("notes/\n", encoding="utf-8") + capsys.readouterr() + + assert _rebuild_code(corpus, changed_paths=[Path("auth.py")], acquire_lock=False) is True + labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "brainstorm.md" in labels, ( + "nodes from an excluded-but-alive file must be preserved, not evicted" + ) + assert "fail-closed: kept" in capsys.readouterr().out + + +def test_rebuild_code_still_evicts_when_excluded_file_is_also_deleted(tmp_path): + """The fail-closed preserve must not weaken true-deletion eviction: once the + excluded file is actually gone from disk, its nodes are evicted as before.""" + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + (corpus / "notes").mkdir(parents=True) + (corpus / "auth.py").write_text("def login(): pass\n", encoding="utf-8") + (corpus / "notes" / "brainstorm.md").write_text("# Brainstorm\n", encoding="utf-8") + + assert _rebuild_code(corpus, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + + (corpus / "notes" / "brainstorm.md").unlink() + + assert _rebuild_code(corpus, changed_paths=[Path("auth.py")], acquire_lock=False) is True + labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "brainstorm.md" not in labels, "deleted file's nodes must still be evicted" + assert "login()" in labels