Fix silent crawler failures, dropped regex matches, and stale-branch detection

pyproject.toml

@@ -42,6 +42,7 @@ dependencies = [
   "termcolor>=2.4.0",
   "texttable>=1.7.0",
   "threadpoolctl>=3.5.0",
+  "tomli>=2.0.1; python_version < '3.11'",
   "urllib3>=2.7.0",
   "validators>=0.22.0",
   "treelib>=1.7.1",
@@ -52,7 +53,6 @@ dependencies = [
   "phonenumbers>=8.13.37",
   "pytest>=9.0.3",
   "yattag>=1.15.2",
-  "toml>=0.10.2",
 ]
 
 [project.scripts]

requirements.txt

@@ -36,7 +36,6 @@ tabulate==0.9.0
 termcolor==2.4.0
 texttable==1.7.0
 threadpoolctl==3.5.0
-toml==0.10.2
 tomli==2.0.1
 treelib==1.7.1
 unipath==1.1

src/torbot/cli.py

@@ -5,7 +5,11 @@
 from pathlib import Path
 
 import httpx
-import toml
+
+try:
+    import tomllib
+except ModuleNotFoundError:  # pragma: no cover - Python < 3.11
+    import tomli as tomllib
 
 from torbot.modules.api import get_ip
 from torbot.modules.app_launcher import launch_torbot_app
@@ -23,8 +27,8 @@ def get_version() -> str:
         repo_root = Path(__file__).resolve().parents[2]
         config_file_path = repo_root / "pyproject.toml"
         try:
-            with config_file_path.open("r", encoding="utf-8") as handle:
-                data = toml.load(handle)
+            with config_file_path.open("rb") as handle:
+                data = tomllib.load(handle)
                 return data["project"]["version"]
         except Exception as exc:
             raise RuntimeError("unable to find version from pyproject.toml.") from exc

src/torbot/modules/info.py

@@ -154,7 +154,7 @@ def get_intel(client: httpx.Client, url: str, response: str) -> None:
     """
     intel = set()
     regex = r"""([\w\.-]+s[\w\.-]+\.amazonaws\.com)|([\w\.-]+@[\w\.-]+\.[\.\w]+)"""
-    matches = re.findall(regex, response)
+    matches = re.findall(regex, response.text)
     print("Intel\n--------\n\n")
     for match in matches:
         intel.add(match)
@@ -185,7 +185,7 @@ def get_bitcoin_address(client: httpx.Client, target: str, response: str) -> Non
         target (str): URL to be checked.
         response (object): Response object containing data to check.
     """
-    bitcoins = re.findall(r"^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$", response)
+    bitcoins = re.findall(r"^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$", response.text)
     print("BTC FOUND: ", len(bitcoins))
     for bitcoin in bitcoins:
         print("BTC: ", bitcoin)

src/torbot/modules/linktree.py

@@ -97,7 +97,7 @@ def _build_tree(self, url: str, depth: int) -> None:
                 logging.warning("Skipping subtree from %s due to request error: %s", url, exc)
                 return
 
-            children = parse_links(resp.text)
+            children = parse_links(resp.text, base_url=url)
             for child in children:
                 try:
                     self._append_node(id=child, parent_id=url)

src/torbot/modules/updater.py

@@ -12,7 +12,7 @@ def check_version():
                 "remote",
                 "add",
                 "origin",
-                "https://github.com/DedSecInside/TorBoT.git",
+                "https://github.com/DedSecInside/TorBot.git",
             ],
             capture_output=True,
         )
@@ -21,7 +21,7 @@ def check_version():
     branch_out = subprocess.run(
         ["git", "rev-parse", "--abbrev-ref", "HEAD"], capture_output=True, text=True
     )
-    branch = branch_out.stdout
+    branch = branch_out.stdout.strip()
     if branch == "master":
         update_out = subprocess.run(
             ["git", "pull", "origin", "master"], capture_output=True, text=True

tests/test_cli_and_crawler.py

@@ -1,4 +1,7 @@
+from importlib import metadata
+
 from main import set_arguments
+from torbot.cli import get_version
 from torbot.modules.app_launcher import find_torbot_app, launch_torbot_app
 from torbot.modules.linktree import parse_links
 
@@ -27,6 +30,17 @@ def test_app_flag_does_not_require_url() -> None:
     assert args.url is None
 
 
+def test_get_version_reads_pyproject_when_package_metadata_is_unavailable(
+    monkeypatch,
+) -> None:
+    def missing_package(_: str) -> str:
+        raise metadata.PackageNotFoundError
+
+    monkeypatch.setattr(metadata, "version", missing_package)
+
+    assert get_version() == "4.3.0"
+
+
 def test_find_torbot_app_from_explicit_directory(tmp_path) -> None:
     app_dir = tmp_path / "TorBotApp"
     app_dir.mkdir()

tests/test_linktree_tree.py

@@ -333,7 +333,8 @@ def test_linktree_handles_non_200_status():
 
 
 def test_linktree_filters_invalid_links():
-    """Ensure only valid absolute URLs are added as children."""
+    """Ensure non-crawlable links are filtered, and relative links are
+    resolved against the page URL rather than dropped."""
     html = """
     <html>
         <title>Root</title>
@@ -355,7 +356,10 @@ def test_linktree_filters_invalid_links():
     tree = LinkTree("https://example.com", depth=1, client=client)
     tree.load()
 
-    # Should have 2 nodes: root + 1 valid child
+    # Should have 3 nodes: root + the absolute child + the relative child
+    # (resolved against the root URL). javascript:/mailto:/# links are
+    # filtered out entirely.
     all_nodes = tree.all_nodes()
-    assert len(all_nodes) == 2
+    assert len(all_nodes) == 3
     assert tree.get_node("https://valid.com") is not None
+    assert tree.get_node("https://example.com/relative/path") is not None