From 2ae88a188e34fb110875981dc214ff4ebf374bbc Mon Sep 17 00:00:00 2001 From: johnnyc20 Date: Sat, 25 Jul 2026 17:49:42 -0400 Subject: [PATCH 1/2] Fix silent crawler failures, dropped regex matches, and stale-branch detection - Pass base_url through to parse_links() when building the link tree so relative hrefs resolve instead of being silently dropped; update the integration test that had encoded the old dropped-links behavior as expected. - Fix get_intel/get_bitcoin_address in info.py running regex against the raw httpx.Response instead of response.text (always raised TypeError, swallowed by a bare except). - Fix updater.py: strip trailing newline from `git rev-parse` output before comparing to "master" (comparison never matched), and fix a typo'd fallback remote URL (TorBoT.git -> TorBot.git). - Remove deprecated `toml` dependency, now unused. (This branch originally also restructured the CLI into src/torbot/main.py and fixed the httpx.Client(proxies=...) argument rename, to fix a ModuleNotFoundError crash on install. Both are dropped from this PR: dev's own CLI restructuring into src/torbot/cli.py already fixes the same install/CLI crash via a different module layout, and already includes the corrected httpx Client argument.) Co-Authored-By: Claude Sonnet 5 --- pyproject.toml | 1 - requirements.txt | 1 - src/torbot/modules/info.py | 4 ++-- src/torbot/modules/linktree.py | 2 +- src/torbot/modules/updater.py | 4 ++-- tests/test_linktree_tree.py | 10 +++++++--- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6f8e52c2..d1d83dd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,6 @@ dependencies = [ "phonenumbers>=8.13.37", "pytest>=9.0.3", "yattag>=1.15.2", - "toml>=0.10.2", ] [project.scripts] diff --git a/requirements.txt b/requirements.txt index 85eeaa33..e0e432b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,6 @@ tabulate==0.9.0 termcolor==2.4.0 texttable==1.7.0 threadpoolctl==3.5.0 -toml==0.10.2 tomli==2.0.1 treelib==1.7.1 unipath==1.1 diff --git a/src/torbot/modules/info.py b/src/torbot/modules/info.py index 53c84b63..54cf7425 100644 --- a/src/torbot/modules/info.py +++ b/src/torbot/modules/info.py @@ -154,7 +154,7 @@ def get_intel(client: httpx.Client, url: str, response: str) -> None: """ intel = set() regex = r"""([\w\.-]+s[\w\.-]+\.amazonaws\.com)|([\w\.-]+@[\w\.-]+\.[\.\w]+)""" - matches = re.findall(regex, response) + matches = re.findall(regex, response.text) print("Intel\n--------\n\n") for match in matches: intel.add(match) @@ -185,7 +185,7 @@ def get_bitcoin_address(client: httpx.Client, target: str, response: str) -> Non target (str): URL to be checked. response (object): Response object containing data to check. """ - bitcoins = re.findall(r"^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$", response) + bitcoins = re.findall(r"^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$", response.text) print("BTC FOUND: ", len(bitcoins)) for bitcoin in bitcoins: print("BTC: ", bitcoin) diff --git a/src/torbot/modules/linktree.py b/src/torbot/modules/linktree.py index a84fe4e1..38b91fff 100644 --- a/src/torbot/modules/linktree.py +++ b/src/torbot/modules/linktree.py @@ -97,7 +97,7 @@ def _build_tree(self, url: str, depth: int) -> None: logging.warning("Skipping subtree from %s due to request error: %s", url, exc) return - children = parse_links(resp.text) + children = parse_links(resp.text, base_url=url) for child in children: try: self._append_node(id=child, parent_id=url) diff --git a/src/torbot/modules/updater.py b/src/torbot/modules/updater.py index b704308a..33001133 100644 --- a/src/torbot/modules/updater.py +++ b/src/torbot/modules/updater.py @@ -12,7 +12,7 @@ def check_version(): "remote", "add", "origin", - "https://github.com/DedSecInside/TorBoT.git", + "https://github.com/DedSecInside/TorBot.git", ], capture_output=True, ) @@ -21,7 +21,7 @@ def check_version(): branch_out = subprocess.run( ["git", "rev-parse", "--abbrev-ref", "HEAD"], capture_output=True, text=True ) - branch = branch_out.stdout + branch = branch_out.stdout.strip() if branch == "master": update_out = subprocess.run( ["git", "pull", "origin", "master"], capture_output=True, text=True diff --git a/tests/test_linktree_tree.py b/tests/test_linktree_tree.py index 91ec929b..0fbaf7d9 100644 --- a/tests/test_linktree_tree.py +++ b/tests/test_linktree_tree.py @@ -333,7 +333,8 @@ def test_linktree_handles_non_200_status(): def test_linktree_filters_invalid_links(): - """Ensure only valid absolute URLs are added as children.""" + """Ensure non-crawlable links are filtered, and relative links are + resolved against the page URL rather than dropped.""" html = """ Root @@ -355,7 +356,10 @@ def test_linktree_filters_invalid_links(): tree = LinkTree("https://example.com", depth=1, client=client) tree.load() - # Should have 2 nodes: root + 1 valid child + # Should have 3 nodes: root + the absolute child + the relative child + # (resolved against the root URL). javascript:/mailto:/# links are + # filtered out entirely. all_nodes = tree.all_nodes() - assert len(all_nodes) == 2 + assert len(all_nodes) == 3 assert tree.get_node("https://valid.com") is not None + assert tree.get_node("https://example.com/relative/path") is not None From b3a2beb8bfe4c89d10be732277b742468271297e Mon Sep 17 00:00:00 2001 From: Akeem King Date: Thu, 13 Aug 2026 07:58:04 -0400 Subject: [PATCH 2/2] Fix pyproject version fallback dependency --- pyproject.toml | 1 + src/torbot/cli.py | 10 +++++++--- tests/test_cli_and_crawler.py | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d1d83dd8..2b5fe298 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ "termcolor>=2.4.0", "texttable>=1.7.0", "threadpoolctl>=3.5.0", + "tomli>=2.0.1; python_version < '3.11'", "urllib3>=2.7.0", "validators>=0.22.0", "treelib>=1.7.1", diff --git a/src/torbot/cli.py b/src/torbot/cli.py index 62860585..45e867cd 100644 --- a/src/torbot/cli.py +++ b/src/torbot/cli.py @@ -5,7 +5,11 @@ from pathlib import Path import httpx -import toml + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python < 3.11 + import tomli as tomllib from torbot.modules.api import get_ip from torbot.modules.app_launcher import launch_torbot_app @@ -23,8 +27,8 @@ def get_version() -> str: repo_root = Path(__file__).resolve().parents[2] config_file_path = repo_root / "pyproject.toml" try: - with config_file_path.open("r", encoding="utf-8") as handle: - data = toml.load(handle) + with config_file_path.open("rb") as handle: + data = tomllib.load(handle) return data["project"]["version"] except Exception as exc: raise RuntimeError("unable to find version from pyproject.toml.") from exc diff --git a/tests/test_cli_and_crawler.py b/tests/test_cli_and_crawler.py index 3b33d7ff..5936bcd7 100644 --- a/tests/test_cli_and_crawler.py +++ b/tests/test_cli_and_crawler.py @@ -1,4 +1,7 @@ +from importlib import metadata + from main import set_arguments +from torbot.cli import get_version from torbot.modules.app_launcher import find_torbot_app, launch_torbot_app from torbot.modules.linktree import parse_links @@ -27,6 +30,17 @@ def test_app_flag_does_not_require_url() -> None: assert args.url is None +def test_get_version_reads_pyproject_when_package_metadata_is_unavailable( + monkeypatch, +) -> None: + def missing_package(_: str) -> str: + raise metadata.PackageNotFoundError + + monkeypatch.setattr(metadata, "version", missing_package) + + assert get_version() == "4.3.0" + + def test_find_torbot_app_from_explicit_directory(tmp_path) -> None: app_dir = tmp_path / "TorBotApp" app_dir.mkdir()