Improve CLI handling and crawler link resolution

.flake8

@@ -1,2 +1,3 @@
 [flake8]
 max-line-length = 119
+exclude = .venv,build,dist

README.md

@@ -39,44 +39,49 @@
 8. Built-in Updater
 9. Build a visual tree of link relationships that can be quickly viewed or saved to a file
 
+### Recent improvements
+- The CLI now allows `--version` and `--update` to run without requiring a URL.
+- The crawler now resolves relative links against the current page and skips unsupported or malformed hrefs more gracefully.
+
 ...(will be updated)
 
 ### Dependencies
 - Tor (Optional)
-- Python ^3.9
-- Poetry (Optional)
-
-### Python Dependencies
+- Python 3.9+
+- pip
 
-(see pyproject.toml or requirements.txt for more details)
+## Quick start
 
-## Installation
-
-### TorBot
+### Local setup
+```sh
+python3 -m venv .venv
+source .venv/bin/activate
+python -m pip install --upgrade pip
+python -m pip install -r requirements.txt
+python -m pip install -e .
+```
 
-#### Using `venv`
-* If using Python ^3.4,
+### Run the CLI
 ```sh
-python -m venv torbot_venv
-source torbot_venv/bin/activate
-pip install -r requirements.txt
-pip install -e .
-./main.py --help
+python main.py --help
+python main.py --version
+python main.py -u https://example.com --depth 2 --visualize table
 ```
 
-#### Using `docker`
+### Common examples
 ```sh
-docker build -t {image_name} .
+# Gather basic site information
+python main.py -u https://example.com --info
 
-# Running without Tor
-docker run {image_name} poetry run python torbot -u https://example.com --depth 2 --visualize tree --save json --disable-socks5
+# Save the crawl tree as JSON
+python main.py -u https://example.com --depth 2 --save json
 
-# Running with Tor
-docker run --network="host" {image_name} poetry run python torbot -u https://example.com --depth 2 --visualize tree --save json --disable-socks5
+# Disable SOCKS5 if you are not using Tor locally
+python main.py -u https://example.com --disable-socks5 --visualize tree
 ```
 
 ### Options
-<pre>
+```text
 usage: Gather and analyze data from Tor sites.
 
 optional arguments:
@@ -92,11 +97,14 @@ optional arguments:
   --save FORMAT         Save results in a file. (tree, JSON)
   --visualize FORMAT    Visualizes tree of data gathered. (tree, JSON, table)
   -i, --info            Info displays basic info of the scanned site
-  --disable-socks5      Executes HTTP requests without using SOCKS5 proxy</pre>
+  --disable-socks5      Executes HTTP requests without using SOCKS5 proxy
+```
+
+> `-u/--url` is required for crawl-related commands such as `--info`, `--save`, and `--visualize`.
 
-* NOTE: -u is a mandatory for crawling
+If you are using Tor locally, keep the SOCKS5 proxy enabled. If you are not using Tor, add `--disable-socks5` to avoid proxy connection errors.
 
-Read more about torrc here : [Torrc](https://github.com/DedSecInside/TorBoT/blob/master/Tor.md)
+Read more about torrc here: [Torrc](https://github.com/DedSecInside/TorBoT/blob/master/Tor.md)
 
 ## Curated Features
 - [x] Visualization Module Revamp

docs/CHANGELOG.md

@@ -2,6 +2,12 @@
 --------------------
 All notable changes to this project will be documented in this file.
 
+## 4.1.1
+
+### Fixed
+- Adjusted the CLI so `--version` and `--update` can run without requiring a URL.
+- Improved crawler link parsing so relative links are resolved against the current page while unsupported hrefs are skipped cleanly.
+
 ## 2.1.0
 
 ### Added

main.py

@@ -61,11 +61,6 @@ def run(arg_parser: argparse.ArgumentParser, version: str) -> None:
     logging_lvl = logging.DEBUG if args.v else logging.INFO
     logging.basicConfig(level=logging_lvl, format=logging_fmt, datefmt=date_fmt)
 
-    # URL is a required argument
-    if not args.url:
-        arg_parser.print_help()
-        sys.exit()
-
     # Print verison then exit
     if args.version:
         print(f"TorBot Version: {version}")
@@ -76,6 +71,11 @@ def run(arg_parser: argparse.ArgumentParser, version: str) -> None:
         check_version()
         sys.exit()
 
+    # URL is required for crawl-related actions
+    if not args.url:
+        arg_parser.print_help()
+        sys.exit()
+
     socks5_host = args.host
     socks5_port = str(args.port)
     socks5_proxy = f"socks5://{socks5_host}:{socks5_port}"
@@ -123,7 +123,11 @@ def set_arguments() -> argparse.ArgumentParser:
         prog="TorBot", usage="Gather and analayze data from Tor sites."
     )
     parser.add_argument(
-        "-u", "--url", type=str, required=True, help="Specifiy a website link to crawl"
+        "-u",
+        "--url",
+        type=str,
+        default=None,
+        help="Specifiy a website link to crawl",
     )
     parser.add_argument(
         "--depth", type=int, help="Specifiy max depth of crawler (default 1)", default=1

pyproject.toml

@@ -4,51 +4,57 @@ build-backend = "hatchling.build"
 
 [project]
 name = "torbot"
-version = "4.1.0"
+version = "4.1.1"
 authors = [
     { name="Akeem King", email="akeemtlking@gmail.com" },
     { name="PS Narayanan", email="thepsnarayanan@gmail.com" },
 ]
 description = "TorBot is an OSINT tool for the dark web."
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.9"
 classifiers = [
   "Programming Language :: Python :: 3",
-  "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Operating System :: OS Independent",
 ]
 dependencies = [
-  "altgraph==0.17.2",
-  "beautifulsoup4==4.11.1",
-  "certifi==2024.7.4",
-  "charset-normalizer==2.0.12",
-  "decorator==5.1.1",
-  "idna==3.15",
-  "igraph==0.10.6",
-  "joblib==1.2.0",
-  "macholib==1.16",
-  "progress==1.6",
-  "pyinstaller==6.0.0",
-  "pyinstaller-hooks-contrib==2022.7",
-  "PySocks==1.7.1",
-  "python-dotenv==1.2.2",
-  "scikit-learn==1.3.0",
-  "scipy==1.10.0",
-  "six==1.16.0",
-  "sklearn==0.0",
-  "soupsieve==2.8.4",
-  "termcolor==1.1.0",
-  "texttable==1.6.4",
-  "threadpoolctl==3.1.0",
-  "urllib3==2.7.0",
-  "validators==0.20.0",
-  "treelib==1.6.1",
-  "numpy==1.24.4",
-  "unipath==1.1",
-  "httpx[socks]==0.25.0",
-  "tabulate==0.9.0",
-  "phonenumbers==8.13.22",
-  "pytest==7.4.2",
-  "yattag==1.15.1",
-  "toml==0.10.2",
+  "altgraph>=0.17.4",
+  "beautifulsoup4>=4.12.3",
+  "certifi>=2024.7.4",
+  "charset-normalizer>=3.3.2",
+  "decorator>=5.1.1",
+  "idna>=3.15",
+  "igraph>=0.11.4",
+  "joblib>=1.4.2",
+  "macholib>=1.16.3",
+  "progress>=1.6",
+  "pyinstaller>=6.8.0",
+  "pyinstaller-hooks-contrib>=2024.6",
+  "PySocks>=1.7.1",
+  "python-dotenv>=1.2.2",
+  "scikit-learn>=1.4.2",
+  "scipy>=1.13.0",
+  "six>=1.16.0",
+  "sklearn>=0.0",
+  "soupsieve>=2.8.4",
+  "termcolor>=2.4.0",
+  "texttable>=1.7.0",
+  "threadpoolctl>=3.5.0",
+  "urllib3>=2.7.0",
+  "validators>=0.22.0",
+  "treelib>=1.7.1",
+  "numpy>=1.26.4",
+  "unipath>=1.1",
+  "httpx[socks]>=0.27.0",
+  "tabulate>=0.9.0",
+  "phonenumbers>=8.13.37",
+  "pytest>=8.1.1",
+  "yattag>=1.15.2",
+  "toml>=0.10.2",
 ]
+
+[project.scripts]
+torbot = "torbot.cli:main"

scripts/publish.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+python3 -m build
+python3 -m twine check dist/*
+echo "Publishing is ready. Run the following when you have your PyPI token:"
+echo "python3 -m twine upload dist/*"

src/torbot/cli.py

@@ -0,0 +1,28 @@
+import os
+import sys
+
+import toml
+
+REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+if REPO_ROOT not in sys.path:
+    sys.path.insert(0, REPO_ROOT)
+
+from main import run as run_cli, set_arguments  # noqa: E402
+
+
+def main() -> None:
+    """Run the TorBot CLI from an installed entry point."""
+    arg_parser = set_arguments()
+    config_file_path = os.path.join(REPO_ROOT, "pyproject.toml")
+    try:
+        with open(config_file_path, "r", encoding="utf-8") as handle:
+            data = toml.load(handle)
+            version = data["project"]["version"]
+    except Exception as exc:
+        raise RuntimeError("unable to find version from pyproject.toml.") from exc
+
+    run_cli(arg_parser, version)
+
+
+if __name__ == "__main__":
+    main()

src/torbot/modules/linktree.py

@@ -2,23 +2,27 @@
 Module is used for analyzing link relationships
 """
 import http.client
-import os
 import json
-import httpx
-import validators
 import logging
-import phonenumbers
-
+import os
 from urllib import parse
-from tabulate import tabulate
-from treelib import Tree, exceptions, Node
+
+import httpx
+import phonenumbers
+import validators
 from bs4 import BeautifulSoup
+from tabulate import tabulate
+from treelib import Node, Tree, exceptions
 
 from .color import color
 from .config import project_root_directory
 from .nlp.main import classify
 
 
+class RequestError(Exception):
+    """Raised when a page request fails during crawling."""
+
+
 class LinkNode(Node):
     def __init__(
         self,
@@ -56,7 +60,16 @@ def _append_node(self, id: str, parent_id: str or None) -> None:
         Creates a node for a tree using the given ID which corresponds to a URL.
         If the parent_id is None, this will be considered a root node.
         """
-        resp = self._client.get(id)
+        try:
+            resp = self._client.get(id)
+        except Exception as exc:
+            logging.warning("Skipping URL %s due to request error: %s", id, exc)
+            raise RequestError(str(exc)) from exc
+
+        if not getattr(resp, "text", None):
+            logging.warning("Skipping URL %s because the response body was empty", id)
+            raise RequestError("empty response")
+
         soup = BeautifulSoup(resp.text, "html.parser")
         title = (
             soup.title.text.strip() if soup.title is not None else parse_hostname(id)
@@ -78,11 +91,19 @@ def _build_tree(self, url: str, depth: int) -> None:
         """
         if depth > 0:
             depth -= 1
-            resp = self._client.get(url)
+            try:
+                resp = self._client.get(url)
+            except Exception as exc:
+                logging.warning("Skipping subtree from %s due to request error: %s", url, exc)
+                return
+
             children = parse_links(resp.text)
             for child in children:
-                self._append_node(id=child, parent_id=url)
-                self._build_tree(url=child, depth=depth)
+                try:
+                    self._append_node(id=child, parent_id=url)
+                    self._build_tree(url=child, depth=depth)
+                except RequestError:
+                    continue
 
     def _get_tree_file_name(self) -> str:
         root_id = self.root
@@ -164,17 +185,35 @@ def parse_hostname(url: str) -> str:
     raise Exception("unable to parse hostname from URL")
 
 
-def parse_links(html: str) -> list[str]:
+def parse_links(html: str, base_url: str | None = None) -> list[str]:
     """
     Finds all anchor tags and parses the href attribute.
+
+    Relative links are resolved against the page URL when a base URL is provided.
+    Only absolute http(s) links are returned.
     """
     soup = BeautifulSoup(html, "html.parser")
-    tags = soup.find_all("a")
-    return [
-        tag["href"]
-        for tag in tags
-        if tag.has_attr("href") and validators.url(tag["href"])
-    ]
+    links = []
+    for tag in soup.find_all("a"):
+        href = tag.get("href")
+        if not href or not isinstance(href, str):
+            continue
+
+        cleaned_href = href.strip()
+        if not cleaned_href or cleaned_href.startswith(("#", "mailto:", "tel:", "javascript:")):
+            continue
+
+        if base_url:
+            resolved = parse.urljoin(base_url, cleaned_href)
+        else:
+            resolved = cleaned_href
+
+        if validators.url(resolved) and resolved.startswith(("http://", "https://")):
+            links.append(resolved)
+        elif base_url and validators.url(cleaned_href):
+            links.append(cleaned_href)
+
+    return links
 
 
 def parse_emails(soup: BeautifulSoup) -> list[str]: