diff --git a/.flake8 b/.flake8 index 791f075d..e5a5e569 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,3 @@ [flake8] max-line-length = 119 +exclude = .venv,build,dist diff --git a/README.md b/README.md index 671ae8f9..b9dae779 100755 --- a/README.md +++ b/README.md @@ -39,44 +39,49 @@ 8. Built-in Updater 9. Build a visual tree of link relationships that can be quickly viewed or saved to a file +### Recent improvements +- The CLI now allows `--version` and `--update` to run without requiring a URL. +- The crawler now resolves relative links against the current page and skips unsupported or malformed hrefs more gracefully. + ...(will be updated) ### Dependencies - Tor (Optional) -- Python ^3.9 -- Poetry (Optional) - -### Python Dependencies +- Python 3.9+ +- pip -(see pyproject.toml or requirements.txt for more details) +## Quick start -## Installation - -### TorBot +### Local setup +```sh +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -r requirements.txt +python -m pip install -e . +``` -#### Using `venv` -* If using Python ^3.4, +### Run the CLI ```sh -python -m venv torbot_venv -source torbot_venv/bin/activate -pip install -r requirements.txt -pip install -e . -./main.py --help +python main.py --help +python main.py --version +python main.py -u https://example.com --depth 2 --visualize table ``` -#### Using `docker` +### Common examples ```sh -docker build -t {image_name} . +# Gather basic site information +python main.py -u https://example.com --info -# Running without Tor -docker run {image_name} poetry run python torbot -u https://example.com --depth 2 --visualize tree --save json --disable-socks5 +# Save the crawl tree as JSON +python main.py -u https://example.com --depth 2 --save json -# Running with Tor -docker run --network="host" {image_name} poetry run python torbot -u https://example.com --depth 2 --visualize tree --save json --disable-socks5 +# Disable SOCKS5 if you are not using Tor locally +python main.py -u https://example.com --disable-socks5 --visualize tree ``` ### Options -
+```text
 usage: Gather and analyze data from Tor sites.
 
 optional arguments:
@@ -92,11 +97,14 @@ optional arguments:
   --save FORMAT         Save results in a file. (tree, JSON)
   --visualize FORMAT    Visualizes tree of data gathered. (tree, JSON, table)
   -i, --info            Info displays basic info of the scanned site
-  --disable-socks5      Executes HTTP requests without using SOCKS5 proxy
+ --disable-socks5 Executes HTTP requests without using SOCKS5 proxy +``` + +> `-u/--url` is required for crawl-related commands such as `--info`, `--save`, and `--visualize`. -* NOTE: -u is a mandatory for crawling +If you are using Tor locally, keep the SOCKS5 proxy enabled. If you are not using Tor, add `--disable-socks5` to avoid proxy connection errors. -Read more about torrc here : [Torrc](https://github.com/DedSecInside/TorBoT/blob/master/Tor.md) +Read more about torrc here: [Torrc](https://github.com/DedSecInside/TorBoT/blob/master/Tor.md) ## Curated Features - [x] Visualization Module Revamp diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fc220433..68fc45a4 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,12 @@ -------------------- All notable changes to this project will be documented in this file. +## 4.1.1 + +### Fixed +- Adjusted the CLI so `--version` and `--update` can run without requiring a URL. +- Improved crawler link parsing so relative links are resolved against the current page while unsupported hrefs are skipped cleanly. + ## 2.1.0 ### Added diff --git a/main.py b/main.py index bf9207c9..7b8029a9 100755 --- a/main.py +++ b/main.py @@ -61,11 +61,6 @@ def run(arg_parser: argparse.ArgumentParser, version: str) -> None: logging_lvl = logging.DEBUG if args.v else logging.INFO logging.basicConfig(level=logging_lvl, format=logging_fmt, datefmt=date_fmt) - # URL is a required argument - if not args.url: - arg_parser.print_help() - sys.exit() - # Print verison then exit if args.version: print(f"TorBot Version: {version}") @@ -76,6 +71,11 @@ def run(arg_parser: argparse.ArgumentParser, version: str) -> None: check_version() sys.exit() + # URL is required for crawl-related actions + if not args.url: + arg_parser.print_help() + sys.exit() + socks5_host = args.host socks5_port = str(args.port) socks5_proxy = f"socks5://{socks5_host}:{socks5_port}" @@ -123,7 +123,11 @@ def set_arguments() -> argparse.ArgumentParser: prog="TorBot", usage="Gather and analayze data from Tor sites." ) parser.add_argument( - "-u", "--url", type=str, required=True, help="Specifiy a website link to crawl" + "-u", + "--url", + type=str, + default=None, + help="Specifiy a website link to crawl", ) parser.add_argument( "--depth", type=int, help="Specifiy max depth of crawler (default 1)", default=1 diff --git a/pyproject.toml b/pyproject.toml index 4ab04023..69233888 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,51 +4,57 @@ build-backend = "hatchling.build" [project] name = "torbot" -version = "4.1.0" +version = "4.1.1" authors = [ { name="Akeem King", email="akeemtlking@gmail.com" }, { name="PS Narayanan", email="thepsnarayanan@gmail.com" }, ] description = "TorBot is an OSINT tool for the dark web." readme = "README.md" -requires-python = ">=3.7" +requires-python = ">=3.9" classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Operating System :: OS Independent", ] dependencies = [ - "altgraph==0.17.2", - "beautifulsoup4==4.11.1", - "certifi==2024.7.4", - "charset-normalizer==2.0.12", - "decorator==5.1.1", - "idna==3.15", - "igraph==0.10.6", - "joblib==1.2.0", - "macholib==1.16", - "progress==1.6", - "pyinstaller==6.0.0", - "pyinstaller-hooks-contrib==2022.7", - "PySocks==1.7.1", - "python-dotenv==1.2.2", - "scikit-learn==1.3.0", - "scipy==1.10.0", - "six==1.16.0", - "sklearn==0.0", - "soupsieve==2.8.4", - "termcolor==1.1.0", - "texttable==1.6.4", - "threadpoolctl==3.1.0", - "urllib3==2.7.0", - "validators==0.20.0", - "treelib==1.6.1", - "numpy==1.24.4", - "unipath==1.1", - "httpx[socks]==0.25.0", - "tabulate==0.9.0", - "phonenumbers==8.13.22", - "pytest==7.4.2", - "yattag==1.15.1", - "toml==0.10.2", + "altgraph>=0.17.4", + "beautifulsoup4>=4.12.3", + "certifi>=2024.7.4", + "charset-normalizer>=3.3.2", + "decorator>=5.1.1", + "idna>=3.15", + "igraph>=0.11.4", + "joblib>=1.4.2", + "macholib>=1.16.3", + "progress>=1.6", + "pyinstaller>=6.8.0", + "pyinstaller-hooks-contrib>=2024.6", + "PySocks>=1.7.1", + "python-dotenv>=1.2.2", + "scikit-learn>=1.4.2", + "scipy>=1.13.0", + "six>=1.16.0", + "sklearn>=0.0", + "soupsieve>=2.8.4", + "termcolor>=2.4.0", + "texttable>=1.7.0", + "threadpoolctl>=3.5.0", + "urllib3>=2.7.0", + "validators>=0.22.0", + "treelib>=1.7.1", + "numpy>=1.26.4", + "unipath>=1.1", + "httpx[socks]>=0.27.0", + "tabulate>=0.9.0", + "phonenumbers>=8.13.37", + "pytest>=8.1.1", + "yattag>=1.15.2", + "toml>=0.10.2", ] + +[project.scripts] +torbot = "torbot.cli:main" diff --git a/scripts/publish.sh b/scripts/publish.sh new file mode 100755 index 00000000..c1564f1d --- /dev/null +++ b/scripts/publish.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +python3 -m build +python3 -m twine check dist/* +echo "Publishing is ready. Run the following when you have your PyPI token:" +echo "python3 -m twine upload dist/*" diff --git a/src/torbot/cli.py b/src/torbot/cli.py new file mode 100644 index 00000000..1eb287a1 --- /dev/null +++ b/src/torbot/cli.py @@ -0,0 +1,28 @@ +import os +import sys + +import toml + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +from main import run as run_cli, set_arguments # noqa: E402 + + +def main() -> None: + """Run the TorBot CLI from an installed entry point.""" + arg_parser = set_arguments() + config_file_path = os.path.join(REPO_ROOT, "pyproject.toml") + try: + with open(config_file_path, "r", encoding="utf-8") as handle: + data = toml.load(handle) + version = data["project"]["version"] + except Exception as exc: + raise RuntimeError("unable to find version from pyproject.toml.") from exc + + run_cli(arg_parser, version) + + +if __name__ == "__main__": + main() diff --git a/src/torbot/modules/linktree.py b/src/torbot/modules/linktree.py index 89477664..a84fe4e1 100644 --- a/src/torbot/modules/linktree.py +++ b/src/torbot/modules/linktree.py @@ -2,23 +2,27 @@ Module is used for analyzing link relationships """ import http.client -import os import json -import httpx -import validators import logging -import phonenumbers - +import os from urllib import parse -from tabulate import tabulate -from treelib import Tree, exceptions, Node + +import httpx +import phonenumbers +import validators from bs4 import BeautifulSoup +from tabulate import tabulate +from treelib import Node, Tree, exceptions from .color import color from .config import project_root_directory from .nlp.main import classify +class RequestError(Exception): + """Raised when a page request fails during crawling.""" + + class LinkNode(Node): def __init__( self, @@ -56,7 +60,16 @@ def _append_node(self, id: str, parent_id: str or None) -> None: Creates a node for a tree using the given ID which corresponds to a URL. If the parent_id is None, this will be considered a root node. """ - resp = self._client.get(id) + try: + resp = self._client.get(id) + except Exception as exc: + logging.warning("Skipping URL %s due to request error: %s", id, exc) + raise RequestError(str(exc)) from exc + + if not getattr(resp, "text", None): + logging.warning("Skipping URL %s because the response body was empty", id) + raise RequestError("empty response") + soup = BeautifulSoup(resp.text, "html.parser") title = ( soup.title.text.strip() if soup.title is not None else parse_hostname(id) @@ -78,11 +91,19 @@ def _build_tree(self, url: str, depth: int) -> None: """ if depth > 0: depth -= 1 - resp = self._client.get(url) + try: + resp = self._client.get(url) + except Exception as exc: + logging.warning("Skipping subtree from %s due to request error: %s", url, exc) + return + children = parse_links(resp.text) for child in children: - self._append_node(id=child, parent_id=url) - self._build_tree(url=child, depth=depth) + try: + self._append_node(id=child, parent_id=url) + self._build_tree(url=child, depth=depth) + except RequestError: + continue def _get_tree_file_name(self) -> str: root_id = self.root @@ -164,17 +185,35 @@ def parse_hostname(url: str) -> str: raise Exception("unable to parse hostname from URL") -def parse_links(html: str) -> list[str]: +def parse_links(html: str, base_url: str | None = None) -> list[str]: """ Finds all anchor tags and parses the href attribute. + + Relative links are resolved against the page URL when a base URL is provided. + Only absolute http(s) links are returned. """ soup = BeautifulSoup(html, "html.parser") - tags = soup.find_all("a") - return [ - tag["href"] - for tag in tags - if tag.has_attr("href") and validators.url(tag["href"]) - ] + links = [] + for tag in soup.find_all("a"): + href = tag.get("href") + if not href or not isinstance(href, str): + continue + + cleaned_href = href.strip() + if not cleaned_href or cleaned_href.startswith(("#", "mailto:", "tel:", "javascript:")): + continue + + if base_url: + resolved = parse.urljoin(base_url, cleaned_href) + else: + resolved = cleaned_href + + if validators.url(resolved) and resolved.startswith(("http://", "https://")): + links.append(resolved) + elif base_url and validators.url(cleaned_href): + links.append(cleaned_href) + + return links def parse_emails(soup: BeautifulSoup) -> list[str]: diff --git a/tests/test_cli_and_crawler.py b/tests/test_cli_and_crawler.py new file mode 100644 index 00000000..b06cf709 --- /dev/null +++ b/tests/test_cli_and_crawler.py @@ -0,0 +1,26 @@ +from main import set_arguments +from torbot.modules.linktree import parse_links + + +def test_version_flag_does_not_require_url() -> None: + parser = set_arguments() + args = parser.parse_args(["--version"]) + + assert args.version is True + assert args.url is None + + +def test_parse_links_resolves_relative_urls_against_base_url() -> None: + html = """ + + + About + Docs + JS + + + """ + + links = parse_links(html, base_url="https://example.com/index") + + assert links == ["https://example.com/about", "https://example.com/docs"]