diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 491c773..1729f85 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,4 +18,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - run: python -m pip install . + - run: github-dorks --version + - run: python -m github_dorks --version - run: python -m unittest discover -s tests -v diff --git a/Dockerfile b/Dockerfile index 92e04ec..5748224 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,9 @@ FROM python:3.12-slim WORKDIR /app # Copy only the files needed to install and run the project -COPY github-dork.py /app/ +COPY github_dorks /app/github_dorks COPY github-dorks.txt /app/ -COPY setup.py /app/ +COPY pyproject.toml /app/ COPY README.md /app/ RUN pip install --no-cache-dir . @@ -18,4 +18,4 @@ ENV PYTHONIOENCODING=UTF-8 # Create volume for potential output files VOLUME ["/app/output"] -ENTRYPOINT ["python", "github-dork.py"] +ENTRYPOINT ["github-dorks"] diff --git a/README.md b/README.md index 29f6093..f86a61f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ ## GitHub Dork Search Tool -[github-dork.py](github-dork.py) is a simple python tool that can search through your repository or your organization/user repositories. It's not a perfect tool at the moment but provides basic functionality to automate the search on your repositories against the dorks specified in the text file. +`github-dorks` is a Python command-line tool that searches a repository or an +organization/user's repositories using the queries in a dorks file. ### Installation @@ -48,19 +49,22 @@ GH_URL - Environment variable to specify GitHub Enterprise base URL Some example usages are listed below: ```shell -github-dork.py -r techgaun/github-dorks # search a single repo +github-dorks -r techgaun/github-dorks # search a single repo -github-dork.py -u techgaun # search all repos of a user +github-dorks -u techgaun # search all repos of a user -github-dork.py -u dev-nepal # search all repos of an organization +github-dorks -u dev-nepal # search all repos of an organization -GH_USER=techgaun GH_PWD= github-dork.py -u dev-nepal # search as authenticated user +GH_USER=techgaun GH_PWD= github-dorks -u dev-nepal # search as authenticated user -GH_TOKEN= github-dork.py -u dev-nepal # search using auth token +GH_TOKEN= github-dorks -u dev-nepal # search using auth token -GH_URL=https://github.example.com github-dork.py -u dev-nepal # search a GitHub Enterprise instance +GH_URL=https://github.example.com github-dorks -u dev-nepal # search a GitHub Enterprise instance ``` +The legacy `python github-dork.py ...` invocation remains available for +compatibility. The package can also run as `python -m github_dorks ...`. + ### Development Run the dependency-free unit test suite with: diff --git a/github-dork.py b/github-dork.py index 72e1bea..97d2ff5 100644 --- a/github-dork.py +++ b/github-dork.py @@ -1,226 +1,7 @@ #!/usr/bin/env python -# -*- encoding: utf-8 -*- +"""Backward-compatible entry point for existing users.""" -import github3 as github -import os -import argparse -import csv -import time -import feedparser -from copy import copy -from contextlib import nullcontext -from sys import stderr, prefix - -__version__ = '0.1.1' - -gh_user = os.getenv('GH_USER', None) -gh_pass = os.getenv('GH_PWD', None) -gh_token = os.getenv('GH_TOKEN', None) -gh_url = os.getenv('GH_URL', None) - -if gh_url is None: - gh = github.GitHub(username=gh_user, password=gh_pass, token=gh_token) -else: - gh = github.GitHubEnterprise( - url=gh_url, username=gh_user, password=gh_pass, token=gh_token) - - -def search_wrapper(gen): - while True: - gen_back = copy(gen) - try: - yield next(gen) - except StopIteration: - return - except github.exceptions.ForbiddenError: - search_rate_limit = gh.rate_limit()['resources']['search'] - # limit_remaining = search_rate_limit['remaining'] - reset_time = search_rate_limit['reset'] - current_time = int(time.time()) - sleep_time = reset_time - current_time + 1 - stderr.write( - 'GitHub Search API rate limit reached. Sleeping for %d seconds.\n\n' - % (sleep_time)) - time.sleep(sleep_time) - yield next(gen_back) - except Exception as e: - raise e - - -def metasearch(repo_to_search=None, - user_to_search=None, - gh_dorks_file=None, - active_monit=None, - output_filename=None, - refresh_time=60): - if active_monit is None: - search(repo_to_search, user_to_search, gh_dorks_file, active_monit, output_filename) - else: - monit(gh_dorks_file, active_monit, refresh_time) - - -def monit(gh_dorks_file=None, active_monit=None, refresh_time=60): - if gh_user is None: - raise Exception('Error, env Github user variable needed') - else: - print( - 'Monitoring user private feed searching new code to be dorked.' + - 'Every new merged pull request trigger user scan.' - ) - print('-----') - items_history = list() - gh_private_feed = "https://github.com/{}.private.atom?token={}".format( - gh_user, active_monit) - while True: - feed = feedparser.parse(gh_private_feed) - for i in feed['items']: - if 'merged pull' in i['title']: - if i['title'] not in items_history: - search( - user_to_search=i['author_detail']['name'], - gh_dorks_file=gh_dorks_file) - items_history.append(i['title']) - print('Waiting for new items...') - time.sleep(refresh_time) - - -def search(repo_to_search=None, - user_to_search=None, - gh_dorks_file=None, - active_monit=None, - output_filename=None): - - if gh_dorks_file is None: - for path_prefix in ['.', os.path.join(prefix, 'github-dorks/')]: - filename = os.path.join(path_prefix, 'github-dorks.txt') - if os.path.isfile(filename): - gh_dorks_file = filename - break - - if gh_dorks_file is None or not os.path.isfile(gh_dorks_file): - raise Exception('Error, the dorks file path is not valid') - if user_to_search: - print("Scanning User: ", user_to_search) - if repo_to_search: - print("Scanning Repo: ", repo_to_search) - found = False - - output_context = ( - open(output_filename, 'w', newline='', encoding='utf-8') - if output_filename else nullcontext(None) - ) - - with open(gh_dorks_file, 'r', encoding='utf-8') as dork_file, output_context as output_file: - # Write CSV Header - csv_writer = None - if output_file: - csv_writer = csv.writer(output_file) - csv_writer.writerow([ - 'Issue Type (Dork)', 'Text Matches', 'File Path', - 'Score/Relevance', 'URL of File' - ]) - for dork in dork_file: - dork = dork.strip() - if not dork or dork[0] in '#;': - continue - addendum = '' - if repo_to_search: - addendum = ' repo:' + repo_to_search - elif user_to_search: - addendum = ' user:' + user_to_search - - dork = dork + addendum - search_results = search_wrapper(gh.search_code(dork)) - try: - for search_result in search_results: - found = True - fmt_args = { - 'dork': dork, - 'text_matches': search_result.text_matches, - 'path': search_result.path, - 'score': search_result.score, - 'url': search_result.html_url - } - - # Either write to file or print output - if csv_writer: - csv_writer.writerow([ - fmt_args['dork'], fmt_args['text_matches'], - fmt_args['path'], fmt_args['score'], fmt_args['url'] - ]) - else: - result = '\n'.join([ - 'Found result for {dork}', - 'Text matches: {text_matches}', 'File path: {path}', - 'Score/Relevance: {score}', 'URL of File: {url}', '' - ]).format(**fmt_args) - print(result) - - except github.exceptions.GitHubError as e: - print('GitHubError encountered on search of dork: ' + dork) - print(e) - return - except Exception as e: - print(e) - print('Error encountered on search of dork: ' + dork) - - if not found: - print('No results for your dork search' + addendum + '. Hurray!') - - -def main(): - parser = argparse.ArgumentParser( - description='Search github for github dorks', - epilog='Use responsibly, Enjoy pentesting') - - parser.add_argument( - '-v', '--version', action='version', version='%(prog)s ' + __version__) - - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument( - '-u', - '--user', - dest='user_to_search', - action='store', - help='Github user/org to search within. Eg: techgaun') - - group.add_argument( - '-r', - '--repo', - dest='repo_to_search', - action='store', - help='Github repo to search within. Eg: techgaun/github-dorks') - - parser.add_argument( - '-d', - '--dork', - dest='gh_dorks_file', - action='store', - help='Github dorks file. Eg: github-dorks.txt') - - group.add_argument( - '-m', - '--monit', - dest='active_monit', - action='store', - help='Monitors Github user private feed with feed token' - ) - - parser.add_argument( - '-o', - '--outputFile', - dest='output_filename', - action='store', - help='CSV File to write results to. This overwrites the file provided! Eg: out.csv' - ) - - args = parser.parse_args() - metasearch( - repo_to_search=args.repo_to_search, - user_to_search=args.user_to_search, - gh_dorks_file=args.gh_dorks_file, - active_monit=args.active_monit, - output_filename=args.output_filename) +from github_dorks.cli import main if __name__ == '__main__': diff --git a/github_dorks/__init__.py b/github_dorks/__init__.py new file mode 100644 index 0000000..b40c3ea --- /dev/null +++ b/github_dorks/__init__.py @@ -0,0 +1,3 @@ +"""Search GitHub repositories for accidentally committed sensitive data.""" + +__version__ = '0.1.1' diff --git a/github_dorks/__main__.py b/github_dorks/__main__.py new file mode 100644 index 0000000..acea987 --- /dev/null +++ b/github_dorks/__main__.py @@ -0,0 +1,5 @@ +from github_dorks.cli import main + + +if __name__ == '__main__': + main() diff --git a/github_dorks/cli.py b/github_dorks/cli.py new file mode 100644 index 0000000..b7d4dbe --- /dev/null +++ b/github_dorks/cli.py @@ -0,0 +1,225 @@ +import github3 as github +import os +import argparse +import csv +import time +import feedparser +from copy import copy +from contextlib import nullcontext +from sys import stderr, prefix + +from github_dorks import __version__ + +gh_user = os.getenv('GH_USER', None) +gh_pass = os.getenv('GH_PWD', None) +gh_token = os.getenv('GH_TOKEN', None) +gh_url = os.getenv('GH_URL', None) + +if gh_url is None: + gh = github.GitHub(username=gh_user, password=gh_pass, token=gh_token) +else: + gh = github.GitHubEnterprise( + url=gh_url, username=gh_user, password=gh_pass, token=gh_token) + + +def search_wrapper(gen): + while True: + gen_back = copy(gen) + try: + yield next(gen) + except StopIteration: + return + except github.exceptions.ForbiddenError: + search_rate_limit = gh.rate_limit()['resources']['search'] + # limit_remaining = search_rate_limit['remaining'] + reset_time = search_rate_limit['reset'] + current_time = int(time.time()) + sleep_time = reset_time - current_time + 1 + stderr.write( + 'GitHub Search API rate limit reached. Sleeping for %d seconds.\n\n' + % (sleep_time)) + time.sleep(sleep_time) + yield next(gen_back) + except Exception as e: + raise e + + +def metasearch(repo_to_search=None, + user_to_search=None, + gh_dorks_file=None, + active_monit=None, + output_filename=None, + refresh_time=60): + if active_monit is None: + search(repo_to_search, user_to_search, gh_dorks_file, active_monit, output_filename) + else: + monit(gh_dorks_file, active_monit, refresh_time) + + +def monit(gh_dorks_file=None, active_monit=None, refresh_time=60): + if gh_user is None: + raise Exception('Error, env Github user variable needed') + else: + print( + 'Monitoring user private feed searching new code to be dorked.' + + 'Every new merged pull request trigger user scan.' + ) + print('-----') + items_history = list() + gh_private_feed = "https://github.com/{}.private.atom?token={}".format( + gh_user, active_monit) + while True: + feed = feedparser.parse(gh_private_feed) + for i in feed['items']: + if 'merged pull' in i['title']: + if i['title'] not in items_history: + search( + user_to_search=i['author_detail']['name'], + gh_dorks_file=gh_dorks_file) + items_history.append(i['title']) + print('Waiting for new items...') + time.sleep(refresh_time) + + +def search(repo_to_search=None, + user_to_search=None, + gh_dorks_file=None, + active_monit=None, + output_filename=None): + + if gh_dorks_file is None: + for path_prefix in ['.', os.path.join(prefix, 'github-dorks/')]: + filename = os.path.join(path_prefix, 'github-dorks.txt') + if os.path.isfile(filename): + gh_dorks_file = filename + break + + if gh_dorks_file is None or not os.path.isfile(gh_dorks_file): + raise Exception('Error, the dorks file path is not valid') + if user_to_search: + print("Scanning User: ", user_to_search) + if repo_to_search: + print("Scanning Repo: ", repo_to_search) + found = False + + output_context = ( + open(output_filename, 'w', newline='', encoding='utf-8') + if output_filename else nullcontext(None) + ) + + with open(gh_dorks_file, 'r', encoding='utf-8') as dork_file, output_context as output_file: + # Write CSV Header + csv_writer = None + if output_file: + csv_writer = csv.writer(output_file) + csv_writer.writerow([ + 'Issue Type (Dork)', 'Text Matches', 'File Path', + 'Score/Relevance', 'URL of File' + ]) + for dork in dork_file: + dork = dork.strip() + if not dork or dork[0] in '#;': + continue + addendum = '' + if repo_to_search: + addendum = ' repo:' + repo_to_search + elif user_to_search: + addendum = ' user:' + user_to_search + + dork = dork + addendum + search_results = search_wrapper(gh.search_code(dork)) + try: + for search_result in search_results: + found = True + fmt_args = { + 'dork': dork, + 'text_matches': search_result.text_matches, + 'path': search_result.path, + 'score': search_result.score, + 'url': search_result.html_url + } + + # Either write to file or print output + if csv_writer: + csv_writer.writerow([ + fmt_args['dork'], fmt_args['text_matches'], + fmt_args['path'], fmt_args['score'], fmt_args['url'] + ]) + else: + result = '\n'.join([ + 'Found result for {dork}', + 'Text matches: {text_matches}', 'File path: {path}', + 'Score/Relevance: {score}', 'URL of File: {url}', '' + ]).format(**fmt_args) + print(result) + + except github.exceptions.GitHubError as e: + print('GitHubError encountered on search of dork: ' + dork) + print(e) + return + except Exception as e: + print(e) + print('Error encountered on search of dork: ' + dork) + + if not found: + print('No results for your dork search' + addendum + '. Hurray!') + + +def main(): + parser = argparse.ArgumentParser( + prog='github-dorks', + description='Search GitHub for sensitive data patterns', + epilog='Use responsibly. Only scan repositories you are authorized to assess.') + + parser.add_argument( + '-v', '--version', action='version', version='%(prog)s ' + __version__) + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + '-u', + '--user', + dest='user_to_search', + action='store', + help='GitHub user/org to search within. Eg: techgaun') + + group.add_argument( + '-r', + '--repo', + dest='repo_to_search', + action='store', + help='GitHub repo to search within. Eg: techgaun/github-dorks') + + parser.add_argument( + '-d', + '--dork', + dest='gh_dorks_file', + action='store', + help='GitHub dorks file. Eg: github-dorks.txt') + + group.add_argument( + '-m', + '--monit', + dest='active_monit', + action='store', + help='Monitors GitHub user private feed with feed token' + ) + + parser.add_argument( + '-o', + '--outputFile', + dest='output_filename', + action='store', + help='CSV File to write results to. This overwrites the file provided! Eg: out.csv' + ) + + args = parser.parse_args() + metasearch( + repo_to_search=args.repo_to_search, + user_to_search=args.user_to_search, + gh_dorks_file=args.gh_dorks_file, + active_monit=args.active_monit, + output_filename=args.output_filename) + + +if __name__ == '__main__': + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..87673d4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "github-dorks" +dynamic = ["version"] +description = "Find leaked secrets with GitHub code search." +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +authors = [ + { name = "Samar Dhwoj Acharya", email = "coolsamar207@gmail.com" }, +] +dependencies = [ + "feedparser>=6.0.12,<7", + "github3.py==4.0.1", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Information Technology", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Security", +] + +[project.urls] +Homepage = "https://github.com/techgaun/github-dorks" +Issues = "https://github.com/techgaun/github-dorks/issues" + +[project.scripts] +github-dorks = "github_dorks.cli:main" + +[tool.setuptools.dynamic] +version = { attr = "github_dorks.__version__" } + +[tool.setuptools.data-files] +github-dorks = ["github-dorks.txt"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 31ad16d..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -github3.py==4.0.1 -feedparser>=6.0.12,<7 diff --git a/setup.py b/setup.py deleted file mode 100644 index 8563c59..0000000 --- a/setup.py +++ /dev/null @@ -1,21 +0,0 @@ -from setuptools import setup - -with open('README.md', 'r') as f: - long_description = f.read() - -setup( - name='github-dorks', - version='0.1.1', - description='Find leaked secrets via github search.', - license='Apache License 2.0', - long_description=long_description, - author='Samar Dhwoj Acharya (@techgaun)', - long_description_content_type='text/markdown', - scripts=['github-dork.py'], - data_files=[('github-dorks', ['github-dorks.txt'])], - python_requires='>=3.10', - install_requires=[ - 'github3.py==4.0.1', - 'feedparser>=6.0.12,<7', - ], -) diff --git a/tests/test_github_dork.py b/tests/test_github_dork.py index 2cbc79b..54616f2 100644 --- a/tests/test_github_dork.py +++ b/tests/test_github_dork.py @@ -1,5 +1,4 @@ import csv -import importlib.util import io import sys import tempfile @@ -28,11 +27,7 @@ class FakeForbiddenError(FakeGitHubError): sys.modules.setdefault('github3', fake_github3) sys.modules.setdefault('feedparser', types.ModuleType('feedparser')) -spec = importlib.util.spec_from_file_location( - 'github_dork', Path(__file__).parents[1] / 'github-dork.py' -) -github_dork = importlib.util.module_from_spec(spec) -spec.loader.exec_module(github_dork) +from github_dorks import __version__, cli as github_dork # noqa: E402 class SearchResult: @@ -90,6 +85,17 @@ def test_rejects_missing_dorks_file_with_clear_error(self): github_dork.search(gh_dorks_file='/does/not/exist') +class CommandLineTests(unittest.TestCase): + def test_version_comes_from_package_metadata(self): + stdout = io.StringIO() + with patch.object(sys, 'argv', ['github-dorks', '--version']): + with redirect_stdout(stdout), self.assertRaises(SystemExit) as exit_info: + github_dork.main() + + self.assertEqual(exit_info.exception.code, 0) + self.assertEqual(stdout.getvalue().strip(), f'github-dorks {__version__}') + + class DorkDictionaryTests(unittest.TestCase): @classmethod def setUpClass(cls):