From a26d4a970718626a2dfb09370ff67647dd6c0c2a Mon Sep 17 00:00:00 2001 From: techgaun Date: Mon, 7 Sep 2026 15:19:08 -0500 Subject: [PATCH] feat: add structured output formats --- README.md | 13 +++++ github_dorks/cli.py | 26 ++++++++- github_dorks/output.py | 107 ++++++++++++++++++++++++++++++++++++++ github_dorks/search.py | 106 +++++++++++++++++++------------------ tests/test_github_dork.py | 97 ++++++++++++++++++++++++++++++++++ 5 files changed, 296 insertions(+), 53 deletions(-) create mode 100644 github_dorks/output.py diff --git a/README.md b/README.md index 0f334fc..8d3b65d 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,12 @@ GH_TOKEN= github-dorks -u dev-nepal # search using au GH_URL=https://github.example.com github-dorks -u dev-nepal # search a GitHub Enterprise instance github-dorks -r techgaun/github-dorks --max-retries 5 # retry recoverable failures up to five times + +github-dorks -r techgaun/github-dorks --format json # stream JSON to stdout + +github-dorks -u dev-nepal --format jsonl -o results.jsonl # write newline-delimited JSON + +github-dorks -u dev-nepal --format csv -o results.csv --force # explicitly replace an existing file ``` The legacy `python github-dork.py ...` invocation remains available for @@ -72,6 +78,13 @@ elapsed time. The command exits with status `0` after a complete scan, `2` when one or more queries failed, and `1` for fatal configuration, file, or authentication errors. +Supported result formats are `text`, `csv`, `json`, and `jsonl`. Text is the +default for terminal output; using `-o/--output` without `--format` preserves +the historical CSV default. Existing files are protected unless `--force` is +provided. Use `--quiet` to suppress progress and summaries, or `--verbose` to +print every query as it runs. Structured stdout remains machine-readable because +status messages are sent to stderr. + ### Development Run the dependency-free unit test suite with: diff --git a/github_dorks/cli.py b/github_dorks/cli.py index da1b634..e546339 100644 --- a/github_dorks/cli.py +++ b/github_dorks/cli.py @@ -60,8 +60,26 @@ def build_parser(): help='GitHub dorks file. Eg: github-dorks.txt', ) parser.add_argument( - '-o', '--outputFile', dest='output_filename', - help='CSV file to write results to. This overwrites the provided file.', + '-o', '--output', '--outputFile', dest='output_filename', + help='Write results to this file instead of stdout', + ) + parser.add_argument( + '--format', choices=('text', 'csv', 'json', 'jsonl'), + dest='output_format', + help='Result format (default: text, or CSV when -o is used)', + ) + parser.add_argument( + '-f', '--force', action='store_true', + help='Overwrite an existing output file', + ) + detail_group = parser.add_mutually_exclusive_group() + detail_group.add_argument( + '-q', '--quiet', action='store_true', + help='Suppress progress and summary messages', + ) + detail_group.add_argument( + '--verbose', action='store_true', + help='Report each query as it runs', ) parser.add_argument( '--max-retries', type=int, default=3, @@ -84,6 +102,10 @@ def main(): user_to_search=args.user_to_search, gh_dorks_file=args.gh_dorks_file, output_filename=args.output_filename, + output_format=args.output_format, + force=args.force, + quiet=args.quiet, + verbose=args.verbose, client=create_client(), max_retries=args.max_retries, ) diff --git a/github_dorks/output.py b/github_dorks/output.py new file mode 100644 index 0000000..692cb73 --- /dev/null +++ b/github_dorks/output.py @@ -0,0 +1,107 @@ +"""Streaming result writers for supported output formats.""" + +import csv +import json +from dataclasses import asdict + + +FIELDNAMES = ('dork', 'text_matches', 'path', 'score', 'url') +CSV_HEADERS = ( + 'Issue Type (Dork)', 'Text Matches', 'File Path', + 'Score/Relevance', 'URL of File', +) + + +class TextWriter: + def __init__(self, stream): + self.stream = stream + + def start(self): + pass + + def write(self, result): + self.stream.write('\n'.join([ + 'Found result for {dork}', + 'Text matches: {text_matches}', + 'File path: {path}', + 'Score/Relevance: {score}', + 'URL of File: {url}', + '', + ]).format(**result) + '\n') + + def finish(self, stats): + pass + + +class CsvWriter: + def __init__(self, stream): + self.writer = csv.DictWriter(stream, fieldnames=FIELDNAMES) + + def start(self): + self.writer.writerow(dict(zip(FIELDNAMES, CSV_HEADERS))) + + def write(self, result): + self.writer.writerow(result) + + def finish(self, stats): + pass + + +class JsonWriter: + def __init__(self, stream): + self.stream = stream + self.first_result = True + + def start(self): + self.stream.write('{"results":[') + + def write(self, result): + if not self.first_result: + self.stream.write(',') + json.dump(result, self.stream, default=str, separators=(',', ':')) + self.first_result = False + + def finish(self, stats): + self.stream.write('],"summary":') + json.dump(asdict(stats), self.stream, separators=(',', ':')) + self.stream.write('}\n') + + +class JsonLinesWriter: + def __init__(self, stream): + self.stream = stream + + def start(self): + pass + + def write(self, result): + json.dump( + {'type': 'result', **result}, self.stream, + default=str, separators=(',', ':'), + ) + self.stream.write('\n') + + def finish(self, stats): + json.dump( + {'type': 'summary', **asdict(stats)}, self.stream, + separators=(',', ':'), + ) + self.stream.write('\n') + + +WRITERS = { + 'text': TextWriter, + 'csv': CsvWriter, + 'json': JsonWriter, + 'jsonl': JsonLinesWriter, +} + + +def create_writer(output_format, stream): + try: + return WRITERS[output_format](stream) + except KeyError as error: + supported = ', '.join(WRITERS) + raise ValueError( + f'unsupported output format {output_format!r}; choose from {supported}' + ) from error diff --git a/github_dorks/search.py b/github_dorks/search.py index 5dbcdf2..8c48185 100644 --- a/github_dorks/search.py +++ b/github_dorks/search.py @@ -1,6 +1,5 @@ """GitHub client construction and reliable code-search execution.""" -import csv import os import time from contextlib import nullcontext @@ -11,6 +10,8 @@ import github3 as github from requests import exceptions as requests_exceptions +from github_dorks.output import create_writer + @dataclass class ScanStats: @@ -128,29 +129,19 @@ def iter_search_results(client, query, stats, max_retries=3, sleep(delay) -def _write_result(result, query, csv_writer, stdout): - values = { +def _result_record(result, query): + return { 'dork': query, 'text_matches': result.text_matches, 'path': result.path, 'score': result.score, 'url': result.html_url, } - if csv_writer: - csv_writer.writerow(values.values()) - return - stdout.write('\n'.join([ - 'Found result for {dork}', - 'Text matches: {text_matches}', - 'File path: {path}', - 'Score/Relevance: {score}', - 'URL of File: {url}', - '', - ]).format(**values) + '\n') def search(repo_to_search=None, user_to_search=None, gh_dorks_file=None, - output_filename=None, client=None, max_retries=3, + output_filename=None, output_format=None, force=False, + quiet=False, verbose=False, client=None, max_retries=3, sleep=time.sleep, now=time.time, monotonic=time.monotonic, stdout=None, stderr=None): """Run every dork and return statistics, isolating per-query failures.""" @@ -162,50 +153,63 @@ def search(repo_to_search=None, user_to_search=None, gh_dorks_file=None, raise ValueError('exactly one repository or user scope is required') client = create_client() if client is None else client dorks_path = find_dorks_file(gh_dorks_file) + output_format = output_format or ('csv' if output_filename else 'text') stats = ScanStats() started_at = monotonic() scope = f' repo:{repo_to_search}' if repo_to_search else f' user:{user_to_search}' label = 'Repo' if repo_to_search else 'User' - stdout.write(f'Scanning {label}: {scope.split(":", 1)[1]}\n') + status_stream = stderr if not output_filename and output_format != 'text' else stdout + if not quiet: + status_stream.write(f'Scanning {label}: {scope.split(":", 1)[1]}\n') output_context = ( - open(output_filename, 'w', newline='', encoding='utf-8') - if output_filename else nullcontext(None) + open( + output_filename, 'w' if force else 'x', newline='', encoding='utf-8' + ) if output_filename else nullcontext(stdout) ) with dorks_path.open(encoding='utf-8') as dork_file, output_context as output_file: - csv_writer = csv.writer(output_file) if output_file else None - if csv_writer: - csv_writer.writerow([ - 'Issue Type (Dork)', 'Text Matches', 'File Path', - 'Score/Relevance', 'URL of File', - ]) - for line in dork_file: - dork = line.strip() - if not dork or dork[0] in '#;': - continue - query = dork + scope - stats.queries += 1 - try: - for result in iter_search_results( + writer = create_writer(output_format, output_file) + writer.start() + try: + for line in dork_file: + dork = line.strip() + if not dork or dork[0] in '#;': + continue + query = dork + scope + stats.queries += 1 + if verbose and not quiet: + status_stream.write(f'Searching: {query}\n') + results = iter_search_results( client, query, stats, max_retries, sleep, now, stderr - ): - stats.matches += 1 - _write_result(result, query, csv_writer, stdout) - except Exception as error: - authentication_error = getattr( - github.exceptions, 'AuthenticationFailed', () ) - if authentication_error and isinstance(error, authentication_error): - raise - stats.failures += 1 - stderr.write(f'Query failed: {query}\n{error}\n') - - stats.elapsed_seconds = monotonic() - started_at - if not stats.matches and not stats.failures: - stdout.write(f'No results for your dork search{scope}. Hurray!\n') - stdout.write( - f'Summary: {stats.queries} queries, {stats.matches} matches, ' - f'{stats.failures} failures, {stats.retries} retries, ' - f'{stats.elapsed_seconds:.1f}s elapsed\n' - ) + while True: + try: + result = next(results) + except StopIteration: + break + except Exception as error: + authentication_error = getattr( + github.exceptions, 'AuthenticationFailed', () + ) + if authentication_error and isinstance( + error, authentication_error + ): + raise + stats.failures += 1 + stderr.write(f'Query failed: {query}\n{error}\n') + break + stats.matches += 1 + writer.write(_result_record(result, query)) + finally: + stats.elapsed_seconds = monotonic() - started_at + writer.finish(stats) + + if not quiet: + if not stats.matches and not stats.failures: + status_stream.write(f'No results for your dork search{scope}. Hurray!\n') + status_stream.write( + f'Summary: {stats.queries} queries, {stats.matches} matches, ' + f'{stats.failures} failures, {stats.retries} retries, ' + f'{stats.elapsed_seconds:.1f}s elapsed\n' + ) return stats diff --git a/tests/test_github_dork.py b/tests/test_github_dork.py index 119820a..9045296 100644 --- a/tests/test_github_dork.py +++ b/tests/test_github_dork.py @@ -1,5 +1,6 @@ import csv import io +import json import sys import tempfile import types @@ -107,11 +108,95 @@ def test_writes_valid_csv_and_scopes_query_to_repository(self): self.assertEqual(client.query, 'filename:.env PASSWORD repo:owner/repo') self.assertEqual(len(rows), 2) + self.assertEqual(rows[0][0], 'Issue Type (Dork)') self.assertEqual(rows[1][0], client.query) self.assertEqual(rows[1][1], "['secret, with comma']") self.assertEqual(stats.matches, 1) self.assertEqual(stats.exit_code, 0) + def test_streams_json_document_to_stdout(self): + with tempfile.TemporaryDirectory() as directory: + dorks = Path(directory) / 'dorks.txt' + dorks.write_text('query\n', encoding='utf-8') + stdout = io.StringIO() + stderr = io.StringIO() + github_dork.search( + repo_to_search='owner/repo', gh_dorks_file=str(dorks), + output_format='json', client=GitHubClient(), + stdout=stdout, stderr=stderr, monotonic=lambda: 10, + ) + + document = json.loads(stdout.getvalue()) + self.assertEqual(document['results'][0]['path'], SearchResult.path) + self.assertEqual(document['summary']['matches'], 1) + self.assertNotIn('Scanning', stdout.getvalue()) + self.assertIn('Summary: 1 queries, 1 matches', stderr.getvalue()) + + def test_streams_typed_json_lines(self): + with tempfile.TemporaryDirectory() as directory: + dorks = Path(directory) / 'dorks.txt' + dorks.write_text('query\n', encoding='utf-8') + stdout = io.StringIO() + github_dork.search( + repo_to_search='owner/repo', gh_dorks_file=str(dorks), + output_format='jsonl', quiet=True, client=GitHubClient(), + stdout=stdout, stderr=io.StringIO(), + ) + + records = [json.loads(line) for line in stdout.getvalue().splitlines()] + self.assertEqual([record['type'] for record in records], ['result', 'summary']) + self.assertEqual(records[1]['matches'], 1) + + def test_quiet_suppresses_status_but_not_text_results(self): + with tempfile.TemporaryDirectory() as directory: + dorks = Path(directory) / 'dorks.txt' + dorks.write_text('query\n', encoding='utf-8') + stdout = io.StringIO() + github_dork.search( + repo_to_search='owner/repo', gh_dorks_file=str(dorks), + quiet=True, client=GitHubClient(), stdout=stdout, + ) + + self.assertIn('Found result', stdout.getvalue()) + self.assertNotIn('Scanning', stdout.getvalue()) + self.assertNotIn('Summary', stdout.getvalue()) + + def test_refuses_to_overwrite_output_without_force(self): + with tempfile.TemporaryDirectory() as directory: + dorks = Path(directory) / 'dorks.txt' + output = Path(directory) / 'results.json' + dorks.write_text('query\n', encoding='utf-8') + output.write_text('keep me', encoding='utf-8') + + with self.assertRaises(FileExistsError): + github_dork.search( + repo_to_search='owner/repo', gh_dorks_file=str(dorks), + output_filename=str(output), output_format='json', + client=GitHubClient(), + ) + self.assertEqual(output.read_text(encoding='utf-8'), 'keep me') + + github_dork.search( + repo_to_search='owner/repo', gh_dorks_file=str(dorks), + output_filename=str(output), output_format='json', force=True, + quiet=True, client=GitHubClient(), + ) + self.assertEqual(json.loads(output.read_text())['summary']['matches'], 1) + + def test_output_failure_is_fatal(self): + class BrokenStream(io.StringIO): + def write(self, value): + raise OSError('disk full') + + with tempfile.TemporaryDirectory() as directory: + dorks = Path(directory) / 'dorks.txt' + dorks.write_text('query\n', encoding='utf-8') + with self.assertRaisesRegex(OSError, 'disk full'): + github_dork.search( + repo_to_search='owner/repo', gh_dorks_file=str(dorks), + quiet=True, client=GitHubClient(), stdout=BrokenStream(), + ) + def test_reports_when_no_results_are_found(self): client = GitHubClient() client.search_code = lambda query: iter([]) @@ -245,6 +330,18 @@ def test_authentication_failure_is_fatal(self): class CommandLineTests(unittest.TestCase): + def test_parses_output_controls(self): + arguments = [ + '-r', 'owner/repo', '--format', 'json', '--output', 'results.json', + '--force', '--verbose', + ] + args = github_dork.build_parser().parse_args(arguments) + + self.assertEqual(args.output_format, 'json') + self.assertEqual(args.output_filename, 'results.json') + self.assertTrue(args.force) + self.assertTrue(args.verbose) + def test_version_comes_from_package_metadata(self): stdout = io.StringIO() with patch.object(sys, 'argv', ['github-dorks', '--version']):