From 3d9a3920be8e8f7d3fcc8b0a4651d6dc6e18ada7 Mon Sep 17 00:00:00 2001 From: zodetron Date: Mon, 12 Oct 2026 01:42:50 +0530 Subject: [PATCH 1/4] add exporter to export WARC/WET/WAT record metadata to CSV or JSON --- .gitignore | 1 + CHANGES.md | 17 ++++++++++++ Makefile | 8 ++++++ exporter/README.md | 36 ++++++++++++++++++++++++ exporter/export_records.py | 56 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 118 insertions(+) create mode 100644 CHANGES.md create mode 100644 exporter/README.md create mode 100644 exporter/export_records.py diff --git a/.gitignore b/.gitignore index 1075339..aa75e36 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ collinfo.json .venv/ .vscode/ .idea/ +exporter/output/ diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..fd1625a --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,17 @@ +# Changes + +## Record metadata export + +Added `exporter/export_records.py`, which exports record metadata to CSV or JSON. + +- Reads WARC, WET, or WAT files, local or remote, using `fsspec` and `warcio`, the same way `warcio-iterator.py` does. +- Extracts one row per record: type, URI, date, content type, length, and HTTP status. +- Reads records one at a time with a generator and keeps only their headers, so page content is never loaded into memory. +- Writes with `csv.DictWriter` or `json.dump`, chosen by `-f`. +- Uses only the standard library and packages already in `requirements.txt`. + +Other changes: + +- `Makefile`: added `make export`, which writes the outputs to `exporter/output/`. +- `.gitignore`: ignores `exporter/output/`. +- `exporter/README.md`: usage and field reference. diff --git a/Makefile b/Makefile index c3a33a9..ee1e261 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,14 @@ extract-remote-s3: @echo "extracting before-you-ship.18f.gov record from End-of-Term Common Crawl repackage WARC over S3 (offset 18595):" warcio extract $(EOT_CC_WARC_S3) 18595 +export: + @echo "exporting record metadata from the local warcs to exporter/output/" + mkdir -p exporter/output + python exporter/export_records.py whirlwind.warc.gz -o exporter/output/warc.csv + python exporter/export_records.py whirlwind.warc.wet.gz -o exporter/output/wet.csv + python exporter/export_records.py whirlwind.warc.wat.gz -o exporter/output/wat.csv + python exporter/export_records.py whirlwind.warc.gz -f json -o exporter/output/warc.json + cdx_toolkit: @echo demonstrate that we have this entry in the index cdxt --crawl CC-MAIN-2024-22 --from 20240518015810 --to 20240518015810 iter an.wikipedia.org/wiki/Escopete diff --git a/exporter/README.md b/exporter/README.md new file mode 100644 index 0000000..5235a96 --- /dev/null +++ b/exporter/README.md @@ -0,0 +1,36 @@ +# Exporter + +Exports record metadata from a WARC, WET, or WAT file to CSV or JSON. + +## Usage + +``` +python exporter/export_records.py [-f csv|json] [-o output] +``` + +- `input`: local path or remote URL (`https://`, `s3://`) +- `-f`: output format, `csv` (default) or `json` +- `-o`: output file, stdout if omitted + +Example: + +``` +python exporter/export_records.py whirlwind.warc.gz -f json -o records.json +``` + +Or export all local files into `exporter/output/`: + +``` +make export +``` + +## Fields + +| Field | Source | +|---|---| +| `warc_type` | `WARC-Type` | +| `target_uri` | `WARC-Target-URI` | +| `date` | `WARC-Date` | +| `content_type` | `Content-Type` | +| `content_length` | `Content-Length` | +| `http_status` | HTTP status code (response records only) | diff --git a/exporter/export_records.py b/exporter/export_records.py new file mode 100644 index 0000000..b072f73 --- /dev/null +++ b/exporter/export_records.py @@ -0,0 +1,56 @@ +'''Export WARC, WET, or WAT record metadata to CSV or JSON.''' + +import argparse +import contextlib +import csv +import json +import sys + +import fsspec +from warcio.archiveiterator import ArchiveIterator + +FIELDS = ('warc_type', 'target_uri', 'date', 'content_type', 'content_length', 'http_status') + + +def read_records(path): + with fsspec.open(path, 'rb') as stream: + for record in ArchiveIterator(stream): + headers = record.rec_headers + yield { + 'warc_type': record.rec_type, + 'target_uri': headers.get_header('WARC-Target-URI'), + 'date': headers.get_header('WARC-Date'), + 'content_type': headers.get_header('Content-Type'), + 'content_length': int(headers.get_header('Content-Length')), + 'http_status': int(record.http_headers.get_statuscode()) if record.rec_type == 'response' else None, + } + + +def write_csv(records, out): + writer = csv.DictWriter(out, fieldnames=FIELDS) + writer.writeheader() + writer.writerows(records) + + +def write_json(records, out): + json.dump(list(records), out, indent=2) + out.write('\n') + + +WRITERS = {'csv': write_csv, 'json': write_json} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('input', help='local path or URL (https://, s3://) of a WARC, WET, or WAT file') + parser.add_argument('-f', '--format', choices=WRITERS, default='csv') + parser.add_argument('-o', '--output', help='output file (default: stdout)') + args = parser.parse_args() + + out = open(args.output, 'w', newline='', encoding='utf-8') if args.output else contextlib.nullcontext(sys.stdout) + with out as stream: + WRITERS[args.format](read_records(args.input), stream) + + +if __name__ == '__main__': + main() From c320224595c9d968c07f904859abf83f061ea014 Mon Sep 17 00:00:00 2001 From: zodetron Date: Sat, 17 Oct 2026 00:12:40 +0530 Subject: [PATCH 2/4] add unit tests for record metadata exporter --- .gitignore | 1 + tests/test_exporter.py | 58 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 tests/test_exporter.py diff --git a/.gitignore b/.gitignore index aa75e36..8521baf 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ collinfo.json .vscode/ .idea/ exporter/output/ +progress.md diff --git a/tests/test_exporter.py b/tests/test_exporter.py new file mode 100644 index 0000000..ceef84f --- /dev/null +++ b/tests/test_exporter.py @@ -0,0 +1,58 @@ +'''Tests for exporter/export_records.py against the sample whirlwind files.''' + +import csv +import io +import json +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'exporter')) + +from export_records import FIELDS, read_records, write_csv, write_json # noqa: E402 + + +def records(name): + return list(read_records(str(ROOT / name))) + + +class TestReadRecords(unittest.TestCase): + def test_record_types(self): + cases = { + 'whirlwind.warc.gz': ['warcinfo', 'request', 'response', 'metadata'], + 'whirlwind.warc.wet.gz': ['warcinfo', 'conversion'], + 'whirlwind.warc.wat.gz': ['warcinfo', 'metadata'], + } + for name, types in cases.items(): + with self.subTest(name=name): + self.assertEqual([r['warc_type'] for r in records(name)], types) + + def test_response_fields(self): + response = records('whirlwind.warc.gz')[2] + self.assertEqual(response['target_uri'], 'https://an.wikipedia.org/wiki/Escopete') + self.assertEqual(response['http_status'], 200) + self.assertGreater(response['content_length'], 0) + + def test_status_only_on_response(self): + for record in records('whirlwind.warc.gz'): + if record['warc_type'] != 'response': + self.assertIsNone(record['http_status']) + + +class TestWriters(unittest.TestCase): + def test_csv(self): + out = io.StringIO() + write_csv(records('whirlwind.warc.gz'), out) + rows = list(csv.DictReader(io.StringIO(out.getvalue()))) + self.assertEqual(len(rows), 4) + self.assertEqual(tuple(rows[0]), FIELDS) + + def test_json(self): + out = io.StringIO() + write_json(records('whirlwind.warc.gz'), out) + self.assertEqual(json.loads(out.getvalue()), records('whirlwind.warc.gz')) + + +if __name__ == '__main__': + unittest.main() From c124fbcf9dac9242c2b61fbd9b67e110c5be834f Mon Sep 17 00:00:00 2001 From: zodetron Date: Sat, 17 Oct 2026 01:53:37 +0530 Subject: [PATCH 3/4] add text mode to export WET page text --- CHANGES.md | 14 +++++++++ Makefile | 3 +- exporter/README.md | 17 +++++++++-- exporter/export_records.py | 60 ++++++++++++++++++++++++++------------ tests/test_exporter.py | 21 +++++++++++-- 5 files changed, 90 insertions(+), 25 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fd1625a..208daa1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,3 +15,17 @@ Other changes: - `Makefile`: added `make export`, which writes the outputs to `exporter/output/`. - `.gitignore`: ignores `exporter/output/`. - `exporter/README.md`: usage and field reference. + +## Page text export + +Added a `text` mode to `exporter/export_records.py` (`-m text`), which exports one row per WET `conversion` record: URL, language, word count, and a 200-character preview. + +- `iter_archive()` now holds the shared open-and-iterate code, used by both readers. +- `-m` picks a reader and its CSV fields, so the writers stay format-only. +- JSON output keeps non-ASCII characters as-is (`ensure_ascii=False`). + +Other changes: + +- `Makefile`: `make export` also writes `exporter/output/text.csv`. +- `exporter/README.md`: documents `-m` and the `text` fields. +- `tests/test_exporter.py`: tests for the `text` mode. diff --git a/Makefile b/Makefile index ee1e261..fe8c2ef 100644 --- a/Makefile +++ b/Makefile @@ -80,12 +80,13 @@ extract-remote-s3: warcio extract $(EOT_CC_WARC_S3) 18595 export: - @echo "exporting record metadata from the local warcs to exporter/output/" + @echo "exporting record metadata and page text from the local warcs to exporter/output/" mkdir -p exporter/output python exporter/export_records.py whirlwind.warc.gz -o exporter/output/warc.csv python exporter/export_records.py whirlwind.warc.wet.gz -o exporter/output/wet.csv python exporter/export_records.py whirlwind.warc.wat.gz -o exporter/output/wat.csv python exporter/export_records.py whirlwind.warc.gz -f json -o exporter/output/warc.json + python exporter/export_records.py whirlwind.warc.wet.gz -m text -o exporter/output/text.csv cdx_toolkit: @echo demonstrate that we have this entry in the index diff --git a/exporter/README.md b/exporter/README.md index 5235a96..dc53fb2 100644 --- a/exporter/README.md +++ b/exporter/README.md @@ -1,14 +1,15 @@ # Exporter -Exports record metadata from a WARC, WET, or WAT file to CSV or JSON. +Exports record metadata from a WARC, WET, or WAT file, or page text from a WET file, to CSV or JSON. ## Usage ``` -python exporter/export_records.py [-f csv|json] [-o output] +python exporter/export_records.py [-m records|text] [-f csv|json] [-o output] ``` - `input`: local path or remote URL (`https://`, `s3://`) +- `-m`: mode, `records` (default) or `text` - `-f`: output format, `csv` (default) or `json` - `-o`: output file, stdout if omitted @@ -16,6 +17,7 @@ Example: ``` python exporter/export_records.py whirlwind.warc.gz -f json -o records.json +python exporter/export_records.py whirlwind.warc.wet.gz -m text ``` Or export all local files into `exporter/output/`: @@ -26,6 +28,8 @@ make export ## Fields +`records` mode, one row per record: + | Field | Source | |---|---| | `warc_type` | `WARC-Type` | @@ -34,3 +38,12 @@ make export | `content_type` | `Content-Type` | | `content_length` | `Content-Length` | | `http_status` | HTTP status code (response records only) | + +`text` mode, one row per WET `conversion` record: + +| Field | Source | +|---|---| +| `url` | `WARC-Target-URI` | +| `language` | `WARC-Identified-Content-Language` | +| `word_count` | number of whitespace-separated words in the text | +| `preview` | first 200 characters of the text, whitespace collapsed | diff --git a/exporter/export_records.py b/exporter/export_records.py index b072f73..74c8a0a 100644 --- a/exporter/export_records.py +++ b/exporter/export_records.py @@ -1,4 +1,4 @@ -'''Export WARC, WET, or WAT record metadata to CSV or JSON.''' +'''Export WARC, WET, or WAT record metadata or WET page text to CSV or JSON.''' import argparse import contextlib @@ -10,46 +10,68 @@ from warcio.archiveiterator import ArchiveIterator FIELDS = ('warc_type', 'target_uri', 'date', 'content_type', 'content_length', 'http_status') +TEXT_FIELDS = ('url', 'language', 'word_count', 'preview') +PREVIEW_CHARS = 200 -def read_records(path): +def iter_archive(path): with fsspec.open(path, 'rb') as stream: - for record in ArchiveIterator(stream): - headers = record.rec_headers - yield { - 'warc_type': record.rec_type, - 'target_uri': headers.get_header('WARC-Target-URI'), - 'date': headers.get_header('WARC-Date'), - 'content_type': headers.get_header('Content-Type'), - 'content_length': int(headers.get_header('Content-Length')), - 'http_status': int(record.http_headers.get_statuscode()) if record.rec_type == 'response' else None, - } - - -def write_csv(records, out): - writer = csv.DictWriter(out, fieldnames=FIELDS) + yield from ArchiveIterator(stream) + + +def read_records(path): + for record in iter_archive(path): + headers = record.rec_headers + yield { + 'warc_type': record.rec_type, + 'target_uri': headers.get_header('WARC-Target-URI'), + 'date': headers.get_header('WARC-Date'), + 'content_type': headers.get_header('Content-Type'), + 'content_length': int(headers.get_header('Content-Length')), + 'http_status': int(record.http_headers.get_statuscode()) if record.rec_type == 'response' else None, + } + + +def read_text(path): + for record in iter_archive(path): + if record.rec_type != 'conversion': + continue + words = record.content_stream().read().decode('utf-8', errors='replace').split() + yield { + 'url': record.rec_headers.get_header('WARC-Target-URI'), + 'language': record.rec_headers.get_header('WARC-Identified-Content-Language'), + 'word_count': len(words), + 'preview': ' '.join(words)[:PREVIEW_CHARS], + } + + +def write_csv(records, fields, out): + writer = csv.DictWriter(out, fieldnames=fields) writer.writeheader() writer.writerows(records) -def write_json(records, out): - json.dump(list(records), out, indent=2) +def write_json(records, fields, out): + json.dump(list(records), out, indent=2, ensure_ascii=False) out.write('\n') +MODES = {'records': (read_records, FIELDS), 'text': (read_text, TEXT_FIELDS)} WRITERS = {'csv': write_csv, 'json': write_json} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('input', help='local path or URL (https://, s3://) of a WARC, WET, or WAT file') + parser.add_argument('-m', '--mode', choices=MODES, default='records', help='records: metadata of every record; text: page text from WET') parser.add_argument('-f', '--format', choices=WRITERS, default='csv') parser.add_argument('-o', '--output', help='output file (default: stdout)') args = parser.parse_args() + reader, fields = MODES[args.mode] out = open(args.output, 'w', newline='', encoding='utf-8') if args.output else contextlib.nullcontext(sys.stdout) with out as stream: - WRITERS[args.format](read_records(args.input), stream) + WRITERS[args.format](reader(args.input), fields, stream) if __name__ == '__main__': diff --git a/tests/test_exporter.py b/tests/test_exporter.py index ceef84f..caf8608 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / 'exporter')) -from export_records import FIELDS, read_records, write_csv, write_json # noqa: E402 +from export_records import FIELDS, TEXT_FIELDS, read_records, read_text, write_csv, write_json # noqa: E402 def records(name): @@ -40,17 +40,32 @@ def test_status_only_on_response(self): self.assertIsNone(record['http_status']) +class TestReadText(unittest.TestCase): + def test_wet(self): + rows = list(read_text(str(ROOT / 'whirlwind.warc.wet.gz'))) + self.assertEqual(len(rows), 1) + self.assertEqual(tuple(rows[0]), TEXT_FIELDS) + self.assertEqual(rows[0]['url'], 'https://an.wikipedia.org/wiki/Escopete') + self.assertEqual(rows[0]['language'], 'spa') + self.assertGreater(rows[0]['word_count'], 0) + self.assertTrue(rows[0]['preview'].startswith('Escopete')) + self.assertLessEqual(len(rows[0]['preview']), 200) + + def test_skips_non_conversion(self): + self.assertEqual(list(read_text(str(ROOT / 'whirlwind.warc.gz'))), []) + + class TestWriters(unittest.TestCase): def test_csv(self): out = io.StringIO() - write_csv(records('whirlwind.warc.gz'), out) + write_csv(records('whirlwind.warc.gz'), FIELDS, out) rows = list(csv.DictReader(io.StringIO(out.getvalue()))) self.assertEqual(len(rows), 4) self.assertEqual(tuple(rows[0]), FIELDS) def test_json(self): out = io.StringIO() - write_json(records('whirlwind.warc.gz'), out) + write_json(records('whirlwind.warc.gz'), FIELDS, out) self.assertEqual(json.loads(out.getvalue()), records('whirlwind.warc.gz')) From 4b12d648db33e79d5b359ed5476ab7f76ca33c84 Mon Sep 17 00:00:00 2001 From: zodetron Date: Sat, 17 Oct 2026 13:52:23 +0530 Subject: [PATCH 4/4] add links mode to export page links from WARC HTML --- CHANGES.md | 15 +++++++++++++ Makefile | 3 ++- exporter/README.md | 15 ++++++++++--- exporter/export_records.py | 45 +++++++++++++++++++++++++++++++++++--- tests/test_exporter.py | 20 ++++++++++++++++- 5 files changed, 90 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 208daa1..9fa139d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,3 +29,18 @@ Other changes: - `Makefile`: `make export` also writes `exporter/output/text.csv`. - `exporter/README.md`: documents `-m` and the `text` fields. - `tests/test_exporter.py`: tests for the `text` mode. + +## Page links export + +Added a `links` mode to `exporter/export_records.py` (`-m links`), which exports one row per link found in the HTML of each WARC `response` record: page URL, link URL, and link text. + +- Links come from the WARC HTML because the sample WAT file has no link data (its only metadata record describes the crawl request). +- `LinkParser` uses the standard library's `html.parser`, so no new packages are needed. +- Relative and protocol-relative links (`/wiki/...`, `//donate...`) are resolved to absolute URLs with `urljoin`. +- Non-HTML responses are skipped. + +Other changes: + +- `Makefile`: `make export` also writes `exporter/output/links.csv`. +- `exporter/README.md`: documents the `links` fields. +- `tests/test_exporter.py`: tests for the `links` mode and `LinkParser`. diff --git a/Makefile b/Makefile index fe8c2ef..a5cc3ba 100644 --- a/Makefile +++ b/Makefile @@ -80,13 +80,14 @@ extract-remote-s3: warcio extract $(EOT_CC_WARC_S3) 18595 export: - @echo "exporting record metadata and page text from the local warcs to exporter/output/" + @echo "exporting record metadata, page text, and links from the local warcs to exporter/output/" mkdir -p exporter/output python exporter/export_records.py whirlwind.warc.gz -o exporter/output/warc.csv python exporter/export_records.py whirlwind.warc.wet.gz -o exporter/output/wet.csv python exporter/export_records.py whirlwind.warc.wat.gz -o exporter/output/wat.csv python exporter/export_records.py whirlwind.warc.gz -f json -o exporter/output/warc.json python exporter/export_records.py whirlwind.warc.wet.gz -m text -o exporter/output/text.csv + python exporter/export_records.py whirlwind.warc.gz -m links -o exporter/output/links.csv cdx_toolkit: @echo demonstrate that we have this entry in the index diff --git a/exporter/README.md b/exporter/README.md index dc53fb2..5cd11ed 100644 --- a/exporter/README.md +++ b/exporter/README.md @@ -1,15 +1,15 @@ # Exporter -Exports record metadata from a WARC, WET, or WAT file, or page text from a WET file, to CSV or JSON. +Exports record metadata from a WARC, WET, or WAT file, page text from a WET file, or page links from a WARC file, to CSV or JSON. ## Usage ``` -python exporter/export_records.py [-m records|text] [-f csv|json] [-o output] +python exporter/export_records.py [-m records|text|links] [-f csv|json] [-o output] ``` - `input`: local path or remote URL (`https://`, `s3://`) -- `-m`: mode, `records` (default) or `text` +- `-m`: mode, `records` (default), `text`, or `links` - `-f`: output format, `csv` (default) or `json` - `-o`: output file, stdout if omitted @@ -18,6 +18,7 @@ Example: ``` python exporter/export_records.py whirlwind.warc.gz -f json -o records.json python exporter/export_records.py whirlwind.warc.wet.gz -m text +python exporter/export_records.py whirlwind.warc.gz -m links ``` Or export all local files into `exporter/output/`: @@ -47,3 +48,11 @@ make export | `language` | `WARC-Identified-Content-Language` | | `word_count` | number of whitespace-separated words in the text | | `preview` | first 200 characters of the text, whitespace collapsed | + +`links` mode, one row per `` in each HTML WARC `response` record: + +| Field | Source | +|---|---| +| `page_url` | `WARC-Target-URI` | +| `link_url` | `href`, resolved against `page_url` | +| `link_text` | text inside the `` tag, whitespace collapsed | diff --git a/exporter/export_records.py b/exporter/export_records.py index 74c8a0a..9e193e2 100644 --- a/exporter/export_records.py +++ b/exporter/export_records.py @@ -1,16 +1,19 @@ -'''Export WARC, WET, or WAT record metadata or WET page text to CSV or JSON.''' +'''Export WARC, WET, or WAT record metadata, WET page text, or WARC page links to CSV or JSON.''' import argparse import contextlib import csv import json import sys +from html.parser import HTMLParser +from urllib.parse import urljoin import fsspec from warcio.archiveiterator import ArchiveIterator FIELDS = ('warc_type', 'target_uri', 'date', 'content_type', 'content_length', 'http_status') TEXT_FIELDS = ('url', 'language', 'word_count', 'preview') +LINK_FIELDS = ('page_url', 'link_url', 'link_text') PREVIEW_CHARS = 200 @@ -45,6 +48,42 @@ def read_text(path): } +class LinkParser(HTMLParser): + def __init__(self): + super().__init__() + self.links, self.href, self.text = [], None, [] + + def handle_starttag(self, tag, attrs): + if tag == 'a': + self.close_link() + self.href = dict(attrs).get('href') + + def handle_data(self, data): + if self.href is not None: + self.text.append(data) + + def handle_endtag(self, tag): + if tag == 'a': + self.close_link() + + def close_link(self): + if self.href: + self.links.append((self.href, ' '.join(''.join(self.text).split()))) + self.href, self.text = None, [] + + +def read_links(path): + for record in iter_archive(path): + if record.rec_type != 'response' or 'html' not in (record.http_headers.get_header('Content-Type') or ''): + continue + page_url = record.rec_headers.get_header('WARC-Target-URI') + parser = LinkParser() + parser.feed(record.content_stream().read().decode('utf-8', errors='replace')) + parser.close() + for href, text in parser.links: + yield {'page_url': page_url, 'link_url': urljoin(page_url, href), 'link_text': text} + + def write_csv(records, fields, out): writer = csv.DictWriter(out, fieldnames=fields) writer.writeheader() @@ -56,14 +95,14 @@ def write_json(records, fields, out): out.write('\n') -MODES = {'records': (read_records, FIELDS), 'text': (read_text, TEXT_FIELDS)} +MODES = {'records': (read_records, FIELDS), 'text': (read_text, TEXT_FIELDS), 'links': (read_links, LINK_FIELDS)} WRITERS = {'csv': write_csv, 'json': write_json} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('input', help='local path or URL (https://, s3://) of a WARC, WET, or WAT file') - parser.add_argument('-m', '--mode', choices=MODES, default='records', help='records: metadata of every record; text: page text from WET') + parser.add_argument('-m', '--mode', choices=MODES, default='records', help='records: metadata of every record; text: page text from WET; links: page links from WARC') parser.add_argument('-f', '--format', choices=WRITERS, default='csv') parser.add_argument('-o', '--output', help='output file (default: stdout)') args = parser.parse_args() diff --git a/tests/test_exporter.py b/tests/test_exporter.py index caf8608..d337e65 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / 'exporter')) -from export_records import FIELDS, TEXT_FIELDS, read_records, read_text, write_csv, write_json # noqa: E402 +from export_records import FIELDS, LINK_FIELDS, TEXT_FIELDS, LinkParser, read_links, read_records, read_text, write_csv, write_json # noqa: E402 def records(name): @@ -55,6 +55,24 @@ def test_skips_non_conversion(self): self.assertEqual(list(read_text(str(ROOT / 'whirlwind.warc.gz'))), []) +class TestReadLinks(unittest.TestCase): + def test_warc(self): + rows = list(read_links(str(ROOT / 'whirlwind.warc.gz'))) + self.assertEqual(len(rows), 207) + self.assertEqual(tuple(rows[0]), LINK_FIELDS) + self.assertTrue(all(r['page_url'] == 'https://an.wikipedia.org/wiki/Escopete' for r in rows)) + self.assertTrue(all(r['link_url'].startswith(('http://', 'https://')) for r in rows)) + + def test_skips_non_response(self): + self.assertEqual(list(read_links(str(ROOT / 'whirlwind.warc.wet.gz'))), []) + + def test_parser(self): + parser = LinkParser() + parser.feed('One twono href\n three ') + parser.close() + self.assertEqual(parser.links, [('/a', 'One two'), ('//b.org/', 'three')]) + + class TestWriters(unittest.TestCase): def test_csv(self): out = io.StringIO()