diff --git a/.gitignore b/.gitignore index 1075339..8521baf 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ collinfo.json .venv/ .vscode/ .idea/ +exporter/output/ +progress.md diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..9fa139d --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,46 @@ +# Changes + +## Record metadata export + +Added `exporter/export_records.py`, which exports record metadata to CSV or JSON. + +- Reads WARC, WET, or WAT files, local or remote, using `fsspec` and `warcio`, the same way `warcio-iterator.py` does. +- Extracts one row per record: type, URI, date, content type, length, and HTTP status. +- Reads records one at a time with a generator and keeps only their headers, so page content is never loaded into memory. +- Writes with `csv.DictWriter` or `json.dump`, chosen by `-f`. +- Uses only the standard library and packages already in `requirements.txt`. + +Other changes: + +- `Makefile`: added `make export`, which writes the outputs to `exporter/output/`. +- `.gitignore`: ignores `exporter/output/`. +- `exporter/README.md`: usage and field reference. + +## Page text export + +Added a `text` mode to `exporter/export_records.py` (`-m text`), which exports one row per WET `conversion` record: URL, language, word count, and a 200-character preview. + +- `iter_archive()` now holds the shared open-and-iterate code, used by both readers. +- `-m` picks a reader and its CSV fields, so the writers stay format-only. +- JSON output keeps non-ASCII characters as-is (`ensure_ascii=False`). + +Other changes: + +- `Makefile`: `make export` also writes `exporter/output/text.csv`. +- `exporter/README.md`: documents `-m` and the `text` fields. +- `tests/test_exporter.py`: tests for the `text` mode. + +## Page links export + +Added a `links` mode to `exporter/export_records.py` (`-m links`), which exports one row per link found in the HTML of each WARC `response` record: page URL, link URL, and link text. + +- Links come from the WARC HTML because the sample WAT file has no link data (its only metadata record describes the crawl request). +- `LinkParser` uses the standard library's `html.parser`, so no new packages are needed. +- Relative and protocol-relative links (`/wiki/...`, `//donate...`) are resolved to absolute URLs with `urljoin`. +- Non-HTML responses are skipped. + +Other changes: + +- `Makefile`: `make export` also writes `exporter/output/links.csv`. +- `exporter/README.md`: documents the `links` fields. +- `tests/test_exporter.py`: tests for the `links` mode and `LinkParser`. diff --git a/Makefile b/Makefile index c3a33a9..a5cc3ba 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,16 @@ extract-remote-s3: @echo "extracting before-you-ship.18f.gov record from End-of-Term Common Crawl repackage WARC over S3 (offset 18595):" warcio extract $(EOT_CC_WARC_S3) 18595 +export: + @echo "exporting record metadata, page text, and links from the local warcs to exporter/output/" + mkdir -p exporter/output + python exporter/export_records.py whirlwind.warc.gz -o exporter/output/warc.csv + python exporter/export_records.py whirlwind.warc.wet.gz -o exporter/output/wet.csv + python exporter/export_records.py whirlwind.warc.wat.gz -o exporter/output/wat.csv + python exporter/export_records.py whirlwind.warc.gz -f json -o exporter/output/warc.json + python exporter/export_records.py whirlwind.warc.wet.gz -m text -o exporter/output/text.csv + python exporter/export_records.py whirlwind.warc.gz -m links -o exporter/output/links.csv + cdx_toolkit: @echo demonstrate that we have this entry in the index cdxt --crawl CC-MAIN-2024-22 --from 20240518015810 --to 20240518015810 iter an.wikipedia.org/wiki/Escopete diff --git a/exporter/README.md b/exporter/README.md new file mode 100644 index 0000000..5cd11ed --- /dev/null +++ b/exporter/README.md @@ -0,0 +1,58 @@ +# Exporter + +Exports record metadata from a WARC, WET, or WAT file, page text from a WET file, or page links from a WARC file, to CSV or JSON. + +## Usage + +``` +python exporter/export_records.py [-m records|text|links] [-f csv|json] [-o output] +``` + +- `input`: local path or remote URL (`https://`, `s3://`) +- `-m`: mode, `records` (default), `text`, or `links` +- `-f`: output format, `csv` (default) or `json` +- `-o`: output file, stdout if omitted + +Example: + +``` +python exporter/export_records.py whirlwind.warc.gz -f json -o records.json +python exporter/export_records.py whirlwind.warc.wet.gz -m text +python exporter/export_records.py whirlwind.warc.gz -m links +``` + +Or export all local files into `exporter/output/`: + +``` +make export +``` + +## Fields + +`records` mode, one row per record: + +| Field | Source | +|---|---| +| `warc_type` | `WARC-Type` | +| `target_uri` | `WARC-Target-URI` | +| `date` | `WARC-Date` | +| `content_type` | `Content-Type` | +| `content_length` | `Content-Length` | +| `http_status` | HTTP status code (response records only) | + +`text` mode, one row per WET `conversion` record: + +| Field | Source | +|---|---| +| `url` | `WARC-Target-URI` | +| `language` | `WARC-Identified-Content-Language` | +| `word_count` | number of whitespace-separated words in the text | +| `preview` | first 200 characters of the text, whitespace collapsed | + +`links` mode, one row per `` in each HTML WARC `response` record: + +| Field | Source | +|---|---| +| `page_url` | `WARC-Target-URI` | +| `link_url` | `href`, resolved against `page_url` | +| `link_text` | text inside the `` tag, whitespace collapsed | diff --git a/exporter/export_records.py b/exporter/export_records.py new file mode 100644 index 0000000..9e193e2 --- /dev/null +++ b/exporter/export_records.py @@ -0,0 +1,117 @@ +'''Export WARC, WET, or WAT record metadata, WET page text, or WARC page links to CSV or JSON.''' + +import argparse +import contextlib +import csv +import json +import sys +from html.parser import HTMLParser +from urllib.parse import urljoin + +import fsspec +from warcio.archiveiterator import ArchiveIterator + +FIELDS = ('warc_type', 'target_uri', 'date', 'content_type', 'content_length', 'http_status') +TEXT_FIELDS = ('url', 'language', 'word_count', 'preview') +LINK_FIELDS = ('page_url', 'link_url', 'link_text') +PREVIEW_CHARS = 200 + + +def iter_archive(path): + with fsspec.open(path, 'rb') as stream: + yield from ArchiveIterator(stream) + + +def read_records(path): + for record in iter_archive(path): + headers = record.rec_headers + yield { + 'warc_type': record.rec_type, + 'target_uri': headers.get_header('WARC-Target-URI'), + 'date': headers.get_header('WARC-Date'), + 'content_type': headers.get_header('Content-Type'), + 'content_length': int(headers.get_header('Content-Length')), + 'http_status': int(record.http_headers.get_statuscode()) if record.rec_type == 'response' else None, + } + + +def read_text(path): + for record in iter_archive(path): + if record.rec_type != 'conversion': + continue + words = record.content_stream().read().decode('utf-8', errors='replace').split() + yield { + 'url': record.rec_headers.get_header('WARC-Target-URI'), + 'language': record.rec_headers.get_header('WARC-Identified-Content-Language'), + 'word_count': len(words), + 'preview': ' '.join(words)[:PREVIEW_CHARS], + } + + +class LinkParser(HTMLParser): + def __init__(self): + super().__init__() + self.links, self.href, self.text = [], None, [] + + def handle_starttag(self, tag, attrs): + if tag == 'a': + self.close_link() + self.href = dict(attrs).get('href') + + def handle_data(self, data): + if self.href is not None: + self.text.append(data) + + def handle_endtag(self, tag): + if tag == 'a': + self.close_link() + + def close_link(self): + if self.href: + self.links.append((self.href, ' '.join(''.join(self.text).split()))) + self.href, self.text = None, [] + + +def read_links(path): + for record in iter_archive(path): + if record.rec_type != 'response' or 'html' not in (record.http_headers.get_header('Content-Type') or ''): + continue + page_url = record.rec_headers.get_header('WARC-Target-URI') + parser = LinkParser() + parser.feed(record.content_stream().read().decode('utf-8', errors='replace')) + parser.close() + for href, text in parser.links: + yield {'page_url': page_url, 'link_url': urljoin(page_url, href), 'link_text': text} + + +def write_csv(records, fields, out): + writer = csv.DictWriter(out, fieldnames=fields) + writer.writeheader() + writer.writerows(records) + + +def write_json(records, fields, out): + json.dump(list(records), out, indent=2, ensure_ascii=False) + out.write('\n') + + +MODES = {'records': (read_records, FIELDS), 'text': (read_text, TEXT_FIELDS), 'links': (read_links, LINK_FIELDS)} +WRITERS = {'csv': write_csv, 'json': write_json} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('input', help='local path or URL (https://, s3://) of a WARC, WET, or WAT file') + parser.add_argument('-m', '--mode', choices=MODES, default='records', help='records: metadata of every record; text: page text from WET; links: page links from WARC') + parser.add_argument('-f', '--format', choices=WRITERS, default='csv') + parser.add_argument('-o', '--output', help='output file (default: stdout)') + args = parser.parse_args() + + reader, fields = MODES[args.mode] + out = open(args.output, 'w', newline='', encoding='utf-8') if args.output else contextlib.nullcontext(sys.stdout) + with out as stream: + WRITERS[args.format](reader(args.input), fields, stream) + + +if __name__ == '__main__': + main() diff --git a/tests/test_exporter.py b/tests/test_exporter.py new file mode 100644 index 0000000..d337e65 --- /dev/null +++ b/tests/test_exporter.py @@ -0,0 +1,91 @@ +'''Tests for exporter/export_records.py against the sample whirlwind files.''' + +import csv +import io +import json +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'exporter')) + +from export_records import FIELDS, LINK_FIELDS, TEXT_FIELDS, LinkParser, read_links, read_records, read_text, write_csv, write_json # noqa: E402 + + +def records(name): + return list(read_records(str(ROOT / name))) + + +class TestReadRecords(unittest.TestCase): + def test_record_types(self): + cases = { + 'whirlwind.warc.gz': ['warcinfo', 'request', 'response', 'metadata'], + 'whirlwind.warc.wet.gz': ['warcinfo', 'conversion'], + 'whirlwind.warc.wat.gz': ['warcinfo', 'metadata'], + } + for name, types in cases.items(): + with self.subTest(name=name): + self.assertEqual([r['warc_type'] for r in records(name)], types) + + def test_response_fields(self): + response = records('whirlwind.warc.gz')[2] + self.assertEqual(response['target_uri'], 'https://an.wikipedia.org/wiki/Escopete') + self.assertEqual(response['http_status'], 200) + self.assertGreater(response['content_length'], 0) + + def test_status_only_on_response(self): + for record in records('whirlwind.warc.gz'): + if record['warc_type'] != 'response': + self.assertIsNone(record['http_status']) + + +class TestReadText(unittest.TestCase): + def test_wet(self): + rows = list(read_text(str(ROOT / 'whirlwind.warc.wet.gz'))) + self.assertEqual(len(rows), 1) + self.assertEqual(tuple(rows[0]), TEXT_FIELDS) + self.assertEqual(rows[0]['url'], 'https://an.wikipedia.org/wiki/Escopete') + self.assertEqual(rows[0]['language'], 'spa') + self.assertGreater(rows[0]['word_count'], 0) + self.assertTrue(rows[0]['preview'].startswith('Escopete')) + self.assertLessEqual(len(rows[0]['preview']), 200) + + def test_skips_non_conversion(self): + self.assertEqual(list(read_text(str(ROOT / 'whirlwind.warc.gz'))), []) + + +class TestReadLinks(unittest.TestCase): + def test_warc(self): + rows = list(read_links(str(ROOT / 'whirlwind.warc.gz'))) + self.assertEqual(len(rows), 207) + self.assertEqual(tuple(rows[0]), LINK_FIELDS) + self.assertTrue(all(r['page_url'] == 'https://an.wikipedia.org/wiki/Escopete' for r in rows)) + self.assertTrue(all(r['link_url'].startswith(('http://', 'https://')) for r in rows)) + + def test_skips_non_response(self): + self.assertEqual(list(read_links(str(ROOT / 'whirlwind.warc.wet.gz'))), []) + + def test_parser(self): + parser = LinkParser() + parser.feed('One twono href\n three ') + parser.close() + self.assertEqual(parser.links, [('/a', 'One two'), ('//b.org/', 'three')]) + + +class TestWriters(unittest.TestCase): + def test_csv(self): + out = io.StringIO() + write_csv(records('whirlwind.warc.gz'), FIELDS, out) + rows = list(csv.DictReader(io.StringIO(out.getvalue()))) + self.assertEqual(len(rows), 4) + self.assertEqual(tuple(rows[0]), FIELDS) + + def test_json(self): + out = io.StringIO() + write_json(records('whirlwind.warc.gz'), FIELDS, out) + self.assertEqual(json.loads(out.getvalue()), records('whirlwind.warc.gz')) + + +if __name__ == '__main__': + unittest.main()