diff --git a/.gitignore b/.gitignore
index 1075339..8521baf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,5 @@ collinfo.json
.venv/
.vscode/
.idea/
+exporter/output/
+progress.md
diff --git a/CHANGES.md b/CHANGES.md
new file mode 100644
index 0000000..9fa139d
--- /dev/null
+++ b/CHANGES.md
@@ -0,0 +1,46 @@
+# Changes
+
+## Record metadata export
+
+Added `exporter/export_records.py`, which exports record metadata to CSV or JSON.
+
+- Reads WARC, WET, or WAT files, local or remote, using `fsspec` and `warcio`, the same way `warcio-iterator.py` does.
+- Extracts one row per record: type, URI, date, content type, length, and HTTP status.
+- Reads records one at a time with a generator and keeps only their headers, so page content is never loaded into memory.
+- Writes with `csv.DictWriter` or `json.dump`, chosen by `-f`.
+- Uses only the standard library and packages already in `requirements.txt`.
+
+Other changes:
+
+- `Makefile`: added `make export`, which writes the outputs to `exporter/output/`.
+- `.gitignore`: ignores `exporter/output/`.
+- `exporter/README.md`: usage and field reference.
+
+## Page text export
+
+Added a `text` mode to `exporter/export_records.py` (`-m text`), which exports one row per WET `conversion` record: URL, language, word count, and a 200-character preview.
+
+- `iter_archive()` now holds the shared open-and-iterate code, used by both readers.
+- `-m` picks a reader and its CSV fields, so the writers stay format-only.
+- JSON output keeps non-ASCII characters as-is (`ensure_ascii=False`).
+
+Other changes:
+
+- `Makefile`: `make export` also writes `exporter/output/text.csv`.
+- `exporter/README.md`: documents `-m` and the `text` fields.
+- `tests/test_exporter.py`: tests for the `text` mode.
+
+## Page links export
+
+Added a `links` mode to `exporter/export_records.py` (`-m links`), which exports one row per link found in the HTML of each WARC `response` record: page URL, link URL, and link text.
+
+- Links come from the WARC HTML because the sample WAT file has no link data (its only metadata record describes the crawl request).
+- `LinkParser` uses the standard library's `html.parser`, so no new packages are needed.
+- Relative and protocol-relative links (`/wiki/...`, `//donate...`) are resolved to absolute URLs with `urljoin`.
+- Non-HTML responses are skipped.
+
+Other changes:
+
+- `Makefile`: `make export` also writes `exporter/output/links.csv`.
+- `exporter/README.md`: documents the `links` fields.
+- `tests/test_exporter.py`: tests for the `links` mode and `LinkParser`.
diff --git a/Makefile b/Makefile
index c3a33a9..a5cc3ba 100644
--- a/Makefile
+++ b/Makefile
@@ -79,6 +79,16 @@ extract-remote-s3:
@echo "extracting before-you-ship.18f.gov record from End-of-Term Common Crawl repackage WARC over S3 (offset 18595):"
warcio extract $(EOT_CC_WARC_S3) 18595
+export:
+ @echo "exporting record metadata, page text, and links from the local warcs to exporter/output/"
+ mkdir -p exporter/output
+ python exporter/export_records.py whirlwind.warc.gz -o exporter/output/warc.csv
+ python exporter/export_records.py whirlwind.warc.wet.gz -o exporter/output/wet.csv
+ python exporter/export_records.py whirlwind.warc.wat.gz -o exporter/output/wat.csv
+ python exporter/export_records.py whirlwind.warc.gz -f json -o exporter/output/warc.json
+ python exporter/export_records.py whirlwind.warc.wet.gz -m text -o exporter/output/text.csv
+ python exporter/export_records.py whirlwind.warc.gz -m links -o exporter/output/links.csv
+
cdx_toolkit:
@echo demonstrate that we have this entry in the index
cdxt --crawl CC-MAIN-2024-22 --from 20240518015810 --to 20240518015810 iter an.wikipedia.org/wiki/Escopete
diff --git a/exporter/README.md b/exporter/README.md
new file mode 100644
index 0000000..5cd11ed
--- /dev/null
+++ b/exporter/README.md
@@ -0,0 +1,58 @@
+# Exporter
+
+Exports record metadata from a WARC, WET, or WAT file, page text from a WET file, or page links from a WARC file, to CSV or JSON.
+
+## Usage
+
+```
+python exporter/export_records.py [-m records|text|links] [-f csv|json] [-o output]
+```
+
+- `input`: local path or remote URL (`https://`, `s3://`)
+- `-m`: mode, `records` (default), `text`, or `links`
+- `-f`: output format, `csv` (default) or `json`
+- `-o`: output file, stdout if omitted
+
+Example:
+
+```
+python exporter/export_records.py whirlwind.warc.gz -f json -o records.json
+python exporter/export_records.py whirlwind.warc.wet.gz -m text
+python exporter/export_records.py whirlwind.warc.gz -m links
+```
+
+Or export all local files into `exporter/output/`:
+
+```
+make export
+```
+
+## Fields
+
+`records` mode, one row per record:
+
+| Field | Source |
+|---|---|
+| `warc_type` | `WARC-Type` |
+| `target_uri` | `WARC-Target-URI` |
+| `date` | `WARC-Date` |
+| `content_type` | `Content-Type` |
+| `content_length` | `Content-Length` |
+| `http_status` | HTTP status code (response records only) |
+
+`text` mode, one row per WET `conversion` record:
+
+| Field | Source |
+|---|---|
+| `url` | `WARC-Target-URI` |
+| `language` | `WARC-Identified-Content-Language` |
+| `word_count` | number of whitespace-separated words in the text |
+| `preview` | first 200 characters of the text, whitespace collapsed |
+
+`links` mode, one row per `` in each HTML WARC `response` record:
+
+| Field | Source |
+|---|---|
+| `page_url` | `WARC-Target-URI` |
+| `link_url` | `href`, resolved against `page_url` |
+| `link_text` | text inside the `` tag, whitespace collapsed |
diff --git a/exporter/export_records.py b/exporter/export_records.py
new file mode 100644
index 0000000..9e193e2
--- /dev/null
+++ b/exporter/export_records.py
@@ -0,0 +1,117 @@
+'''Export WARC, WET, or WAT record metadata, WET page text, or WARC page links to CSV or JSON.'''
+
+import argparse
+import contextlib
+import csv
+import json
+import sys
+from html.parser import HTMLParser
+from urllib.parse import urljoin
+
+import fsspec
+from warcio.archiveiterator import ArchiveIterator
+
+FIELDS = ('warc_type', 'target_uri', 'date', 'content_type', 'content_length', 'http_status')
+TEXT_FIELDS = ('url', 'language', 'word_count', 'preview')
+LINK_FIELDS = ('page_url', 'link_url', 'link_text')
+PREVIEW_CHARS = 200
+
+
+def iter_archive(path):
+ with fsspec.open(path, 'rb') as stream:
+ yield from ArchiveIterator(stream)
+
+
+def read_records(path):
+ for record in iter_archive(path):
+ headers = record.rec_headers
+ yield {
+ 'warc_type': record.rec_type,
+ 'target_uri': headers.get_header('WARC-Target-URI'),
+ 'date': headers.get_header('WARC-Date'),
+ 'content_type': headers.get_header('Content-Type'),
+ 'content_length': int(headers.get_header('Content-Length')),
+ 'http_status': int(record.http_headers.get_statuscode()) if record.rec_type == 'response' else None,
+ }
+
+
+def read_text(path):
+ for record in iter_archive(path):
+ if record.rec_type != 'conversion':
+ continue
+ words = record.content_stream().read().decode('utf-8', errors='replace').split()
+ yield {
+ 'url': record.rec_headers.get_header('WARC-Target-URI'),
+ 'language': record.rec_headers.get_header('WARC-Identified-Content-Language'),
+ 'word_count': len(words),
+ 'preview': ' '.join(words)[:PREVIEW_CHARS],
+ }
+
+
+class LinkParser(HTMLParser):
+ def __init__(self):
+ super().__init__()
+ self.links, self.href, self.text = [], None, []
+
+ def handle_starttag(self, tag, attrs):
+ if tag == 'a':
+ self.close_link()
+ self.href = dict(attrs).get('href')
+
+ def handle_data(self, data):
+ if self.href is not None:
+ self.text.append(data)
+
+ def handle_endtag(self, tag):
+ if tag == 'a':
+ self.close_link()
+
+ def close_link(self):
+ if self.href:
+ self.links.append((self.href, ' '.join(''.join(self.text).split())))
+ self.href, self.text = None, []
+
+
+def read_links(path):
+ for record in iter_archive(path):
+ if record.rec_type != 'response' or 'html' not in (record.http_headers.get_header('Content-Type') or ''):
+ continue
+ page_url = record.rec_headers.get_header('WARC-Target-URI')
+ parser = LinkParser()
+ parser.feed(record.content_stream().read().decode('utf-8', errors='replace'))
+ parser.close()
+ for href, text in parser.links:
+ yield {'page_url': page_url, 'link_url': urljoin(page_url, href), 'link_text': text}
+
+
+def write_csv(records, fields, out):
+ writer = csv.DictWriter(out, fieldnames=fields)
+ writer.writeheader()
+ writer.writerows(records)
+
+
+def write_json(records, fields, out):
+ json.dump(list(records), out, indent=2, ensure_ascii=False)
+ out.write('\n')
+
+
+MODES = {'records': (read_records, FIELDS), 'text': (read_text, TEXT_FIELDS), 'links': (read_links, LINK_FIELDS)}
+WRITERS = {'csv': write_csv, 'json': write_json}
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('input', help='local path or URL (https://, s3://) of a WARC, WET, or WAT file')
+ parser.add_argument('-m', '--mode', choices=MODES, default='records', help='records: metadata of every record; text: page text from WET; links: page links from WARC')
+ parser.add_argument('-f', '--format', choices=WRITERS, default='csv')
+ parser.add_argument('-o', '--output', help='output file (default: stdout)')
+ args = parser.parse_args()
+
+ reader, fields = MODES[args.mode]
+ out = open(args.output, 'w', newline='', encoding='utf-8') if args.output else contextlib.nullcontext(sys.stdout)
+ with out as stream:
+ WRITERS[args.format](reader(args.input), fields, stream)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/test_exporter.py b/tests/test_exporter.py
new file mode 100644
index 0000000..d337e65
--- /dev/null
+++ b/tests/test_exporter.py
@@ -0,0 +1,91 @@
+'''Tests for exporter/export_records.py against the sample whirlwind files.'''
+
+import csv
+import io
+import json
+import sys
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT / 'exporter'))
+
+from export_records import FIELDS, LINK_FIELDS, TEXT_FIELDS, LinkParser, read_links, read_records, read_text, write_csv, write_json # noqa: E402
+
+
+def records(name):
+ return list(read_records(str(ROOT / name)))
+
+
+class TestReadRecords(unittest.TestCase):
+ def test_record_types(self):
+ cases = {
+ 'whirlwind.warc.gz': ['warcinfo', 'request', 'response', 'metadata'],
+ 'whirlwind.warc.wet.gz': ['warcinfo', 'conversion'],
+ 'whirlwind.warc.wat.gz': ['warcinfo', 'metadata'],
+ }
+ for name, types in cases.items():
+ with self.subTest(name=name):
+ self.assertEqual([r['warc_type'] for r in records(name)], types)
+
+ def test_response_fields(self):
+ response = records('whirlwind.warc.gz')[2]
+ self.assertEqual(response['target_uri'], 'https://an.wikipedia.org/wiki/Escopete')
+ self.assertEqual(response['http_status'], 200)
+ self.assertGreater(response['content_length'], 0)
+
+ def test_status_only_on_response(self):
+ for record in records('whirlwind.warc.gz'):
+ if record['warc_type'] != 'response':
+ self.assertIsNone(record['http_status'])
+
+
+class TestReadText(unittest.TestCase):
+ def test_wet(self):
+ rows = list(read_text(str(ROOT / 'whirlwind.warc.wet.gz')))
+ self.assertEqual(len(rows), 1)
+ self.assertEqual(tuple(rows[0]), TEXT_FIELDS)
+ self.assertEqual(rows[0]['url'], 'https://an.wikipedia.org/wiki/Escopete')
+ self.assertEqual(rows[0]['language'], 'spa')
+ self.assertGreater(rows[0]['word_count'], 0)
+ self.assertTrue(rows[0]['preview'].startswith('Escopete'))
+ self.assertLessEqual(len(rows[0]['preview']), 200)
+
+ def test_skips_non_conversion(self):
+ self.assertEqual(list(read_text(str(ROOT / 'whirlwind.warc.gz'))), [])
+
+
+class TestReadLinks(unittest.TestCase):
+ def test_warc(self):
+ rows = list(read_links(str(ROOT / 'whirlwind.warc.gz')))
+ self.assertEqual(len(rows), 207)
+ self.assertEqual(tuple(rows[0]), LINK_FIELDS)
+ self.assertTrue(all(r['page_url'] == 'https://an.wikipedia.org/wiki/Escopete' for r in rows))
+ self.assertTrue(all(r['link_url'].startswith(('http://', 'https://')) for r in rows))
+
+ def test_skips_non_response(self):
+ self.assertEqual(list(read_links(str(ROOT / 'whirlwind.warc.wet.gz'))), [])
+
+ def test_parser(self):
+ parser = LinkParser()
+ parser.feed('One twono href\n three ')
+ parser.close()
+ self.assertEqual(parser.links, [('/a', 'One two'), ('//b.org/', 'three')])
+
+
+class TestWriters(unittest.TestCase):
+ def test_csv(self):
+ out = io.StringIO()
+ write_csv(records('whirlwind.warc.gz'), FIELDS, out)
+ rows = list(csv.DictReader(io.StringIO(out.getvalue())))
+ self.assertEqual(len(rows), 4)
+ self.assertEqual(tuple(rows[0]), FIELDS)
+
+ def test_json(self):
+ out = io.StringIO()
+ write_json(records('whirlwind.warc.gz'), FIELDS, out)
+ self.assertEqual(json.loads(out.getvalue()), records('whirlwind.warc.gz'))
+
+
+if __name__ == '__main__':
+ unittest.main()