From 030c78e038faa02d17bfcb2fb1defc16629bd2a4 Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Sat, 4 Jul 2026 15:37:42 +0530 Subject: [PATCH 1/9] fix(dataset): keep CSV columns aligned for items with different keys `export_csv_to_stream` built the header from the first item's keys but then wrote every row positionally with `csv.writer` and `item.values()`. Since `Dataset.push_data` accepts arbitrary mappings per item with no schema, items whose keys differ in membership or order from the first item had their values written under the wrong columns (or shifted into columns that do not exist), silently corrupting the CSV export of any non-uniform dataset. Write rows with `csv.DictWriter` keyed on the first item's fields and `extrasaction='ignore'`, so each value lands in its own column and unexpected keys are dropped. This matches the column-aligned behavior of Crawlee for JavaScript. Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- src/crawlee/_utils/file.py | 16 +++++++----- tests/unit/_utils/test_file.py | 48 +++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/src/crawlee/_utils/file.py b/src/crawlee/_utils/file.py index 499d363046..47851f1939 100644 --- a/src/crawlee/_utils/file.py +++ b/src/crawlee/_utils/file.py @@ -203,16 +203,18 @@ async def export_csv_to_stream( if 'lineterminator' not in kwargs: kwargs['lineterminator'] = '\n' - writer = csv.writer(dst, **kwargs) - write_header = True + writer: csv.DictWriter | None = None - # Iterate over the dataset and write to CSV. + # Iterate over the dataset and write to CSV. The header (field names) is taken from the first non-empty + # item, and each row is written by key via `csv.DictWriter` so values stay aligned with their columns even + # when later items have a different key set or order. Keys not present in the header are ignored, matching + # the behavior of Crawlee for JavaScript. async for item in iterator: if not item: continue - if write_header: - writer.writerow(item.keys()) - write_header = False + if writer is None: + writer = csv.DictWriter(dst, fieldnames=list(item.keys()), extrasaction='ignore', **kwargs) + writer.writeheader() - writer.writerow(item.values()) + writer.writerow(item) diff --git a/tests/unit/_utils/test_file.py b/tests/unit/_utils/test_file.py index f4989337e9..e876a861cf 100644 --- a/tests/unit/_utils/test_file.py +++ b/tests/unit/_utils/test_file.py @@ -1,15 +1,19 @@ from __future__ import annotations from datetime import datetime, timezone +from io import StringIO from typing import TYPE_CHECKING import pytest -from crawlee._utils.file import json_dumps, validate_subdirectory +from crawlee._utils.file import export_csv_to_stream, json_dumps, validate_subdirectory if TYPE_CHECKING: + from collections.abc import AsyncIterator, Mapping from pathlib import Path + from crawlee._types import JsonSerializable + async def test_json_dumps() -> None: assert await json_dumps({'key': 'value'}) == '{\n "key": "value"\n}' @@ -19,6 +23,48 @@ async def test_json_dumps() -> None: assert await json_dumps(datetime(2022, 1, 1, tzinfo=timezone.utc)) == '"2022-01-01 00:00:00+00:00"' +# Tests for export_csv_to_stream (dataset CSV export). + + +async def _async_iter( + items: list[Mapping[str, JsonSerializable]], +) -> AsyncIterator[Mapping[str, JsonSerializable]]: + for item in items: + yield item + + +async def test_export_csv_to_stream_keeps_columns_aligned_for_heterogeneous_items() -> None: + """Values must be written under their own header column even when items have different key orders/sets.""" + dst = StringIO() + await export_csv_to_stream( + _async_iter( + [ + {'name': 'Alice', 'age': 30}, + {'name': 'Bob', 'city': 'NYC', 'age': 25}, + {'age': 40, 'name': 'Carol'}, + ] + ), + dst, + lineterminator='\n', + ) + + # The header comes from the first item; later items are mapped by key (extra keys such as `city` are dropped), + # so `Carol`/`40` stay aligned with the `name`/`age` columns instead of being written positionally. + assert dst.getvalue() == 'name,age\nAlice,30\nBob,25\nCarol,40\n' + + +async def test_export_csv_to_stream_skips_empty_items() -> None: + """Empty mappings are skipped and do not define or shift the header.""" + dst = StringIO() + await export_csv_to_stream( + _async_iter([{}, {'id': 1, 'name': 'Item 1'}, {}, {'id': 2, 'name': 'Item 2'}]), + dst, + lineterminator='\n', + ) + + assert dst.getvalue() == 'id,name\n1,Item 1\n2,Item 2\n' + + # Tests for validate_subdirectory (storage name/alias directory validation). From 6e60ba7a3531a5c292c591c28efcd6306e91fd26 Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:38:05 +0530 Subject: [PATCH 2/9] fix(dataset): include all CSV export columns Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- src/crawlee/_utils/file.py | 22 ++++++++-------------- tests/unit/_utils/test_file.py | 4 +--- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/crawlee/_utils/file.py b/src/crawlee/_utils/file.py index 47851f1939..7286c53c5a 100644 --- a/src/crawlee/_utils/file.py +++ b/src/crawlee/_utils/file.py @@ -203,18 +203,12 @@ async def export_csv_to_stream( if 'lineterminator' not in kwargs: kwargs['lineterminator'] = '\n' - writer: csv.DictWriter | None = None - - # Iterate over the dataset and write to CSV. The header (field names) is taken from the first non-empty - # item, and each row is written by key via `csv.DictWriter` so values stay aligned with their columns even - # when later items have a different key set or order. Keys not present in the header are ignored, matching - # the behavior of Crawlee for JavaScript. - async for item in iterator: - if not item: - continue - - if writer is None: - writer = csv.DictWriter(dst, fieldnames=list(item.keys()), extrasaction='ignore', **kwargs) - writer.writeheader() - + items = [item async for item in iterator if item] + if not items: + return + + fieldnames = list(dict.fromkeys(key for item in items for key in item)) + writer = csv.DictWriter(dst, fieldnames=fieldnames, **kwargs) + writer.writeheader() + for item in items: writer.writerow(item) diff --git a/tests/unit/_utils/test_file.py b/tests/unit/_utils/test_file.py index e876a861cf..c6dce0527a 100644 --- a/tests/unit/_utils/test_file.py +++ b/tests/unit/_utils/test_file.py @@ -48,9 +48,7 @@ async def test_export_csv_to_stream_keeps_columns_aligned_for_heterogeneous_item lineterminator='\n', ) - # The header comes from the first item; later items are mapped by key (extra keys such as `city` are dropped), - # so `Carol`/`40` stay aligned with the `name`/`age` columns instead of being written positionally. - assert dst.getvalue() == 'name,age\nAlice,30\nBob,25\nCarol,40\n' + assert dst.getvalue() == 'name,age,city\nAlice,30,\nBob,25,NYC\nCarol,40,\n' async def test_export_csv_to_stream_skips_empty_items() -> None: From bde4ba2108040309ed594ecd1fca8bcc522ea1c2 Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:23:52 +0530 Subject: [PATCH 3/9] fix(dataset): bound CSV export memory usage Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- src/crawlee/_utils/file.py | 27 ++++++++++++++++++--------- tests/unit/_utils/test_file.py | 7 +++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/crawlee/_utils/file.py b/src/crawlee/_utils/file.py index 7286c53c5a..0ec7f2cf9b 100644 --- a/src/crawlee/_utils/file.py +++ b/src/crawlee/_utils/file.py @@ -203,12 +203,21 @@ async def export_csv_to_stream( if 'lineterminator' not in kwargs: kwargs['lineterminator'] = '\n' - items = [item async for item in iterator if item] - if not items: - return - - fieldnames = list(dict.fromkeys(key for item in items for key in item)) - writer = csv.DictWriter(dst, fieldnames=fieldnames, **kwargs) - writer.writeheader() - for item in items: - writer.writerow(item) + fieldnames = dict[str, None]() + with tempfile.TemporaryFile(mode='w+', encoding='utf-8') as items: + async for item in iterator: + if not item: + continue + + fieldnames.update(dict.fromkeys(item)) + json.dump(item, items) + items.write('\n') + + if not fieldnames: + return + + writer = csv.DictWriter(dst, fieldnames=fieldnames, **kwargs) + writer.writeheader() + items.seek(0) + for item in items: + writer.writerow(json.loads(item)) diff --git a/tests/unit/_utils/test_file.py b/tests/unit/_utils/test_file.py index c6dce0527a..625739c709 100644 --- a/tests/unit/_utils/test_file.py +++ b/tests/unit/_utils/test_file.py @@ -63,6 +63,13 @@ async def test_export_csv_to_stream_skips_empty_items() -> None: assert dst.getvalue() == 'id,name\n1,Item 1\n2,Item 2\n' +async def test_export_csv_to_stream_handles_empty_iterator() -> None: + dst = StringIO() + await export_csv_to_stream(_async_iter([]), dst) + + assert dst.getvalue() == '' + + # Tests for validate_subdirectory (storage name/alias directory validation). From 35ee5241914578149a543a8b42944bfff82c399e Mon Sep 17 00:00:00 2001 From: Anas Khan <83116240+anxkhn@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:21:17 +0530 Subject: [PATCH 4/9] fix(dataset): stream CSV exports by default Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com> --- src/crawlee/_types.py | 11 +++++-- src/crawlee/_utils/file.py | 36 +++++++++++++---------- tests/unit/_utils/test_file.py | 54 ++++++++++++++++++++++++++++++---- 3 files changed, 77 insertions(+), 24 deletions(-) diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index ffeb92094c..855211a05a 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -772,8 +772,8 @@ class ExportDataJsonKwargs(TypedDict): """Specifies whether the output JSON object should have keys sorted alphabetically.""" -class ExportDataCsvKwargs(TypedDict): - """Keyword arguments for dataset's `export_data_csv` method.""" +class ExportDataCsvWriterKwargs(TypedDict): + """Keyword arguments forwarded to `csv.DictWriter`.""" dialect: NotRequired[str] """Specifies a dialect to be used in CSV parsing and writing.""" @@ -807,6 +807,13 @@ class ExportDataCsvKwargs(TypedDict): """When True, raises an exception on bad CSV input. Defaults to False.""" +class ExportDataCsvKwargs(ExportDataCsvWriterKwargs): + """Keyword arguments for dataset's CSV export.""" + + collect_all_keys: NotRequired[bool] + """When True, includes keys from all items as CSV columns. Defaults to False.""" + + class ExportDataKwargs(ExportDataJsonKwargs, ExportDataCsvKwargs): """Keyword arguments accepted by `BasicCrawler.export_data`. diff --git a/src/crawlee/_utils/file.py b/src/crawlee/_utils/file.py index 0ec7f2cf9b..f0ff6936ed 100644 --- a/src/crawlee/_utils/file.py +++ b/src/crawlee/_utils/file.py @@ -7,7 +7,7 @@ import sys import tempfile from pathlib import Path -from typing import TYPE_CHECKING, overload +from typing import TYPE_CHECKING, cast, overload if TYPE_CHECKING: from collections.abc import AsyncIterator, Mapping @@ -15,7 +15,7 @@ from typing_extensions import Unpack - from crawlee._types import ExportDataCsvKwargs, ExportDataJsonKwargs, JsonSerializable + from crawlee._types import ExportDataCsvKwargs, ExportDataCsvWriterKwargs, ExportDataJsonKwargs, JsonSerializable if sys.platform == 'win32': @@ -196,6 +196,8 @@ async def export_csv_to_stream( dst: TextIO, **kwargs: Unpack[ExportDataCsvKwargs], ) -> None: + collect_all_keys = kwargs.pop('collect_all_keys', False) + writer_kwargs = cast('ExportDataCsvWriterKwargs', kwargs) # Set lineterminator to '\n' if not explicitly provided. This prevents double line endings on Windows. # The csv.writer default is '\r\n', which when written to a file in text mode on Windows gets converted # to '\r\r\n' due to newline translation. By using '\n', we let the platform handle the line ending @@ -203,21 +205,23 @@ async def export_csv_to_stream( if 'lineterminator' not in kwargs: kwargs['lineterminator'] = '\n' - fieldnames = dict[str, None]() - with tempfile.TemporaryFile(mode='w+', encoding='utf-8') as items: - async for item in iterator: - if not item: - continue - - fieldnames.update(dict.fromkeys(item)) - json.dump(item, items) - items.write('\n') - - if not fieldnames: + if collect_all_keys: + items = [item async for item in iterator if item] + if not items: return - writer = csv.DictWriter(dst, fieldnames=fieldnames, **kwargs) + fieldnames = list(dict.fromkeys(key for item in items for key in item)) + writer = csv.DictWriter(dst, fieldnames=fieldnames, **writer_kwargs) writer.writeheader() - items.seek(0) for item in items: - writer.writerow(json.loads(item)) + writer.writerow(item) + return + + writer = None + async for item in iterator: + if not item: + continue + if writer is None: + writer = csv.DictWriter(dst, fieldnames=list(item), extrasaction='ignore', **writer_kwargs) + writer.writeheader() + writer.writerow(item) diff --git a/tests/unit/_utils/test_file.py b/tests/unit/_utils/test_file.py index 625739c709..6361abea20 100644 --- a/tests/unit/_utils/test_file.py +++ b/tests/unit/_utils/test_file.py @@ -2,7 +2,7 @@ from datetime import datetime, timezone from io import StringIO -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast import pytest @@ -26,7 +26,7 @@ async def test_json_dumps() -> None: # Tests for export_csv_to_stream (dataset CSV export). -async def _async_iter( +async def async_iter( items: list[Mapping[str, JsonSerializable]], ) -> AsyncIterator[Mapping[str, JsonSerializable]]: for item in items: @@ -37,7 +37,7 @@ async def test_export_csv_to_stream_keeps_columns_aligned_for_heterogeneous_item """Values must be written under their own header column even when items have different key orders/sets.""" dst = StringIO() await export_csv_to_stream( - _async_iter( + async_iter( [ {'name': 'Alice', 'age': 30}, {'name': 'Bob', 'city': 'NYC', 'age': 25}, @@ -48,14 +48,27 @@ async def test_export_csv_to_stream_keeps_columns_aligned_for_heterogeneous_item lineterminator='\n', ) - assert dst.getvalue() == 'name,age,city\nAlice,30,\nBob,25,NYC\nCarol,40,\n' + assert dst.getvalue() == 'name,age\nAlice,30\nBob,25\nCarol,40\n' + + +async def test_export_csv_to_stream_collects_all_keys_when_requested() -> None: + """All item keys are included when key collection is enabled.""" + dst = StringIO() + await export_csv_to_stream( + async_iter([{'name': 'Alice', 'age': 30}, {'name': 'Bob', 'city': 'NYC', 'age': 25}]), + dst, + collect_all_keys=True, + lineterminator='\n', + ) + + assert dst.getvalue() == 'name,age,city\nAlice,30,\nBob,25,NYC\n' async def test_export_csv_to_stream_skips_empty_items() -> None: """Empty mappings are skipped and do not define or shift the header.""" dst = StringIO() await export_csv_to_stream( - _async_iter([{}, {'id': 1, 'name': 'Item 1'}, {}, {'id': 2, 'name': 'Item 2'}]), + async_iter([{}, {'id': 1, 'name': 'Item 1'}, {}, {'id': 2, 'name': 'Item 2'}]), dst, lineterminator='\n', ) @@ -64,12 +77,41 @@ async def test_export_csv_to_stream_skips_empty_items() -> None: async def test_export_csv_to_stream_handles_empty_iterator() -> None: + """An empty iterator produces no CSV content.""" dst = StringIO() - await export_csv_to_stream(_async_iter([]), dst) + await export_csv_to_stream(async_iter([]), dst) assert dst.getvalue() == '' +async def test_export_csv_to_stream_writes_before_consuming_all_items() -> None: + """The default export writes its header before requesting the second item.""" + dst = StringIO() + + async def items() -> AsyncIterator[Mapping[str, JsonSerializable]]: + yield {'id': 1} + assert dst.getvalue() == 'id\n1\n' + yield {'id': 2} + + await export_csv_to_stream(items(), dst, lineterminator='\n') + + assert dst.getvalue() == 'id\n1\n2\n' + + +async def test_export_csv_to_stream_preserves_non_json_values() -> None: + """CSV values do not pass through JSON serialization.""" + dst = StringIO() + value = datetime(2020, 1, 1, tzinfo=timezone.utc) + + await export_csv_to_stream( + async_iter([{'created_at': cast('JsonSerializable', value)}]), + dst, + lineterminator='\n', + ) + + assert dst.getvalue() == 'created_at\n2020-01-01 00:00:00+00:00\n' + + # Tests for validate_subdirectory (storage name/alias directory validation). From 15e8783c0126d96579c0db9d886bb3c53013759a Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 28 Jul 2026 10:01:43 +0200 Subject: [PATCH 5/9] fix(dataset): signal dropped CSV columns and validate writer options eagerly Address review findings on the CSV export rewrite: - Warn once per export, naming every key dropped because it is absent from the first item, instead of losing those values silently. Master wrote them to the wrong column, which was wrong but at least visible. - Build a throwaway writer up front so invalid options (e.g. `delimiter='ab'`) raise even when the dataset is empty. Master validated eagerly; the lazy writer made a misconfigured export succeed on a run that scraped nothing and fail on the next one. - Pass `extrasaction='ignore'` on the buffered path too, so both paths construct the writer identically and narrowing `fieldnames` later cannot raise mid-export. - Declare `restval`, which already worked but was rejected by the type checker at every call site, and correct the `ExportDataCsvWriterKwargs` docstring that named `csv.DictWriter` while listing only `csv.writer` options. - Cover the empty-items guard, the `if item` filter on the buffered path, and `collect_all_keys` through `Dataset.export_to` and `BasicCrawler.export_data`, none of which had a test. - Document the column behavior and `collect_all_keys` in the dataset export example. --- .../export_entire_dataset_to_file.mdx | 14 ++- src/crawlee/_types.py | 13 ++- src/crawlee/_utils/file.py | 39 ++++++-- tests/unit/_utils/test_file.py | 94 ++++++++++++++++++- .../crawlers/_basic/test_basic_crawler.py | 16 ++++ tests/unit/storages/test_dataset.py | 40 ++++++++ 6 files changed, 202 insertions(+), 14 deletions(-) diff --git a/docs/examples/export_entire_dataset_to_file.mdx b/docs/examples/export_entire_dataset_to_file.mdx index 5cf4a2da77..a0a7ce4bcc 100644 --- a/docs/examples/export_entire_dataset_to_file.mdx +++ b/docs/examples/export_entire_dataset_to_file.mdx @@ -11,7 +11,7 @@ import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import JsonExample from '!!raw-loader!roa-loader!./code_examples/export_entire_dataset_to_file_json.py'; import CsvExample from '!!raw-loader!roa-loader!./code_examples/export_entire_dataset_to_file_csv.py'; -This example demonstrates how to use the `BasicCrawler.export_data` method of the crawler to export the entire default dataset to a single file. This method supports exporting data in either CSV or JSON format and also accepts additional keyword arguments so you can fine-tune the underlying `json.dump` or `csv.writer` behavior. +This example demonstrates how to use the `BasicCrawler.export_data` method of the crawler to export the entire default dataset to a single file. This method supports exporting data in either CSV or JSON format and also accepts additional keyword arguments so you can fine-tune the underlying `json.dump` or `csv.DictWriter` behavior. :::note @@ -31,3 +31,15 @@ For these examples, we are using the `B + +## CSV columns + +Dataset items don't have to share a schema. Different handlers can push different fields, so the items of one dataset often have different keys. + +By default, the CSV columns are the keys of the first item. When a later item has a key the first one doesn't, its value isn't written, and Crawlee logs a warning naming the dropped keys. To use the keys of all items as columns instead, pass `collect_all_keys=True`: + +```python +await crawler.export_data(path='results.csv', collect_all_keys=True) +``` + +No value is dropped in that mode. Cells for keys an item doesn't have stay empty, or hold the `restval` value if you pass one. Collecting all keys means reading the whole dataset before the first row can be written, so only the default mode writes rows as it goes. diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 855211a05a..6bd500e3a2 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -773,7 +773,11 @@ class ExportDataJsonKwargs(TypedDict): class ExportDataCsvWriterKwargs(TypedDict): - """Keyword arguments forwarded to `csv.DictWriter`.""" + """Keyword arguments forwarded to the underlying `csv` writer. + + The `fieldnames` and `extrasaction` arguments of `csv.DictWriter` are managed by Crawlee and cannot be + overridden. Use `collect_all_keys` to control which columns the export contains. + """ dialect: NotRequired[str] """Specifies a dialect to be used in CSV parsing and writing.""" @@ -800,6 +804,9 @@ class ExportDataCsvWriterKwargs(TypedDict): """Controls when quotes should be generated by the writer and recognized by the reader. Can take any of the `QUOTE_*` constants, with a default of `QUOTE_MINIMAL`.""" + restval: NotRequired[str] + """The value written for columns an item does not contain. Defaults to an empty string.""" + skipinitialspace: NotRequired[bool] """When True, spaces immediately following the delimiter are ignored. Defaults to False.""" @@ -811,7 +818,9 @@ class ExportDataCsvKwargs(ExportDataCsvWriterKwargs): """Keyword arguments for dataset's CSV export.""" collect_all_keys: NotRequired[bool] - """When True, includes keys from all items as CSV columns. Defaults to False.""" + """When True, the columns are the keys of all items combined, and no value is dropped. When False, the columns + are the keys of the first non-empty item, and keys introduced by later items are dropped with a warning. + Defaults to False.""" class ExportDataKwargs(ExportDataJsonKwargs, ExportDataCsvKwargs): diff --git a/src/crawlee/_utils/file.py b/src/crawlee/_utils/file.py index f0ff6936ed..bdf59e8895 100644 --- a/src/crawlee/_utils/file.py +++ b/src/crawlee/_utils/file.py @@ -6,6 +6,7 @@ import os import sys import tempfile +from logging import getLogger from pathlib import Path from typing import TYPE_CHECKING, cast, overload @@ -17,6 +18,8 @@ from crawlee._types import ExportDataCsvKwargs, ExportDataCsvWriterKwargs, ExportDataJsonKwargs, JsonSerializable +logger = getLogger(__name__) + if sys.platform == 'win32': def _write_file(path: Path, data: str | bytes) -> None: @@ -197,31 +200,55 @@ async def export_csv_to_stream( **kwargs: Unpack[ExportDataCsvKwargs], ) -> None: collect_all_keys = kwargs.pop('collect_all_keys', False) - writer_kwargs = cast('ExportDataCsvWriterKwargs', kwargs) - # Set lineterminator to '\n' if not explicitly provided. This prevents double line endings on Windows. - # The csv.writer default is '\r\n', which when written to a file in text mode on Windows gets converted - # to '\r\r\n' due to newline translation. By using '\n', we let the platform handle the line ending - # conversion: '\n' stays as '\n' on Unix, and becomes '\r\n' on Windows. + + # Set lineterminator to '\n' if not explicitly provided. This prevents double line endings on Windows. The + # `csv.DictWriter` default is '\r\n', which when written to a file in text mode on Windows gets converted to + # '\r\r\n' due to newline translation. By using '\n', we let the platform handle the line ending conversion: + # '\n' stays as '\n' on Unix, and becomes '\r\n' on Windows. if 'lineterminator' not in kwargs: kwargs['lineterminator'] = '\n' + writer_kwargs = cast('ExportDataCsvWriterKwargs', kwargs) + + # The real writer cannot be built until the first item's keys are known, so construct a throwaway one up front + # to validate the options. Without this, an export configured with e.g. `delimiter='ab'` would raise only when + # the dataset happens to contain an item, and silently succeed on a run that scraped nothing. + csv.DictWriter(dst, fieldnames=[], extrasaction='ignore', **writer_kwargs) + if collect_all_keys: items = [item async for item in iterator if item] if not items: return fieldnames = list(dict.fromkeys(key for item in items for key in item)) - writer = csv.DictWriter(dst, fieldnames=fieldnames, **writer_kwargs) + writer = csv.DictWriter(dst, fieldnames=fieldnames, extrasaction='ignore', **writer_kwargs) writer.writeheader() for item in items: writer.writerow(item) return writer = None + header_keys: set[str] = set() + dropped_keys: set[str] = set() + async for item in iterator: if not item: continue + if writer is None: writer = csv.DictWriter(dst, fieldnames=list(item), extrasaction='ignore', **writer_kwargs) writer.writeheader() + header_keys = set(writer.fieldnames) + + dropped_keys.update(item.keys() - header_keys) writer.writerow(item) + + # The header comes from the first item, so any key introduced by a later item is dropped. Dropping them is + # intentional and matches Crawlee for JS. Doing it silently is not, so report it once for the whole export. + if dropped_keys: + logger.warning( + 'CSV export dropped %d key(s) not present in the first item: %s. ' + 'Pass collect_all_keys=True to include keys from all items as columns.', + len(dropped_keys), + ', '.join(sorted(dropped_keys)), + ) diff --git a/tests/unit/_utils/test_file.py b/tests/unit/_utils/test_file.py index 6361abea20..243da48ca5 100644 --- a/tests/unit/_utils/test_file.py +++ b/tests/unit/_utils/test_file.py @@ -23,9 +23,6 @@ async def test_json_dumps() -> None: assert await json_dumps(datetime(2022, 1, 1, tzinfo=timezone.utc)) == '"2022-01-01 00:00:00+00:00"' -# Tests for export_csv_to_stream (dataset CSV export). - - async def async_iter( items: list[Mapping[str, JsonSerializable]], ) -> AsyncIterator[Mapping[str, JsonSerializable]]: @@ -64,12 +61,20 @@ async def test_export_csv_to_stream_collects_all_keys_when_requested() -> None: assert dst.getvalue() == 'name,age,city\nAlice,30,\nBob,25,NYC\n' -async def test_export_csv_to_stream_skips_empty_items() -> None: - """Empty mappings are skipped and do not define or shift the header.""" +@pytest.mark.parametrize( + 'collect_all_keys', + [ + pytest.param(False, id='header from first item'), + pytest.param(True, id='all keys collected'), + ], +) +async def test_export_csv_to_stream_skips_empty_items(*, collect_all_keys: bool) -> None: + """Empty mappings are skipped and neither define the header nor emit a blank row, in either column mode.""" dst = StringIO() await export_csv_to_stream( async_iter([{}, {'id': 1, 'name': 'Item 1'}, {}, {'id': 2, 'name': 'Item 2'}]), dst, + collect_all_keys=collect_all_keys, lineterminator='\n', ) @@ -84,6 +89,23 @@ async def test_export_csv_to_stream_handles_empty_iterator() -> None: assert dst.getvalue() == '' +@pytest.mark.parametrize( + 'items', + [ + pytest.param([], id='no items'), + pytest.param([{}, {}], id='only empty items'), + ], +) +async def test_export_csv_to_stream_collects_all_keys_without_writable_items( + items: list[Mapping[str, JsonSerializable]], +) -> None: + """Key collection produces no CSV content, not a bare header line, when there is nothing to write.""" + dst = StringIO() + await export_csv_to_stream(async_iter(items), dst, collect_all_keys=True) + + assert dst.getvalue() == '' + + async def test_export_csv_to_stream_writes_before_consuming_all_items() -> None: """The default export writes its header before requesting the second item.""" dst = StringIO() @@ -112,6 +134,68 @@ async def test_export_csv_to_stream_preserves_non_json_values() -> None: assert dst.getvalue() == 'created_at\n2020-01-01 00:00:00+00:00\n' +async def test_export_csv_to_stream_warns_about_dropped_keys(caplog: pytest.LogCaptureFixture) -> None: + """Keys dropped because they are absent from the first item are reported once, naming every dropped key.""" + dst = StringIO() + with caplog.at_level('WARNING', logger='crawlee._utils.file'): + await export_csv_to_stream( + async_iter([{'name': 'Alice'}, {'name': 'Bob', 'city': 'NYC'}, {'name': 'Carol', 'age': 40}]), + dst, + lineterminator='\n', + ) + + assert dst.getvalue() == 'name\nAlice\nBob\nCarol\n' + assert len(caplog.records) == 1 + assert 'age, city' in caplog.records[0].message + assert 'collect_all_keys=True' in caplog.records[0].message + + +async def test_export_csv_to_stream_does_not_warn_when_nothing_is_dropped( + caplog: pytest.LogCaptureFixture, +) -> None: + """Collecting all keys drops nothing, so it must not warn even for items with differing key sets.""" + dst = StringIO() + with caplog.at_level('WARNING', logger='crawlee._utils.file'): + await export_csv_to_stream( + async_iter([{'name': 'Alice'}, {'name': 'Bob', 'city': 'NYC'}]), + dst, + collect_all_keys=True, + lineterminator='\n', + ) + + assert dst.getvalue() == 'name,city\nAlice,\nBob,NYC\n' + assert caplog.records == [] + + +@pytest.mark.parametrize( + 'collect_all_keys', + [ + pytest.param(False, id='header from first item'), + pytest.param(True, id='all keys collected'), + ], +) +async def test_export_csv_to_stream_rejects_invalid_writer_options_for_empty_iterator( + *, collect_all_keys: bool +) -> None: + """Writer options are validated up front, so a misconfigured export fails even when there is nothing to write.""" + with pytest.raises(TypeError, match='must be a 1-character string'): + await export_csv_to_stream(async_iter([]), StringIO(), delimiter='ab', collect_all_keys=collect_all_keys) + + +async def test_export_csv_to_stream_honors_restval() -> None: + """`restval` fills the cells of columns an item does not contain.""" + dst = StringIO() + await export_csv_to_stream( + async_iter([{'name': 'Alice'}, {'name': 'Bob', 'city': 'NYC'}]), + dst, + collect_all_keys=True, + restval='N/A', + lineterminator='\n', + ) + + assert dst.getvalue() == 'name,city\nAlice,N/A\nBob,NYC\n' + + # Tests for validate_subdirectory (storage name/alias directory validation). diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 09837f1fae..8240795756 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -783,6 +783,22 @@ async def test_crawler_export_data_additional_kwargs(tmp_path: Path) -> None: assert csv_path.read_text() == 'z;a\n1;2\n' +async def test_crawler_export_data_csv_collect_all_keys(tmp_path: Path) -> None: + crawler = BasicCrawler() + dataset = await Dataset.open() + + await dataset.push_data([{'a': 1}, {'a': 2, 'b': 3}]) + + default_path = tmp_path / 'default.csv' + all_keys_path = tmp_path / 'all_keys.csv' + + await crawler.export_data(path=default_path, lineterminator='\n') + await crawler.export_data(path=all_keys_path, collect_all_keys=True, lineterminator='\n') + + assert default_path.read_text() == 'a\n1\n2\n' + assert all_keys_path.read_text() == 'a,b\n1,\n2,3\n' + + async def test_context_push_and_export_data(tmp_path: Path) -> None: crawler = BasicCrawler() diff --git a/tests/unit/storages/test_dataset.py b/tests/unit/storages/test_dataset.py index 6c630552f3..c18c3e54f3 100644 --- a/tests/unit/storages/test_dataset.py +++ b/tests/unit/storages/test_dataset.py @@ -472,6 +472,46 @@ async def test_export_to_csv( await kvs.drop() +async def test_export_to_csv_columns( + dataset: Dataset, + storage_client: StorageClient, +) -> None: + """Test the CSV columns come from the first item by default, and from all items with `collect_all_keys`.""" + kvs = await KeyValueStore.open( + name='export-kvs', + storage_client=storage_client, + ) + + await dataset.push_data( + [ + {'name': 'Alice', 'age': 30}, + {'name': 'Bob', 'city': 'NYC', 'age': 25}, + {'age': 40, 'name': 'Carol'}, + ] + ) + + await dataset.export_to( + key='first_item_keys.csv', + content_type='csv', + to_kvs_name='export-kvs', + to_kvs_storage_client=storage_client, + lineterminator='\n', + ) + await dataset.export_to( + key='all_keys.csv', + content_type='csv', + to_kvs_name='export-kvs', + to_kvs_storage_client=storage_client, + collect_all_keys=True, + lineterminator='\n', + ) + + assert await kvs.get_value(key='first_item_keys.csv') == 'name,age\nAlice,30\nBob,25\nCarol,40\n' + assert await kvs.get_value(key='all_keys.csv') == 'name,age,city\nAlice,30,\nBob,25,NYC\nCarol,40,\n' + + await kvs.drop() + + async def test_export_to_invalid_content_type(dataset: Dataset) -> None: """Test exporting dataset with invalid content type raises error.""" with pytest.raises(ValueError, match=r'Unsupported content type'): From 94ba83e0d78542a7ff9726c8c0f8f8f6c615b864 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 28 Jul 2026 10:20:10 +0200 Subject: [PATCH 6/9] test(dataset): decouple CSV writer option assertion from CPython wording --- tests/unit/_utils/test_file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/_utils/test_file.py b/tests/unit/_utils/test_file.py index 243da48ca5..6d811fdf4c 100644 --- a/tests/unit/_utils/test_file.py +++ b/tests/unit/_utils/test_file.py @@ -178,7 +178,8 @@ async def test_export_csv_to_stream_rejects_invalid_writer_options_for_empty_ite *, collect_all_keys: bool ) -> None: """Writer options are validated up front, so a misconfigured export fails even when there is nothing to write.""" - with pytest.raises(TypeError, match='must be a 1-character string'): + # Match only the option name: the rest of the message is CPython's wording and it changed in 3.14. + with pytest.raises(TypeError, match='"delimiter"'): await export_csv_to_stream(async_iter([]), StringIO(), delimiter='ab', collect_all_keys=collect_all_keys) From d090f55e17ca30c92629ec9e8be95f1615cc314b Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 28 Jul 2026 12:29:00 +0200 Subject: [PATCH 7/9] fix(dataset): avoid CSV export crashes on non-string keys and mismatched kwargs --- .../export_entire_dataset_to_file.mdx | 6 ++-- src/crawlee/_utils/file.py | 5 +++- src/crawlee/crawlers/_basic/_basic_crawler.py | 17 +++++++++-- tests/unit/_utils/test_file.py | 30 +++++++++++++++++++ .../crawlers/_basic/test_basic_crawler.py | 19 ++++++++++++ 5 files changed, 72 insertions(+), 5 deletions(-) diff --git a/docs/examples/export_entire_dataset_to_file.mdx b/docs/examples/export_entire_dataset_to_file.mdx index a0a7ce4bcc..17c2a10a68 100644 --- a/docs/examples/export_entire_dataset_to_file.mdx +++ b/docs/examples/export_entire_dataset_to_file.mdx @@ -36,10 +36,12 @@ For these examples, we are using the `B Dataset items don't have to share a schema. Different handlers can push different fields, so the items of one dataset often have different keys. -By default, the CSV columns are the keys of the first item. When a later item has a key the first one doesn't, its value isn't written, and Crawlee logs a warning naming the dropped keys. To use the keys of all items as columns instead, pass `collect_all_keys=True`: +By default, the CSV columns are the keys of the first non-empty item. When a later item has a key the first one doesn't, its value isn't written, and Crawlee logs a warning naming the dropped keys. To use the keys of all items as columns instead, pass `collect_all_keys=True`: ```python await crawler.export_data(path='results.csv', collect_all_keys=True) ``` -No value is dropped in that mode. Cells for keys an item doesn't have stay empty, or hold the `restval` value if you pass one. Collecting all keys means reading the whole dataset before the first row can be written, so only the default mode writes rows as it goes. +No value is dropped in that mode. Collecting all keys means reading the whole dataset before the first row can be written, so only the default mode writes rows as it goes. + +In both modes, cells for columns an item doesn't have stay empty, or hold the `restval` value if you pass one. diff --git a/src/crawlee/_utils/file.py b/src/crawlee/_utils/file.py index bdf59e8895..9d2f9177cc 100644 --- a/src/crawlee/_utils/file.py +++ b/src/crawlee/_utils/file.py @@ -250,5 +250,8 @@ async def export_csv_to_stream( 'CSV export dropped %d key(s) not present in the first item: %s. ' 'Pass collect_all_keys=True to include keys from all items as columns.', len(dropped_keys), - ', '.join(sorted(dropped_keys)), + # The keys are annotated as `str`, but nothing enforces that at runtime, and a storage client that + # does not round-trip items through JSON hands them over as pushed. Stringify them so building the + # message cannot fail on a key that is not a string, or on a set mixing several key types. + ', '.join(sorted(str(key) for key in dropped_keys)), ) diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 4a8b304da0..726449ca4a 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -111,6 +111,13 @@ FailedRequestHandler = Callable[[TCrawlingContext, Exception], Awaitable[None]] SkippedRequestCallback = Callable[[str, SkippedReason], Awaitable[None]] +# `export_data` accepts the kwargs of both export formats, since the format is only known once the destination path +# is inspected. These are used to forward just the ones the selected format understands. Both key sets are unioned, +# because `NotRequired` is invisible to `TypedDict` under postponed annotation evaluation, which makes every key land +# in `__required_keys__` instead of `__optional_keys__`. +_CSV_EXPORT_KEYS = ExportDataCsvKwargs.__required_keys__ | ExportDataCsvKwargs.__optional_keys__ +_JSON_EXPORT_KEYS = ExportDataJsonKwargs.__required_keys__ | ExportDataJsonKwargs.__optional_keys__ + class _BasicCrawlerOptions(TypedDict): """Non-generic options the `BasicCrawler` constructor.""" @@ -940,12 +947,18 @@ async def export_data( if path.suffix == '.csv': dst = StringIO() - csv_kwargs = cast('ExportDataCsvKwargs', additional_kwargs) + csv_kwargs = cast( + 'ExportDataCsvKwargs', + {key: value for key, value in additional_kwargs.items() if key in _CSV_EXPORT_KEYS}, + ) await export_csv_to_stream(dataset.iterate_items(), dst, **csv_kwargs) await atomic_write(path, dst.getvalue()) elif path.suffix == '.json': dst = StringIO() - json_kwargs = cast('ExportDataJsonKwargs', additional_kwargs) + json_kwargs = cast( + 'ExportDataJsonKwargs', + {key: value for key, value in additional_kwargs.items() if key in _JSON_EXPORT_KEYS}, + ) await export_json_to_stream(dataset.iterate_items(), dst, **json_kwargs) await atomic_write(path, dst.getvalue()) else: diff --git a/tests/unit/_utils/test_file.py b/tests/unit/_utils/test_file.py index 6d811fdf4c..322ad3fe03 100644 --- a/tests/unit/_utils/test_file.py +++ b/tests/unit/_utils/test_file.py @@ -150,6 +150,23 @@ async def test_export_csv_to_stream_warns_about_dropped_keys(caplog: pytest.LogC assert 'collect_all_keys=True' in caplog.records[0].message +async def test_export_csv_to_stream_warns_about_dropped_keys_that_are_not_strings( + caplog: pytest.LogCaptureFixture, +) -> None: + """Dropped keys are reported even when they are not strings, which a storage client may hand over as pushed.""" + dst = StringIO() + items = cast('list[Mapping[str, JsonSerializable]]', [{'name': 'Alice'}, {1: 'x', 'city': 'NYC'}]) + + with caplog.at_level('WARNING', logger='crawlee._utils.file'): + await export_csv_to_stream(async_iter(items), dst, lineterminator='\n') + + # The second item has no value for the only column, so its row holds a single empty field, written as '""' to + # keep it distinguishable from a blank line. + assert dst.getvalue() == 'name\nAlice\n""\n' + assert len(caplog.records) == 1 + assert '1, city' in caplog.records[0].message + + async def test_export_csv_to_stream_does_not_warn_when_nothing_is_dropped( caplog: pytest.LogCaptureFixture, ) -> None: @@ -197,6 +214,19 @@ async def test_export_csv_to_stream_honors_restval() -> None: assert dst.getvalue() == 'name,city\nAlice,N/A\nBob,NYC\n' +async def test_export_csv_to_stream_honors_restval_without_collecting_all_keys() -> None: + """`restval` also fills the cells of header columns a later item does not contain in the default column mode.""" + dst = StringIO() + await export_csv_to_stream( + async_iter([{'name': 'Alice', 'city': 'NYC'}, {'name': 'Bob'}]), + dst, + restval='N/A', + lineterminator='\n', + ) + + assert dst.getvalue() == 'name,city\nAlice,NYC\nBob,N/A\n' + + # Tests for validate_subdirectory (storage name/alias directory validation). diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 8240795756..f853792758 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -799,6 +799,25 @@ async def test_crawler_export_data_csv_collect_all_keys(tmp_path: Path) -> None: assert all_keys_path.read_text() == 'a,b\n1,\n2,3\n' +async def test_crawler_export_data_ignores_kwargs_of_the_other_format(tmp_path: Path) -> None: + """Each exporter receives only the kwargs of its own format, since the format is inferred from the path.""" + crawler = BasicCrawler() + dataset = await Dataset.open() + + await dataset.push_data([{'a': 1}, {'a': 2, 'b': 3}]) + + json_path = tmp_path / 'dataset.json' + csv_path = tmp_path / 'dataset.csv' + + # `collect_all_keys` is CSV-only and `separators` is JSON-only, so each call passes one kwarg the selected + # format does not understand. + await crawler.export_data(path=json_path, separators=(',', ':'), collect_all_keys=True) + await crawler.export_data(path=csv_path, separators=(',', ':'), lineterminator='\n') + + assert json_path.read_text() == '[{"a":1},{"a":2,"b":3}]' + assert csv_path.read_text() == 'a\n1\n2\n' + + async def test_context_push_and_export_data(tmp_path: Path) -> None: crawler = BasicCrawler() From 46da14f05825dbc716b483f16fdbad960f728698 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 28 Jul 2026 12:45:18 +0200 Subject: [PATCH 8/9] fix(dataset): warn about export_data kwargs of the other format --- src/crawlee/_types.py | 2 +- src/crawlee/crawlers/_basic/_basic_crawler.py | 29 ++++++++++++++----- .../crawlers/_basic/test_basic_crawler.py | 21 ++++++++++---- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 6bd500e3a2..815063424e 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -828,5 +828,5 @@ class ExportDataKwargs(ExportDataJsonKwargs, ExportDataCsvKwargs): Combines all `ExportDataJsonKwargs` and `ExportDataCsvKwargs` fields, since the export format is determined dynamically from the file extension at call time. Only the kwargs relevant to the selected - format are forwarded to the underlying exporter. + format are forwarded to the underlying exporter, the rest are ignored and named in a warning. """ diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 261ad2f2a8..622a0bae59 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -947,23 +947,36 @@ async def export_data( if path.suffix == '.csv': dst = StringIO() - csv_kwargs = cast( - 'ExportDataCsvKwargs', - {key: value for key, value in additional_kwargs.items() if key in _CSV_EXPORT_KEYS}, - ) + csv_kwargs = cast('ExportDataCsvKwargs', self._select_export_kwargs(additional_kwargs, 'CSV')) await export_csv_to_stream(dataset.iterate_items(), dst, **csv_kwargs) await atomic_write(path, dst.getvalue()) elif path.suffix == '.json': dst = StringIO() - json_kwargs = cast( - 'ExportDataJsonKwargs', - {key: value for key, value in additional_kwargs.items() if key in _JSON_EXPORT_KEYS}, - ) + json_kwargs = cast('ExportDataJsonKwargs', self._select_export_kwargs(additional_kwargs, 'JSON')) await export_json_to_stream(dataset.iterate_items(), dst, **json_kwargs) await atomic_write(path, dst.getvalue()) else: raise ValueError(f'Unsupported file extension: {path.suffix}') + def _select_export_kwargs(self, kwargs: Mapping[str, Any], export_format: Literal['CSV', 'JSON']) -> dict[str, Any]: + """Pick the kwargs the chosen export format understands, reporting the ones that are left out. + + `export_data` accepts the kwargs of both formats, because the format is only known once the destination + path is inspected, so passing one that belongs to the other format is a plausible mistake. Forwarding it + would make the underlying exporter raise, and dropping it without a word would hide the mistake. + """ + supported_keys = _CSV_EXPORT_KEYS if export_format == 'CSV' else _JSON_EXPORT_KEYS + + if ignored_keys := sorted(key for key in kwargs if key not in supported_keys): + self._logger.warning( + 'Ignoring %d keyword argument(s) of `export_data` that the %s export does not support: %s.', + len(ignored_keys), + export_format, + ', '.join(ignored_keys), + ) + + return {key: value for key, value in kwargs.items() if key in supported_keys} + async def _push_data( self, data: Sequence[Mapping[str, JsonSerializable]] | Mapping[str, JsonSerializable], diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 58049889fe..bca9e02cf4 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -799,24 +799,33 @@ async def test_crawler_export_data_csv_collect_all_keys(tmp_path: Path) -> None: assert all_keys_path.read_text() == 'a,b\n1,\n2,3\n' -async def test_crawler_export_data_ignores_kwargs_of_the_other_format(tmp_path: Path) -> None: - """Each exporter receives only the kwargs of its own format, since the format is inferred from the path.""" +async def test_crawler_export_data_warns_about_kwargs_of_the_other_format( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Each exporter receives only the kwargs of its own format, and the ones left out are named in a warning.""" crawler = BasicCrawler() dataset = await Dataset.open() - await dataset.push_data([{'a': 1}, {'a': 2, 'b': 3}]) + await dataset.push_data([{'a': 1}, {'a': 2}]) json_path = tmp_path / 'dataset.json' csv_path = tmp_path / 'dataset.csv' # `collect_all_keys` is CSV-only and `separators` is JSON-only, so each call passes one kwarg the selected # format does not understand. - await crawler.export_data(path=json_path, separators=(',', ':'), collect_all_keys=True) - await crawler.export_data(path=csv_path, separators=(',', ':'), lineterminator='\n') + with caplog.at_level('WARNING', logger='crawlee.crawlers._basic._basic_crawler'): + await crawler.export_data(path=json_path, separators=(',', ':'), collect_all_keys=True) + await crawler.export_data(path=csv_path, separators=(',', ':'), lineterminator='\n') - assert json_path.read_text() == '[{"a":1},{"a":2,"b":3}]' + assert json_path.read_text() == '[{"a":1},{"a":2}]' assert csv_path.read_text() == 'a\n1\n2\n' + messages = [record.message for record in caplog.records if record.name == 'crawlee.crawlers._basic._basic_crawler'] + assert len(messages) == 2 + assert 'JSON export does not support: collect_all_keys' in messages[0] + assert 'CSV export does not support: separators' in messages[1] + async def test_context_push_and_export_data(tmp_path: Path) -> None: crawler = BasicCrawler() From a7f475dc1ce267ea0179bc0b11896d7dfae08eb5 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 28 Jul 2026 13:31:53 +0200 Subject: [PATCH 9/9] fix(dataset): stop filtering export_data kwargs by format --- src/crawlee/_types.py | 2 +- src/crawlee/crawlers/_basic/_basic_crawler.py | 30 ++----------------- .../crawlers/_basic/test_basic_crawler.py | 28 ----------------- 3 files changed, 3 insertions(+), 57 deletions(-) diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 815063424e..6bd500e3a2 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -828,5 +828,5 @@ class ExportDataKwargs(ExportDataJsonKwargs, ExportDataCsvKwargs): Combines all `ExportDataJsonKwargs` and `ExportDataCsvKwargs` fields, since the export format is determined dynamically from the file extension at call time. Only the kwargs relevant to the selected - format are forwarded to the underlying exporter, the rest are ignored and named in a warning. + format are forwarded to the underlying exporter. """ diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 622a0bae59..554d668075 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -111,13 +111,6 @@ FailedRequestHandler = Callable[[TCrawlingContext, Exception], Awaitable[None]] SkippedRequestCallback = Callable[[str, SkippedReason], Awaitable[None]] -# `export_data` accepts the kwargs of both export formats, since the format is only known once the destination path -# is inspected. These are used to forward just the ones the selected format understands. Both key sets are unioned, -# because `NotRequired` is invisible to `TypedDict` under postponed annotation evaluation, which makes every key land -# in `__required_keys__` instead of `__optional_keys__`. -_CSV_EXPORT_KEYS = ExportDataCsvKwargs.__required_keys__ | ExportDataCsvKwargs.__optional_keys__ -_JSON_EXPORT_KEYS = ExportDataJsonKwargs.__required_keys__ | ExportDataJsonKwargs.__optional_keys__ - class _BasicCrawlerOptions(TypedDict): """Non-generic options the `BasicCrawler` constructor.""" @@ -947,36 +940,17 @@ async def export_data( if path.suffix == '.csv': dst = StringIO() - csv_kwargs = cast('ExportDataCsvKwargs', self._select_export_kwargs(additional_kwargs, 'CSV')) + csv_kwargs = cast('ExportDataCsvKwargs', additional_kwargs) await export_csv_to_stream(dataset.iterate_items(), dst, **csv_kwargs) await atomic_write(path, dst.getvalue()) elif path.suffix == '.json': dst = StringIO() - json_kwargs = cast('ExportDataJsonKwargs', self._select_export_kwargs(additional_kwargs, 'JSON')) + json_kwargs = cast('ExportDataJsonKwargs', additional_kwargs) await export_json_to_stream(dataset.iterate_items(), dst, **json_kwargs) await atomic_write(path, dst.getvalue()) else: raise ValueError(f'Unsupported file extension: {path.suffix}') - def _select_export_kwargs(self, kwargs: Mapping[str, Any], export_format: Literal['CSV', 'JSON']) -> dict[str, Any]: - """Pick the kwargs the chosen export format understands, reporting the ones that are left out. - - `export_data` accepts the kwargs of both formats, because the format is only known once the destination - path is inspected, so passing one that belongs to the other format is a plausible mistake. Forwarding it - would make the underlying exporter raise, and dropping it without a word would hide the mistake. - """ - supported_keys = _CSV_EXPORT_KEYS if export_format == 'CSV' else _JSON_EXPORT_KEYS - - if ignored_keys := sorted(key for key in kwargs if key not in supported_keys): - self._logger.warning( - 'Ignoring %d keyword argument(s) of `export_data` that the %s export does not support: %s.', - len(ignored_keys), - export_format, - ', '.join(ignored_keys), - ) - - return {key: value for key, value in kwargs.items() if key in supported_keys} - async def _push_data( self, data: Sequence[Mapping[str, JsonSerializable]] | Mapping[str, JsonSerializable], diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index bca9e02cf4..763969b3f9 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -799,34 +799,6 @@ async def test_crawler_export_data_csv_collect_all_keys(tmp_path: Path) -> None: assert all_keys_path.read_text() == 'a,b\n1,\n2,3\n' -async def test_crawler_export_data_warns_about_kwargs_of_the_other_format( - tmp_path: Path, - caplog: pytest.LogCaptureFixture, -) -> None: - """Each exporter receives only the kwargs of its own format, and the ones left out are named in a warning.""" - crawler = BasicCrawler() - dataset = await Dataset.open() - - await dataset.push_data([{'a': 1}, {'a': 2}]) - - json_path = tmp_path / 'dataset.json' - csv_path = tmp_path / 'dataset.csv' - - # `collect_all_keys` is CSV-only and `separators` is JSON-only, so each call passes one kwarg the selected - # format does not understand. - with caplog.at_level('WARNING', logger='crawlee.crawlers._basic._basic_crawler'): - await crawler.export_data(path=json_path, separators=(',', ':'), collect_all_keys=True) - await crawler.export_data(path=csv_path, separators=(',', ':'), lineterminator='\n') - - assert json_path.read_text() == '[{"a":1},{"a":2}]' - assert csv_path.read_text() == 'a\n1\n2\n' - - messages = [record.message for record in caplog.records if record.name == 'crawlee.crawlers._basic._basic_crawler'] - assert len(messages) == 2 - assert 'JSON export does not support: collect_all_keys' in messages[0] - assert 'CSV export does not support: separators' in messages[1] - - async def test_context_push_and_export_data(tmp_path: Path) -> None: crawler = BasicCrawler()