From 6befa28abeafe572413bec56620436b0f6627171 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Wed, 21 Oct 2026 13:13:54 +0100 Subject: [PATCH 1/4] Preserve null lists when projecting a list of structs Co-Authored-By: Claude Opus 5.5 --- pyiceberg/io/pyarrow.py | 3 ++- tests/io/test_pyarrow.py | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 7fd0c85b77..1ddb925416 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2111,7 +2111,8 @@ def list(self, list_type: ListType, list_array: pa.Array | None, value_array: pa if isinstance(value_array, pa.StructArray): # This can be removed once this has been fixed: # https://github.com/apache/arrow/issues/38809 - list_array = pa.LargeListArray.from_arrays(list_array.offsets, value_array) + # Pass the validity mask, otherwise null lists become empty lists + list_array = pa.LargeListArray.from_arrays(list_array.offsets, value_array, mask=list_array.is_null()) value_array = self._cast_if_needed(list_type.element_field, value_array) arrow_field = list_initializer(self._construct_field(list_type.element_field, value_array.type)) return list_array.cast(arrow_field) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index e97b7813da..c3d798cbf7 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3579,6 +3579,33 @@ def test__to_requested_schema_null_map_of_structs() -> None: assert result.column(1).to_pylist() == [[], None, [("b", 2), ("c", None)]] +def test__to_requested_schema_null_list_of_structs() -> None: + requested_schema = Schema( + NestedField( + 1, + "col_list_with_struct", + ListType( + 11, + StructType(NestedField(111, "source", StringType(), required=False)), + element_required=False, + ), + required=False, + ), + ) + arrow_schema = pa.schema([pa.field("col_list_with_struct", pa.list_(pa.struct([pa.field("source", pa.string())])))]) + batch = pa.RecordBatch.from_arrays( + [pa.array([[{"source": "x"}], [], None, [{"source": "y"}, None]], type=arrow_schema.field(0).type)], + schema=arrow_schema, + ) + + result = _to_requested_schema(requested_schema, requested_schema, batch) + assert result.column(0).to_pylist() == [[{"source": "x"}], [], None, [{"source": "y"}, None]] + + # A sliced batch has to keep the nulls as well + result = _to_requested_schema(requested_schema, requested_schema, batch.slice(1, 3)) + assert result.column(0).to_pylist() == [[], None, [{"source": "y"}, None]] + + @pytest.mark.parametrize( "arrow_type,iceberg_type,expected_arrow_type", [ From 6cce597d69de45cca1dafc70353bd612430e7085 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Wed, 21 Oct 2026 19:08:08 +0100 Subject: [PATCH 2/4] Expect the null list of structs to stay null in test_null_list_and_map Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_reads.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/integration/test_reads.py b/tests/integration/test_reads.py index a151d62b82..3e07c87b09 100644 --- a/tests/integration/test_reads.py +++ b/tests/integration/test_reads.py @@ -1001,10 +1001,7 @@ def test_null_list_and_map(catalog: Catalog) -> None: arrow_table = table_test_empty_list_and_map.scan().to_arrow() assert arrow_table["col_list"].to_pylist() == [None, []] assert arrow_table["col_map"].to_pylist() == [None, []] - # This should be: - # assert arrow_table["col_list_with_struct"].to_pylist() == [None, [{'test': 1}]] - # Once https://github.com/apache/arrow/issues/38809 has been fixed - assert arrow_table["col_list_with_struct"].to_pylist() == [[], [{"test": 1}]] + assert arrow_table["col_list_with_struct"].to_pylist() == [None, [{"test": 1}]] @pytest.mark.integration From 640f09ec7569a1785934512ac9f88aefe0e893c9 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Thu, 22 Oct 2026 09:11:44 +0100 Subject: [PATCH 3/4] Retrigger CI (integration-test-s3 was cancelled while installing uv) From d28751b621278899c5f208334da430a8d1da579b Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Fri, 23 Oct 2026 09:39:36 +0100 Subject: [PATCH 4/4] Rebase list offsets before passing the null mask LargeListArray.from_arrays refuses a validity mask with sliced offsets, so a sliced large_list of structs raised. Rebase the offsets to start at 0 and slice the values to match, and test list and large_list, whole and sliced. --- pyiceberg/io/pyarrow.py | 10 ++++++++-- tests/io/test_pyarrow.py | 10 +++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 1ddb925416..472db599e0 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2111,8 +2111,14 @@ def list(self, list_type: ListType, list_array: pa.Array | None, value_array: pa if isinstance(value_array, pa.StructArray): # This can be removed once this has been fixed: # https://github.com/apache/arrow/issues/38809 - # Pass the validity mask, otherwise null lists become empty lists - list_array = pa.LargeListArray.from_arrays(list_array.offsets, value_array, mask=list_array.is_null()) + # Pass the validity mask, otherwise null lists become empty lists. Arrow refuses a mask + # with sliced offsets, so rebase the offsets onto the values the list covers. + start, end = list_array.offsets[0].as_py(), list_array.offsets[-1].as_py() + list_array = pa.LargeListArray.from_arrays( + pc.subtract(list_array.offsets, start), + value_array.slice(start, end - start), + mask=list_array.is_null(), + ) value_array = self._cast_if_needed(list_type.element_field, value_array) arrow_field = list_initializer(self._construct_field(list_type.element_field, value_array.type)) return list_array.cast(arrow_field) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index c3d798cbf7..31e7b61fad 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -24,7 +24,7 @@ import uuid import warnings import zlib -from collections.abc import Iterator +from collections.abc import Callable, Iterator from datetime import date, datetime, timezone from pathlib import Path from typing import Any, Literal @@ -3579,7 +3579,8 @@ def test__to_requested_schema_null_map_of_structs() -> None: assert result.column(1).to_pylist() == [[], None, [("b", 2), ("c", None)]] -def test__to_requested_schema_null_list_of_structs() -> None: +@pytest.mark.parametrize("list_type", [pa.list_, pa.large_list]) +def test__to_requested_schema_null_list_of_structs(list_type: Callable[[pa.DataType], pa.DataType]) -> None: requested_schema = Schema( NestedField( 1, @@ -3592,7 +3593,7 @@ def test__to_requested_schema_null_list_of_structs() -> None: required=False, ), ) - arrow_schema = pa.schema([pa.field("col_list_with_struct", pa.list_(pa.struct([pa.field("source", pa.string())])))]) + arrow_schema = pa.schema([pa.field("col_list_with_struct", list_type(pa.struct([pa.field("source", pa.string())])))]) batch = pa.RecordBatch.from_arrays( [pa.array([[{"source": "x"}], [], None, [{"source": "y"}, None]], type=arrow_schema.field(0).type)], schema=arrow_schema, @@ -3605,6 +3606,9 @@ def test__to_requested_schema_null_list_of_structs() -> None: result = _to_requested_schema(requested_schema, requested_schema, batch.slice(1, 3)) assert result.column(0).to_pylist() == [[], None, [{"source": "y"}, None]] + result = _to_requested_schema(requested_schema, requested_schema, batch.slice(2, 0)) + assert result.column(0).to_pylist() == [] + @pytest.mark.parametrize( "arrow_type,iceberg_type,expected_arrow_type",