From 504ae5de35567520ebacffa641621f1d8058fcb5 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:09:40 +0000 Subject: [PATCH 1/2] fix: apply struct field filters when the file schema needs adaptation (#24125) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes https://github.com/apache/datafusion/issues/24109. ## Rationale for this change With `datafusion.execution.parquet.pushdown_filters = true`, a filter on a struct field returns **all rows** when the declared table schema differs from the physical file schema for that column: ```sql -- file stores s as Struct, table declares Struct SELECT id, s['x'] FROM t WHERE s['x'] = 200; -- returns 3 rows instead of 1 ``` The planning-time decision and the runtime construction disagree: 1. `ParquetSource::try_pushdown_filters` evaluates `can_expr_be_pushed_down_with_schemas` against the **table** schema. `get_field(s, 'x')` has a bare column under the `get_field`, so it reports the predicate as fully handled and `FilterExec` is removed from the plan. 2. At open time the expression adapter rewrites the predicate against the **file** schema. Because the struct types differ, `rewrite_column` wraps the whole column in a cast, giving `get_field(cast(s AS Struct), 'x')`. 3. `PushdownChecker` only recognizes `get_field` whose first argument is a `Column`. It now sees a `CastExpr`, falls through to normal traversal, hits the struct `Column`, and rejects pushdown — so no row filter is built and the conjunct is silently dropped. Nothing applies the predicate, and the scan returns unfiltered rows. ## What changes are included in this PR? Narrow the cast to the field that is actually read, in `DefaultPhysicalExprAdapter`: ``` get_field(cast(s AS Struct), 'x') -> cast(get_field(s, 'x') AS Int64) ``` Expressions are rewritten bottom-up, so the new `try_narrow_struct_cast` matches the `get_field` node after its struct argument has already been wrapped, and rebuilds the `get_field` over the uncast struct (recomputing its return field from the physical field type) with the cast moved outside. This keeps the column visible under the `get_field`, so the Parquet row filter builder makes good on what planning promised. Two details worth calling out: - A field that is missing from the file collapses to a typed null literal, matching what the struct cast would have produced (DataFusion's struct casts match by name and fill missing target fields with nulls). - `get_field` on a `Map` column is a runtime key lookup rather than a schema-level field access, so map values keep the whole-column cast. As a side effect this also avoids materializing an entire cast struct just to read one field, which is a small win for any struct-field access over an evolved schema — not only for filters. ### Not addressed here The issue also raises the broader concern that "a static determination made at planning time about what the scan can do, and the runtime construction that has to make good on it, are computed by different code against different schemas, and there is no mechanism forcing them to agree." This PR fixes the reported wrong-results bug; it does not add a mechanism (e.g. post-decode filtering in `ParquetOpener`) that would make any future divergence safe by construction. That seems worth doing separately. ## Are these changes tested? Yes. - `datafusion/physical-expr-adapter/src/schema_rewriter.rs`: unit tests for the narrowed cast (flat and nested field access), the missing-field null literal, and that Map columns keep their cast. - `datafusion/datasource-parquet/src/opener/mod.rs`: end-to-end opener tests reading a `Struct` file through a `Struct` table schema with pushdown enabled, plus a matching-schema control. - `datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt`: a SQL-level regression test. Verified that it fails on `main` (returns all 3 rows) and passes with the fix. Full runs: `cargo clippy --all-targets --all-features -- -D warnings`, the complete sqllogictest suite (498 files), `datafusion-physical-expr-adapter`, `datafusion-datasource-parquet`, and the `datafusion` `core_integration` / `parquet_integration` suites all pass. ## Are there any user-facing changes? A wrong-results bug fix: struct-field predicates are now applied when the scan needs schema adaptation. No public API changes. --- _Generated by [Claude Code](https://claude.ai/code/session_01MebN5PsVnYvXUeVKju5K7P)_ --------- Co-authored-by: Claude --- .../src/schema_rewriter.rs | 533 +++++++++++++++++- .../test_files/parquet_filter_pushdown.slt | 113 ++++ .../parquet_nested_schema_pruning.slt | 12 +- 3 files changed, 649 insertions(+), 9 deletions(-) diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rs index ef25af7d920fb..a5d1c494c8ae4 100644 --- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs +++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs @@ -25,7 +25,7 @@ use std::hash::Hash; use std::sync::Arc; use arrow::array::RecordBatch; -use arrow::datatypes::{DataType, FieldRef, SchemaRef}; +use arrow::datatypes::{DataType, FieldRef, Fields, SchemaRef}; use datafusion_common::{ DataFusionError, Result, ScalarValue, exec_err, metadata::FieldMetadata, @@ -273,6 +273,42 @@ struct DefaultPhysicalExprAdapterRewriter { physical_file_schema: SchemaRef, } +/// Outcome of walking a `get_field` key path through nested struct fields. +enum FieldPathResolution<'a> { + /// The leaf field the path points at. + Found(&'a FieldRef), + /// Some key along the path does not exist, so the access reads as null. + Missing, + /// An intermediate field is not a struct, so the path cannot be resolved + /// statically. + NotAStruct, +} + +/// Follow a `get_field` key path (`['a', 'b']` for `s['a']['b']`) through +/// nested struct fields. +/// +/// The first key is taken separately from the rest so that the "at least one +/// key" invariant is carried by the signature: there is no empty path to +/// resolve. +fn resolve_field_path<'a>( + fields: &'a Fields, + field_name: &str, + rest: &[&str], +) -> FieldPathResolution<'a> { + let Some(field) = fields.iter().find(|f| f.name() == field_name) else { + return FieldPathResolution::Missing; + }; + let Some((next_field_name, rest)) = rest.split_first() else { + return FieldPathResolution::Found(field); + }; + match field.data_type() { + DataType::Struct(nested_fields) => { + resolve_field_path(nested_fields, next_field_name, rest) + } + _ => FieldPathResolution::NotAStruct, + } +} + impl DefaultPhysicalExprAdapterRewriter { fn rewrite_expr( &self, @@ -282,6 +318,10 @@ impl DefaultPhysicalExprAdapterRewriter { return Ok(Transformed::yes(transformed)); } + if let Some(transformed) = self.try_narrow_struct_cast(&expr)? { + return Ok(Transformed::yes(transformed)); + } + if let Some(column) = expr.downcast_ref::() { return self.rewrite_column(Arc::clone(&expr), column); } @@ -289,6 +329,135 @@ impl DefaultPhysicalExprAdapterRewriter { Ok(Transformed::no(expr)) } + /// Rewrite `get_field(cast(s AS Struct<..>), 'f')` into + /// `cast(get_field(s, 'f') AS )`. + /// + /// Expressions are rewritten bottom-up, so by the time we reach a + /// `get_field` node its struct argument has already been wrapped in a cast + /// by [`Self::rewrite_column`] whenever the logical and physical struct + /// types differ. + /// + /// Narrowing that cast is worthwhile for two reasons: + /// + /// 1. Reading one field should not cost a whole struct. The wide form + /// casts every field of the column — including ones the query never + /// reads — to produce a value that is immediately discarded except for + /// one field. + /// 2. It keeps the column visible. Consumers throughout the codebase + /// pattern match on `get_field(column, 'f')` to recognise a struct + /// field access; a cast between the `get_field` and its column defeats + /// that match, and each such consumer then falls back to whatever it + /// does for an unrecognised expression. + /// + /// The Parquet scan is one such consumer, and the reason this is a + /// correctness fix rather than only an optimisation: it decides at + /// planning time, against the table schema, that a struct-field predicate + /// can be evaluated as a row filter, and reports the predicate as fully + /// handled. See . + /// + /// Fixing it here rather than teaching that one consumer to see through + /// casts is deliberate: the adapter is where the obscuring cast is + /// introduced, so every consumer benefits, and no consumer has to loosen + /// its pattern to accept arbitrary casts between a `get_field` and its + /// column. + /// + /// `get_field` also has a flattened multi-key form: `s['a']['b']` is + /// simplified to `get_field(s, 'a', 'b')`, so the whole field path is + /// resolved here rather than just the first key. + /// + /// Only struct casts are narrowed. `get_field` on a Map column performs a + /// runtime key lookup rather than a schema-level field access, so the map + /// value must keep its cast. + fn try_narrow_struct_cast( + &self, + expr: &Arc, + ) -> Result>> { + let Some(get_field_expr) = + ScalarFunctionExpr::try_downcast_func::(expr.as_ref()) + else { + return Ok(None); + }; + let Some((source_expr, field_name_exprs)) = get_field_expr.args().split_first() + else { + return Ok(None); + }; + let Some(cast) = source_expr.downcast_ref::() else { + return Ok(None); + }; + + // Every key has to be a string literal, otherwise the leaf field + // cannot be resolved statically. + let mut field_path = Vec::with_capacity(field_name_exprs.len()); + for field_name_expr in field_name_exprs { + let Some(field_name) = field_name_expr + .downcast_ref::() + .and_then(|lit| lit.value().try_as_str().flatten()) + else { + return Ok(None); + }; + field_path.push(field_name); + } + // A `get_field` with no keys is not a field access we can narrow. + let Some((first_key, rest_keys)) = field_path.split_first() else { + return Ok(None); + }; + + let DataType::Struct(logical_struct_fields) = cast.target_field().data_type() + else { + return Ok(None); + }; + let FieldPathResolution::Found(logical_struct_field) = + resolve_field_path(logical_struct_fields, first_key, rest_keys) + else { + return Ok(None); + }; + + let inner = cast.expr(); + let DataType::Struct(physical_struct_fields) = + inner.data_type(&self.physical_file_schema)? + else { + return Ok(None); + }; + let physical_struct_field = + match resolve_field_path(&physical_struct_fields, first_key, rest_keys) { + FieldPathResolution::Found(field) => field, + FieldPathResolution::Missing => { + // The file does not have this field at all, so reading it + // yields null. Note that the cast would have produced the + // same value: struct casts fill missing target fields with + // nulls. + let null_value = + ScalarValue::Null.cast_to(logical_struct_field.data_type())?; + return Ok(Some(Arc::new(Literal::new_with_metadata( + null_value, + Some(FieldMetadata::from(logical_struct_field.as_ref())), + )))); + } + FieldPathResolution::NotAStruct => return Ok(None), + }; + + // Rebuild `get_field` over the uncast struct so its return field is + // recomputed from the physical field type. + let mut args = Vec::with_capacity(get_field_expr.args().len()); + args.push(Arc::clone(inner)); + args.extend(field_name_exprs.iter().map(Arc::clone)); + let extracted = Arc::new(ScalarFunctionExpr::try_new( + Arc::new(get_field_expr.fun().clone()), + args, + &self.physical_file_schema, + Arc::new(get_field_expr.config_options().clone()), + )?) as Arc; + + if physical_struct_field == logical_struct_field { + return Ok(Some(extracted)); + } + Ok(Some(Arc::new(CastExpr::new_with_target_field( + extracted, + Arc::clone(logical_struct_field), + Some(cast.cast_options().clone()), + )))) + } + /// Attempt to rewrite struct field access expressions to return null if the field does not exist in the physical schema. /// Note that this does *not* handle nested struct fields, only top-level struct field access. /// See for more details. @@ -1426,6 +1595,368 @@ mod tests { // datafusion/core/tests/parquet/schema_adapter.rs provide better coverage for this functionality. } + /// Build `get_field(column, 'field')` against `schema`. + fn get_field_expr( + schema: &Schema, + column: &str, + field: &str, + ) -> Arc { + let index = schema.index_of(column).unwrap(); + Arc::new( + ScalarFunctionExpr::try_new( + Arc::new(datafusion_expr::ScalarUDF::from(GetFieldFunc::new())), + vec![ + Arc::new(Column::new(column, index)), + Arc::new(Literal::new(ScalarValue::from(field))), + ], + schema, + Arc::new(datafusion_common::config::ConfigOptions::default()), + ) + .unwrap(), + ) + } + + fn struct_schemas( + physical_fields: Vec, + logical_fields: Vec, + ) -> (SchemaRef, SchemaRef) { + let physical = Arc::new(Schema::new(vec![Field::new( + "s", + DataType::Struct(physical_fields.into()), + true, + )])); + let logical = Arc::new(Schema::new(vec![Field::new( + "s", + DataType::Struct(logical_fields.into()), + true, + )])); + (logical, physical) + } + + /// `s['x']` where the file stores `x` as `Int32` and the table declares + /// `Int64` must cast the extracted field, not the whole struct, so that + /// the column stays visible under the `get_field`. + /// + /// See . + #[test] + fn test_narrow_struct_cast_to_field_access() { + let (logical_schema, physical_schema) = struct_schemas( + vec![Field::new("x", DataType::Int32, true)], + vec![Field::new("x", DataType::Int64, true)], + ); + + let adapter = DefaultPhysicalExprAdapterFactory + .create(Arc::clone(&logical_schema), physical_schema) + .unwrap(); + let rewritten = adapter + .rewrite(get_field_expr(&logical_schema, "s", "x")) + .unwrap(); + + let cast = assert_cast_expr(&rewritten); + assert_eq!(cast.cast_type(), &DataType::Int64); + let get_field = cast + .expr() + .downcast_ref::() + .expect("Expected get_field under the cast"); + assert_eq!(get_field.return_type(), &DataType::Int32); + assert!( + get_field.args()[0].downcast_ref::().is_some(), + "the struct column must not be hidden behind a cast, got: {rewritten}" + ); + } + + /// A struct field that only differs in a nested leaf type still ends up + /// with a single cast on the extracted field. + #[test] + fn test_narrow_struct_cast_nested_field_access() { + let (logical_schema, physical_schema) = struct_schemas( + vec![Field::new( + "inner", + DataType::Struct(vec![Field::new("x", DataType::Utf8, true)].into()), + true, + )], + vec![Field::new( + "inner", + DataType::Struct(vec![Field::new("x", DataType::Utf8View, true)].into()), + true, + )], + ); + + let adapter = DefaultPhysicalExprAdapterFactory + .create(Arc::clone(&logical_schema), physical_schema) + .unwrap(); + let outer = get_field_expr(&logical_schema, "s", "inner"); + let expr = Arc::new( + ScalarFunctionExpr::try_new( + Arc::new(datafusion_expr::ScalarUDF::from(GetFieldFunc::new())), + vec![outer, Arc::new(Literal::new(ScalarValue::from("x")))], + &logical_schema, + Arc::new(datafusion_common::config::ConfigOptions::default()), + ) + .unwrap(), + ) as Arc; + + let rewritten = adapter.rewrite(expr).unwrap(); + + let cast = assert_cast_expr(&rewritten); + assert_eq!(cast.cast_type(), &DataType::Utf8View); + let outer_get_field = cast + .expr() + .downcast_ref::() + .expect("Expected get_field under the cast"); + let inner_get_field = outer_get_field.args()[0] + .downcast_ref::() + .expect("Expected a nested get_field"); + assert!( + inner_get_field.args()[0].downcast_ref::().is_some(), + "the struct column must not be hidden behind a cast, got: {rewritten}" + ); + } + + /// A struct column that needs no adaptation at all is left completely + /// alone — the narrowing must not disturb the common case. + #[test] + fn test_narrow_struct_cast_leaves_matching_schema_alone() { + let (logical_schema, physical_schema) = struct_schemas( + vec![Field::new("x", DataType::Int32, true)], + vec![Field::new("x", DataType::Int32, true)], + ); + + let adapter = DefaultPhysicalExprAdapterFactory + .create(Arc::clone(&logical_schema), physical_schema) + .unwrap(); + let expr = get_field_expr(&logical_schema, "s", "x"); + + let rewritten = adapter.rewrite(Arc::clone(&expr)).unwrap(); + + assert_eq!( + rewritten.to_string(), + expr.to_string(), + "an unadapted struct column must pass through untouched" + ); + } + + /// When the accessed field has the same type in both schemas, the struct + /// cast disappears entirely rather than being replaced by a field cast: + /// only a sibling field forced the column-level cast in the first place. + #[test] + fn test_narrow_struct_cast_drops_cast_when_field_types_match() { + let (logical_schema, physical_schema) = struct_schemas( + vec![ + Field::new("x", DataType::Int32, true), + Field::new("y", DataType::Int32, true), + ], + vec![ + Field::new("x", DataType::Int32, true), + Field::new("y", DataType::Int64, true), + ], + ); + + let adapter = DefaultPhysicalExprAdapterFactory + .create(Arc::clone(&logical_schema), physical_schema) + .unwrap(); + let expr = Arc::new( + ScalarFunctionExpr::try_new( + Arc::new(datafusion_expr::ScalarUDF::from(GetFieldFunc::new())), + vec![ + Arc::new(Column::new("s", 0)), + Arc::new(Literal::new(ScalarValue::from("x"))), + ], + &logical_schema, + Arc::new(datafusion_common::config::ConfigOptions::default()), + ) + .unwrap(), + ) as Arc; + + let rewritten = adapter.rewrite(expr).unwrap(); + + assert!( + rewritten.downcast_ref::().is_none(), + "`x` has the same type in both schemas, so no cast is needed, got: {rewritten}" + ); + let get_field = rewritten + .downcast_ref::() + .expect("Expected a bare get_field"); + assert_eq!(get_field.return_type(), &DataType::Int32); + assert!( + get_field.args()[0].downcast_ref::().is_some(), + "the struct column must not be hidden behind a cast, got: {rewritten}" + ); + } + + /// `s['inner']['x']` is simplified to the flattened `get_field(s, 'inner', + /// 'x')`, so the whole key path has to be resolved. + #[test] + fn test_narrow_struct_cast_flattened_field_path() { + let (logical_schema, physical_schema) = struct_schemas( + vec![Field::new( + "inner", + DataType::Struct(vec![Field::new("x", DataType::Utf8, true)].into()), + true, + )], + vec![Field::new( + "inner", + DataType::Struct(vec![Field::new("x", DataType::Utf8View, true)].into()), + true, + )], + ); + + let adapter = DefaultPhysicalExprAdapterFactory + .create(Arc::clone(&logical_schema), physical_schema) + .unwrap(); + let expr = Arc::new( + ScalarFunctionExpr::try_new( + Arc::new(datafusion_expr::ScalarUDF::from(GetFieldFunc::new())), + vec![ + Arc::new(Column::new("s", 0)), + Arc::new(Literal::new(ScalarValue::from("inner"))), + Arc::new(Literal::new(ScalarValue::from("x"))), + ], + &logical_schema, + Arc::new(datafusion_common::config::ConfigOptions::default()), + ) + .unwrap(), + ) as Arc; + + let rewritten = adapter.rewrite(expr).unwrap(); + + let cast = assert_cast_expr(&rewritten); + assert_eq!(cast.cast_type(), &DataType::Utf8View); + let get_field = cast + .expr() + .downcast_ref::() + .expect("Expected get_field under the cast"); + assert_eq!(get_field.return_type(), &DataType::Utf8); + assert_eq!( + get_field.args().len(), + 3, + "the full key path must be preserved, got: {rewritten}" + ); + assert!( + get_field.args()[0].downcast_ref::().is_some(), + "the struct column must not be hidden behind a cast, got: {rewritten}" + ); + } + + /// A key path whose leaf is missing from the file still resolves to a + /// typed null literal. + #[test] + fn test_narrow_struct_cast_flattened_field_path_missing_leaf() { + let (logical_schema, physical_schema) = struct_schemas( + vec![Field::new( + "inner", + DataType::Struct(vec![Field::new("x", DataType::Int32, true)].into()), + true, + )], + vec![Field::new( + "inner", + DataType::Struct( + vec![ + Field::new("x", DataType::Int32, true), + Field::new("y", DataType::Utf8, true), + ] + .into(), + ), + true, + )], + ); + + let adapter = DefaultPhysicalExprAdapterFactory + .create(Arc::clone(&logical_schema), physical_schema) + .unwrap(); + let expr = Arc::new( + ScalarFunctionExpr::try_new( + Arc::new(datafusion_expr::ScalarUDF::from(GetFieldFunc::new())), + vec![ + Arc::new(Column::new("s", 0)), + Arc::new(Literal::new(ScalarValue::from("inner"))), + Arc::new(Literal::new(ScalarValue::from("y"))), + ], + &logical_schema, + Arc::new(datafusion_common::config::ConfigOptions::default()), + ) + .unwrap(), + ) as Arc; + + let rewritten = adapter.rewrite(expr).unwrap(); + + let literal = rewritten + .downcast_ref::() + .expect("Expected a null literal"); + assert_eq!(*literal.value(), ScalarValue::Utf8(None)); + } + + /// Accessing a field the file does not have yields a typed null literal. + #[test] + fn test_narrow_struct_cast_missing_field() { + let (logical_schema, physical_schema) = struct_schemas( + vec![Field::new("x", DataType::Int32, true)], + vec![ + Field::new("x", DataType::Int32, true), + Field::new("y", DataType::Utf8, true), + ], + ); + + let adapter = DefaultPhysicalExprAdapterFactory + .create(Arc::clone(&logical_schema), physical_schema) + .unwrap(); + let rewritten = adapter + .rewrite(get_field_expr(&logical_schema, "s", "y")) + .unwrap(); + + let literal = rewritten + .downcast_ref::() + .expect("Expected a null literal"); + assert_eq!(*literal.value(), ScalarValue::Utf8(None)); + } + + /// `get_field` on a Map column is a runtime key lookup, not a schema-level + /// field access, so the map value must keep its cast. + #[test] + fn test_map_field_access_keeps_cast() { + let map_type = |value_type: DataType| { + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("keys", DataType::Utf8, false), + Field::new("values", value_type, true), + ] + .into(), + ), + false, + )), + false, + ) + }; + let physical_schema = Arc::new(Schema::new(vec![Field::new( + "s", + map_type(DataType::Int32), + true, + )])); + let logical_schema = Arc::new(Schema::new(vec![Field::new( + "s", + map_type(DataType::Int64), + true, + )])); + + let adapter = DefaultPhysicalExprAdapterFactory + .create(Arc::clone(&logical_schema), physical_schema) + .unwrap(); + let rewritten = adapter + .rewrite(get_field_expr(&logical_schema, "s", "k")) + .unwrap(); + + let get_field = rewritten + .downcast_ref::() + .expect("Expected the get_field to be preserved"); + assert!( + get_field.args()[0].downcast_ref::().is_some(), + "map columns must keep the whole-column cast, got: {rewritten}" + ); + } + // ============================================================================ // BatchAdapterFactory and BatchAdapter tests // ============================================================================ diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt index cb3be93191fb2..f7d23001fcf50 100644 --- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt +++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt @@ -890,6 +890,119 @@ set datafusion.execution.parquet.pushdown_filters = false; statement ok DROP TABLE t_struct_filter; +########## +# Filters on struct fields, with pushdown enabled, where the declared table +# schema differs from the physical file schema and the scan therefore has to +# adapt the struct column. +# +# See https://github.com/apache/datafusion/issues/24109. +########## + +statement ok +set datafusion.execution.parquet.pushdown_filters = true; + +statement ok +COPY ( + SELECT + column1 as id, + named_struct('x', arrow_cast(column2, 'Int32')) as s + FROM VALUES (1, 100), (2, 200), (3, 300) +) TO 'test_files/scratch/parquet_filter_pushdown/struct_schema_cast.parquet' +STORED AS PARQUET; + +# `x` is stored as Int32 in the file but declared as BIGINT here, which forces +# the scan to adapt the struct column. +statement ok +CREATE EXTERNAL TABLE t_struct_schema_cast (id BIGINT, s STRUCT) +STORED AS PARQUET +LOCATION 'test_files/scratch/parquet_filter_pushdown/struct_schema_cast.parquet'; + +query II +SELECT id, s['x'] FROM t_struct_schema_cast WHERE s['x'] = 200; +---- +2 200 + +# Conjunction of a struct-field filter and a primitive filter. +query II +SELECT id, s['x'] FROM t_struct_schema_cast WHERE s['x'] > 100 AND id > 2; +---- +3 300 + +# Control: the same file read through a schema that matches it exactly, so no +# cast is inserted. This path has always worked. +statement ok +CREATE EXTERNAL TABLE t_struct_no_schema_cast (id BIGINT, s STRUCT) +STORED AS PARQUET +LOCATION 'test_files/scratch/parquet_filter_pushdown/struct_schema_cast.parquet'; + +query II +SELECT id, s['x'] FROM t_struct_no_schema_cast WHERE s['x'] = 200; +---- +2 200 + +query II +SELECT id, s['x'] FROM t_struct_no_schema_cast WHERE s['x'] > 100 AND id > 2; +---- +3 300 + +# Declaring a field the file does not have must not disturb the fields it does +# have. +statement ok +CREATE EXTERNAL TABLE t_struct_missing_field (id BIGINT, s STRUCT) +STORED AS PARQUET +LOCATION 'test_files/scratch/parquet_filter_pushdown/struct_schema_cast.parquet'; + +query II +SELECT id, s['x'] FROM t_struct_missing_field WHERE s['x'] = 200; +---- +2 200 + +query II +SELECT id, s['x'] FROM t_struct_missing_field WHERE s['x'] > 100 AND id > 2; +---- +3 300 + +# The absent field itself reads as null, so it matches nothing. +query II +SELECT id, s['x'] FROM t_struct_missing_field WHERE s['missing'] = 200; +---- + +# Same, for a nested field path. +statement ok +COPY ( + SELECT + column1 as id, + named_struct('inner', named_struct('x', arrow_cast(column2, 'Int32'))) as s + FROM VALUES (1, 100), (2, 200), (3, 300) +) TO 'test_files/scratch/parquet_filter_pushdown/struct_nested_schema_cast.parquet' +STORED AS PARQUET; + +statement ok +CREATE EXTERNAL TABLE t_struct_nested_schema_cast (id BIGINT, s STRUCT>) +STORED AS PARQUET +LOCATION 'test_files/scratch/parquet_filter_pushdown/struct_nested_schema_cast.parquet'; + +query II +SELECT id, s['inner']['x'] FROM t_struct_nested_schema_cast WHERE s['inner']['x'] = 200; +---- +2 200 + +# Clean up +statement ok +set datafusion.execution.parquet.pushdown_filters = false; + +statement ok +DROP TABLE t_struct_schema_cast; + +statement ok +DROP TABLE t_struct_no_schema_cast; + +statement ok +DROP TABLE t_struct_missing_field; + +statement ok +DROP TABLE t_struct_nested_schema_cast; + ########## # Regression test for https://github.com/apache/datafusion/issues/20937 # diff --git a/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt b/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt index d936a89beb9f7..4a2afc18ab72f 100644 --- a/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt +++ b/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt @@ -125,17 +125,13 @@ explain analyze select s from full_schema; ---- Plan with Metrics DataSourceExec: metrics=[output_rows=3, bytes_scanned=219] -# `get_field` on a schema-narrowed struct becomes `get_field(CAST(s), 'x')`; -# the read clips to the cast target (every field the *narrow* schema -# declares), not further down to just `x`. The fair "nothing was clipped" -# baseline is therefore reading every physical leaf of `s` -# (`select s from full_schema` above), not the same `get_field` query against -# `full_schema` -- that one needs no cast at all and takes `get_field`'s own, -# more precise, single-leaf pushdown path. +# Selecting a single field of `s` reads fewer bytes than selecting `s` itself +# (above): the read clips down to `x` rather than to every field the narrow +# schema declares. query TT explain analyze select s['x'] from narrow; ---- -Plan with Metrics DataSourceExec: metrics=[output_rows=3, bytes_scanned=146] +Plan with Metrics DataSourceExec: metrics=[output_rows=3, bytes_scanned=75] # Mixed access -- the whole (narrowed) column and a subfield of it -- still # reads only the narrow schema's leaves. From 7809b9b8c8cb121522aecf79b97059b3336e6097 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 20 Aug 2026 15:04:05 -0400 Subject: [PATCH 2/2] test: adapt mixed-access nested pruning expectation for branch-55 branch-55 does not have #24130/#24315, which taught nested schema pruning to union the leaves needed by mixed whole-column + field-access reads. Without that, `select s, s['y'] from narrow` falls back to reading every physical leaf instead of clipping to the narrow schema, so bytes_scanned is 219 (matching the unclipped full_schema read) rather than 146. --- .../test_files/parquet_nested_schema_pruning.slt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt b/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt index 4a2afc18ab72f..c5af117403c13 100644 --- a/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt +++ b/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt @@ -133,12 +133,14 @@ explain analyze select s['x'] from narrow; ---- Plan with Metrics DataSourceExec: metrics=[output_rows=3, bytes_scanned=75] -# Mixed access -- the whole (narrowed) column and a subfield of it -- still -# reads only the narrow schema's leaves. +# Mixed access -- the whole (narrowed) column and a subfield of it. A root +# carrying both access kinds falls back to reading every physical leaf, so +# this costs the same as the unclipped full_schema read below rather than +# clipping to the narrow schema's leaves. query TT explain analyze select s, s['y'] from narrow; ---- -Plan with Metrics DataSourceExec: metrics=[output_rows=3, bytes_scanned=146] +Plan with Metrics DataSourceExec: metrics=[output_rows=3, bytes_scanned=219] query TT explain analyze select s, s['y'] from full_schema;