From 091dc8a1b6327b90d06c82e5a248feb708c95cf2 Mon Sep 17 00:00:00 2001 From: Fredrik Fornwall Date: Thu, 20 Aug 2026 12:25:30 +0200 Subject: [PATCH 1/2] fix: account for empty scalar subqueries in nullability Signed-off-by: Fredrik Fornwall --- datafusion/expr/src/expr_schema.rs | 37 +++++++++++++++---- .../simplify_expressions/expr_simplifier.rs | 15 ++++++++ datafusion/physical-expr/src/planner.rs | 4 +- .../sqllogictest/test_files/subquery.slt | 12 ++++++ 4 files changed, 59 insertions(+), 9 deletions(-) diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rs index ee70e90153e38..8a9d0b8937cda 100644 --- a/datafusion/expr/src/expr_schema.rs +++ b/datafusion/expr/src/expr_schema.rs @@ -369,9 +369,9 @@ impl ExprSchemable for Expr { Ok(expr_nullable | subquery_nullable) } - Expr::ScalarSubquery(subquery) => { - Ok(subquery.subquery.schema().field(0).is_nullable()) - } + // A scalar subquery may return no rows, in which case it evaluates to NULL + // regardless of the nullability of its projected field. + Expr::ScalarSubquery(_) => Ok(true), Expr::BinaryExpr(BinaryExpr { left, right, .. }) => { Ok(left.nullable(input_schema)? || right.nullable(input_schema)?) } @@ -517,9 +517,15 @@ impl ExprSchemable for Expr { | Expr::Exists { .. } => { Ok(Arc::new(Field::new(&schema_name, DataType::Boolean, false))) } - Expr::ScalarSubquery(subquery) => { - Ok(Arc::clone(&subquery.subquery.schema().fields()[0])) - } + Expr::ScalarSubquery(subquery) => Ok(Arc::new( + subquery + .subquery + .schema() + .field(0) + .as_ref() + .clone() + .with_nullable(true), + )), Expr::BinaryExpr(BinaryExpr { left, right, op }) => { let (left_field, right_field) = (left.to_field(schema)?.1, right.to_field(schema)?.1); @@ -800,7 +806,7 @@ mod tests { use crate::logical_plan::builder::LogicalTableSource; use crate::{ LogicalPlanBuilder, and, col, in_subquery, lit, not, or, - out_ref_col_with_metadata, when, + out_ref_col_with_metadata, scalar_subquery, when, }; use arrow::datatypes::Schema; @@ -1268,6 +1274,23 @@ mod tests { ); } + #[test] + fn scalar_subquery_is_nullable_with_non_nullable_output() { + let subquery = LogicalPlanBuilder::empty(false) + .project(vec![lit(1)]) + .unwrap() + .build() + .unwrap(); + assert!(!subquery.schema().field(0).is_nullable()); + + let expr = scalar_subquery(Arc::new(subquery)); + assert!(expr.nullable(&MockExprSchema::new()).unwrap()); + + let field = expr.to_field(&MockExprSchema::new()).unwrap().1; + assert_eq!(field.data_type(), &DataType::Int32); + assert!(field.is_nullable()); + } + #[test] fn test_scalar_variable() { let mut meta = HashMap::new(); diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index b7e9624946e79..5beb54b5605f4 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -3788,6 +3788,21 @@ mod tests { ); } + #[test] + fn simplify_scalar_subquery_is_null() { + let subquery = LogicalPlanBuilder::empty(false) + .project(vec![lit(1)]) + .unwrap() + .build() + .unwrap(); + let scalar_subquery = scalar_subquery(Arc::new(subquery)); + + assert_eq!( + simplify(scalar_subquery.clone().is_null()), + scalar_subquery.is_null() + ); + } + #[test] fn simplify_expr_is_unknown() { assert_eq!(simplify(col("c2").is_unknown()), col("c2").is_unknown(),); diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs index f80d1b15bdc59..09b2e5ce04727 100644 --- a/datafusion/physical-expr/src/planner.rs +++ b/datafusion/physical-expr/src/planner.rs @@ -536,10 +536,10 @@ pub fn create_physical_expr( ); } let dt = schema.field(0).data_type().clone(); - let nullable = schema.field(0).is_nullable(); Ok(Arc::new(ScalarSubqueryExpr::new( dt, - nullable, + // A scalar subquery may return no rows and evaluate to NULL. + true, index, planning_ctx.results().clone(), ))) diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index c38fb4c7ee24b..f4f3f95ef2ae9 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -2010,6 +2010,18 @@ SELECT (SELECT v FROM sq_empty); ---- NULL +# A zero-row scalar subquery is nullable even when its projected expression is not. +query I +SELECT (SELECT 1 WHERE FALSE); +---- +NULL + +# Its nullability must prevent SimplifyExpressions from folding IS NULL to false. +query B +SELECT (SELECT 1 WHERE FALSE) IS NULL; +---- +true + # Scalar subquery returning zero rows in arithmetic → NULL propagation query I SELECT x + (SELECT v FROM sq_empty) FROM sq_main; From c74d1497cb0b76f529cc6422f47e5d0f4754dce1 Mon Sep 17 00:00:00 2001 From: Fredrik Fornwall Date: Thu, 20 Aug 2026 13:38:17 +0200 Subject: [PATCH 2/2] Add more tests --- .../sqllogictest/test_files/subquery.slt | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index f4f3f95ef2ae9..65760886d1156 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -2022,6 +2022,12 @@ SELECT (SELECT 1 WHERE FALSE) IS NULL; ---- true +# The same holds for the opposite fold direction. +query B +SELECT (SELECT 1 WHERE FALSE) IS NOT NULL; +---- +false + # Scalar subquery returning zero rows in arithmetic → NULL propagation query I SELECT x + (SELECT v FROM sq_empty) FROM sq_main; @@ -2362,6 +2368,21 @@ SELECT (SELECT v FROM sq_empty); ---- NULL +# A zero-row scalar subquery is nullable even when its projected expression is +# not. The rewrite to a left join already produces a nullable column here, so +# this pins both paths to the same result. +query I +SELECT (SELECT 1 WHERE FALSE); +---- +NULL + +# SimplifyExpressions runs before ScalarSubqueryToJoin, so this fold is still +# governed by Expr::ScalarSubquery nullability on this path. +query B +SELECT (SELECT 1 WHERE FALSE) IS NULL; +---- +true + # Scalar subquery returning zero rows in arithmetic → NULL propagation query I SELECT x + (SELECT v FROM sq_empty) FROM sq_main;