fix: account for empty scalar subqueries in nullability

datafusion/expr/src/expr_schema.rs

@@ -369,9 +369,9 @@ impl ExprSchemable for Expr {
 
                 Ok(expr_nullable | subquery_nullable)
             }
-            Expr::ScalarSubquery(subquery) => {
-                Ok(subquery.subquery.schema().field(0).is_nullable())
-            }
+            // A scalar subquery may return no rows, in which case it evaluates to NULL
+            // regardless of the nullability of its projected field.
+            Expr::ScalarSubquery(_) => Ok(true),
             Expr::BinaryExpr(BinaryExpr { left, right, .. }) => {
                 Ok(left.nullable(input_schema)? || right.nullable(input_schema)?)
             }
@@ -517,9 +517,15 @@ impl ExprSchemable for Expr {
             | Expr::Exists { .. } => {
                 Ok(Arc::new(Field::new(&schema_name, DataType::Boolean, false)))
             }
-            Expr::ScalarSubquery(subquery) => {
-                Ok(Arc::clone(&subquery.subquery.schema().fields()[0]))
-            }
+            Expr::ScalarSubquery(subquery) => Ok(Arc::new(
+                subquery
+                    .subquery
+                    .schema()
+                    .field(0)
+                    .as_ref()
+                    .clone()
+                    .with_nullable(true),
+            )),
             Expr::BinaryExpr(BinaryExpr { left, right, op }) => {
                 let (left_field, right_field) =
                     (left.to_field(schema)?.1, right.to_field(schema)?.1);
@@ -800,7 +806,7 @@ mod tests {
     use crate::logical_plan::builder::LogicalTableSource;
     use crate::{
         LogicalPlanBuilder, and, col, in_subquery, lit, not, or,
-        out_ref_col_with_metadata, when,
+        out_ref_col_with_metadata, scalar_subquery, when,
     };
 
     use arrow::datatypes::Schema;
@@ -1268,6 +1274,23 @@ mod tests {
         );
     }
 
+    #[test]
+    fn scalar_subquery_is_nullable_with_non_nullable_output() {
+        let subquery = LogicalPlanBuilder::empty(false)
+            .project(vec![lit(1)])
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!subquery.schema().field(0).is_nullable());
+
+        let expr = scalar_subquery(Arc::new(subquery));
+        assert!(expr.nullable(&MockExprSchema::new()).unwrap());
+
+        let field = expr.to_field(&MockExprSchema::new()).unwrap().1;
+        assert_eq!(field.data_type(), &DataType::Int32);
+        assert!(field.is_nullable());
+    }
+
     #[test]
     fn test_scalar_variable() {
         let mut meta = HashMap::new();

datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs

@@ -3788,6 +3788,21 @@ mod tests {
         );
     }
 
+    #[test]
+    fn simplify_scalar_subquery_is_null() {
+        let subquery = LogicalPlanBuilder::empty(false)
+            .project(vec![lit(1)])
+            .unwrap()
+            .build()
+            .unwrap();
+        let scalar_subquery = scalar_subquery(Arc::new(subquery));
+
+        assert_eq!(
+            simplify(scalar_subquery.clone().is_null()),
+            scalar_subquery.is_null()
+        );
+    }
+
     #[test]
     fn simplify_expr_is_unknown() {
         assert_eq!(simplify(col("c2").is_unknown()), col("c2").is_unknown(),);

datafusion/physical-expr/src/planner.rs

@@ -536,10 +536,10 @@ pub fn create_physical_expr(
                         );
                     }
                     let dt = schema.field(0).data_type().clone();
-                    let nullable = schema.field(0).is_nullable();
                     Ok(Arc::new(ScalarSubqueryExpr::new(
                         dt,
-                        nullable,
+                        // A scalar subquery may return no rows and evaluate to NULL.
+                        true,
                         index,
                         planning_ctx.results().clone(),
                     )))

datafusion/sqllogictest/test_files/subquery.slt

@@ -2010,6 +2010,24 @@ SELECT (SELECT v FROM sq_empty);
 ----
 NULL
 
+# A zero-row scalar subquery is nullable even when its projected expression is not.
+query I
+SELECT (SELECT 1 WHERE FALSE);
+----
+NULL
+
+# Its nullability must prevent SimplifyExpressions from folding IS NULL to false.
+query B
+SELECT (SELECT 1 WHERE FALSE) IS NULL;
+----
+true
+
+# The same holds for the opposite fold direction.
+query B
+SELECT (SELECT 1 WHERE FALSE) IS NOT NULL;
+----
+false
+
 # Scalar subquery returning zero rows in arithmetic → NULL propagation
 query I
 SELECT x + (SELECT v FROM sq_empty) FROM sq_main;
@@ -2350,6 +2368,21 @@ SELECT (SELECT v FROM sq_empty);
 ----
 NULL
 
+# A zero-row scalar subquery is nullable even when its projected expression is
+# not. The rewrite to a left join already produces a nullable column here, so
+# this pins both paths to the same result.
+query I
+SELECT (SELECT 1 WHERE FALSE);
+----
+NULL
+
+# SimplifyExpressions runs before ScalarSubqueryToJoin, so this fold is still
+# governed by Expr::ScalarSubquery nullability on this path.
+query B
+SELECT (SELECT 1 WHERE FALSE) IS NULL;
+----
+true
+
 # Scalar subquery returning zero rows in arithmetic → NULL propagation
 query I
 SELECT x + (SELECT v FROM sq_empty) FROM sq_main;