MDEV-41650: COALESCE with typeless NULL changes hex literal Boolean evaluation

mysql-test/main/mdev_41650.result

@@ -0,0 +1,30 @@
+#
+# MDEV-41650: COALESCE with typeless NULL changes hex literal Boolean evaluation
+#
+SELECT (0x30 XOR 0) AS bare;
+bare
+1
+SELECT (COALESCE(NULL, 0x30) XOR 0) AS coalesce_hex;
+coalesce_hex
+1
+SELECT COALESCE('0', 5) XOR 0 AS coalesce_str_int;
+coalesce_str_int
+0
+SELECT COALESCE(0.5, 'abc') XOR 0 AS coalesce_float_str;
+coalesce_float_str
+1
+SELECT COALESCE(NULL, NULL) IS NULL AS coalesce_nulls;
+coalesce_nulls
+1
+SELECT COALESCE(0x30, NULL) XOR 0 AS coalesce_hex_first;
+coalesce_hex_first
+1
+SELECT COALESCE(NULL, COALESCE(NULL, 0x30)) XOR 0 AS coalesce_nested;
+coalesce_nested
+1
+SELECT IFNULL(NULL, 0x30) XOR 0 AS ifnull_hex;
+ifnull_hex
+0
+SELECT CASE WHEN NULL IS NULL THEN 0x30 END XOR 0 AS case_hex;
+case_hex
+0

mysql-test/main/mdev_41650.test

@@ -0,0 +1,20 @@
+--echo #
+--echo # MDEV-41650: COALESCE with typeless NULL changes hex literal Boolean evaluation
+--echo #
+
+# Bare hex literal is TRUE
+SELECT (0x30 XOR 0) AS bare;
+
+# Wrapped hex literal should also be TRUE
+SELECT (COALESCE(NULL, 0x30) XOR 0) AS coalesce_hex;
+
+# Control cases for mixed types (Ensure our fix didn't break these)
+SELECT COALESCE('0', 5) XOR 0 AS coalesce_str_int;
+SELECT COALESCE(0.5, 'abc') XOR 0 AS coalesce_float_str;
+SELECT COALESCE(NULL, NULL) IS NULL AS coalesce_nulls;
+SELECT COALESCE(0x30, NULL) XOR 0 AS coalesce_hex_first;
+SELECT COALESCE(NULL, COALESCE(NULL, 0x30)) XOR 0 AS coalesce_nested;
+
+# The bug also affects IFNULL and CASE (Still broken in this PR)
+SELECT IFNULL(NULL, 0x30) XOR 0 AS ifnull_hex;
+SELECT CASE WHEN NULL IS NULL THEN 0x30 END XOR 0 AS case_hex;

sql/item_cmpfunc.cc

@@ -3643,6 +3643,20 @@ longlong Item_func_coalesce::int_op()
   return 0;
 }
 
+bool Item_func_coalesce::val_bool()
+{
+  DBUG_ASSERT(fixed());
+  null_value= 0;
+  for (uint i= 0; i < arg_count; i++)
+  {
+    bool res= args[i]->val_bool();
+    if (!args[i]->null_value)
+      return res;
+  }
+  null_value= 1;
+  return 0;
+}
+
 double Item_func_coalesce::real_op()
 {
   DBUG_ASSERT(fixed());

sql/item_cmpfunc.h

@@ -1256,6 +1256,7 @@ class Item_func_coalesce :public Item_func_case_expression
     Item_func_case_expression(thd, list) {}
   double real_op() override;
   longlong int_op() override;
+  bool val_bool() override;
   String *str_op(String *) override;
   my_decimal *decimal_op(my_decimal *) override;
   bool date_op(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override;