Remove stored Union Nullability

vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs

@@ -289,7 +289,7 @@ fn supports_uncompressed_size_in_bytes(dtype: &DType) -> bool {
         DType::Struct(fields, _) => fields
             .fields()
             .all(|field| supports_uncompressed_size_in_bytes(&field)),
-        DType::Union(variants, _) => variants
+        DType::Union(variants) => variants
             .variants()
             .all(|variant| supports_uncompressed_size_in_bytes(&variant)),
         DType::Variant(_) => false,

vortex-array/src/dtype/coercion.rs

@@ -79,6 +79,25 @@ impl DType {
     /// The core primitive — what type can hold both `self` and `other`?
     /// Returns `None` if no common supertype exists.
     pub fn least_supertype(&self, other: &DType) -> Option<DType> {
+        match (self, other) {
+            (DType::Union(lhs), DType::Union(rhs)) => {
+                return (lhs == rhs).then(|| self.clone());
+            }
+            (DType::Null, DType::Union(variants)) => {
+                return variants
+                    .derived_nullability()
+                    .is_nullable()
+                    .then(|| other.clone());
+            }
+            (DType::Union(variants), DType::Null) => {
+                return variants
+                    .derived_nullability()
+                    .is_nullable()
+                    .then(|| self.clone());
+            }
+            _ => {}
+        }
+
         let union_null = self.nullability() | other.nullability();
 
         if let (
@@ -308,6 +327,7 @@ mod tests {
 
     use crate::dtype::DType;
     use crate::dtype::PType;
+    use crate::dtype::UnionVariants;
     use crate::dtype::decimal::DecimalDType;
     use crate::dtype::nullability::Nullability::NonNullable;
     use crate::dtype::nullability::Nullability::Nullable;
@@ -349,6 +369,57 @@ mod tests {
         );
     }
 
+    #[test]
+    fn least_supertype_union_requires_identical_variants() {
+        let nonnullable = DType::Union(
+            UnionVariants::new(
+                ["value"].into(),
+                vec![DType::Primitive(PType::I32, NonNullable)],
+            )
+            .unwrap(),
+        );
+        let nullable = DType::Union(
+            UnionVariants::new(
+                ["value"].into(),
+                vec![DType::Primitive(PType::I32, Nullable)],
+            )
+            .unwrap(),
+        );
+
+        assert_eq!(
+            nonnullable.least_supertype(&nonnullable),
+            Some(nonnullable.clone())
+        );
+        assert!(nonnullable.least_supertype(&nullable).is_none());
+        assert!(nullable.least_supertype(&nonnullable).is_none());
+    }
+
+    #[test]
+    fn least_supertype_null_requires_nullable_union() {
+        let nonnullable = DType::Union(
+            UnionVariants::new(
+                ["value"].into(),
+                vec![DType::Primitive(PType::I32, NonNullable)],
+            )
+            .unwrap(),
+        );
+        let nullable = DType::Union(
+            UnionVariants::new(
+                ["value"].into(),
+                vec![DType::Primitive(PType::I32, Nullable)],
+            )
+            .unwrap(),
+        );
+
+        assert!(DType::Null.least_supertype(&nonnullable).is_none());
+        assert!(nonnullable.least_supertype(&DType::Null).is_none());
+        assert_eq!(
+            DType::Null.least_supertype(&nullable),
+            Some(nullable.clone())
+        );
+        assert_eq!(nullable.least_supertype(&DType::Null), Some(nullable));
+    }
+
     #[test]
     fn least_supertype_unsigned_widening() {
         let u8_nn = DType::Primitive(PType::U8, NonNullable);

vortex-array/src/dtype/dtype_impl.rs

@@ -64,8 +64,8 @@ impl DType {
             | List(_, null)
             | FixedSizeList(_, _, null)
             | Struct(_, null)
-            | Union(_, null)
             | Variant(null) => matches!(null, Nullability::Nullable),
+            Union(variants) => variants.derived_nullability().is_nullable(),
             Extension(ext_dtype) => ext_dtype.storage_dtype().is_nullable(),
         }
     }
@@ -80,7 +80,10 @@ impl DType {
         self.with_nullability(Nullability::Nullable)
     }
 
-    /// Get a new DType with the given nullability (but otherwise the same as `self`)
+    /// Get a new DType with the given nullability (but otherwise the same as `self`).
+    ///
+    /// [`DType::Null`] and [`DType::Union`] have intrinsic nullability and are returned unchanged.
+    /// To change a union's nullability, construct different [`UnionVariants`].
     pub fn with_nullability(&self, nullability: Nullability) -> Self {
         match self {
             Null => Null,
@@ -92,7 +95,7 @@ impl DType {
             List(edt, _) => List(Arc::clone(edt), nullability),
             FixedSizeList(edt, size, _) => FixedSizeList(Arc::clone(edt), *size, nullability),
             Struct(sf, _) => Struct(sf.clone(), nullability),
-            Union(vs, _) => Union(vs.clone(), nullability),
+            Union(vs) => Union(vs.clone()),
             Variant(_) => Variant(nullability),
             Extension(ext) => Extension(ext.with_nullability(nullability)),
         }
@@ -124,7 +127,7 @@ impl DType {
                         .zip_eq(rhs_dtype.fields())
                         .all(|(l, r)| l.eq_ignore_nullability(&r)))
             }
-            (Union(lhs, _), Union(rhs, _)) => {
+            (Union(lhs), Union(rhs)) => {
                 // Equal `names` implies equal length by FieldNames equality.
                 lhs.names() == rhs.names()
                     && lhs.type_ids() == rhs.type_ids()
@@ -436,20 +439,12 @@ impl DType {
 
     /// Get the [`UnionVariants`] if `self` is a [`DType::Union`], otherwise `None`.
     pub fn as_union_variants_opt(&self) -> Option<&UnionVariants> {
-        if let Union(uv, _) = self {
-            Some(uv)
-        } else {
-            None
-        }
+        if let Union(uv) = self { Some(uv) } else { None }
     }
 
     /// Owned version of [Self::as_union_variants_opt].
     pub fn into_union_variants_opt(self) -> Option<UnionVariants> {
-        if let Union(uv, _) = self {
-            Some(uv)
-        } else {
-            None
-        }
+        if let Union(uv) = self { Some(uv) } else { None }
     }
 
     /// Downcast a `DType` to an `ExtDType`
@@ -503,7 +498,7 @@ impl Display for DType {
                     .map(|(field_null, dt)| format!("{field_null}={dt}"))
                     .join(", "),
             ),
-            Union(uv, null) => write!(f, "union({uv}){null}"),
+            Union(uv) => write!(f, "union({uv}){}", uv.derived_nullability()),
             Variant(null) => write!(f, "variant{null}"),
             Extension(ext) => write!(f, "{}", ext),
         }

vortex-array/src/dtype/mod.rs

@@ -114,8 +114,13 @@ pub enum DType {
     /// A `Union` is composed of one or more **variants**, each with a name and a `DType`. A per-row
     /// `i8` tag selects which variant is "live" at that row.
     ///
+    /// Unlike other nested types, a union has no independent outer nullability. Its nullability is
+    /// derived at runtime from its variants: a union is nullable when any variant can contain null.
+    /// A concrete union array has no parent validity bitmap; a row's validity is the validity of
+    /// its selected child.
+    ///
     /// See [`UnionVariants`] for the type-tag conventions and accessors.
-    Union(UnionVariants, Nullability),
+    Union(UnionVariants),
 
     /// Dynamically typed values stored as Vortex scalars.
     ///
@@ -148,7 +153,7 @@ impl PartialEq for DType {
             // StructFields handles its own Arc::ptr_eq in its PartialEq impl.
             (Self::Struct(a, na), Self::Struct(b, nb)) => na == nb && a == b,
             // UnionVariants handles its own Arc::ptr_eq in its PartialEq impl.
-            (Self::Union(a, na), Self::Union(b, nb)) => na == nb && a == b,
+            (Self::Union(a), Self::Union(b)) => a == b,
             (Self::Variant(a), Self::Variant(b)) => a == b,
             (Self::Extension(a), Self::Extension(b)) => a == b,
             // Every variant is listed in the first position so that adding a new

vortex-array/src/dtype/serde/flatbuffers.rs

@@ -11,7 +11,6 @@ use itertools::Itertools;
 use vortex_error::VortexError;
 use vortex_error::VortexResult;
 use vortex_error::vortex_bail;
-use vortex_error::vortex_ensure;
 use vortex_error::vortex_err;
 use vortex_flatbuffers::FlatBuffer;
 use vortex_flatbuffers::FlatBufferRoot;
@@ -22,7 +21,6 @@ use vortex_session::VortexSession;
 use crate::dtype::DType;
 use crate::dtype::DecimalDType;
 use crate::dtype::FieldDType;
-use crate::dtype::Nullability;
 use crate::dtype::PType;
 use crate::dtype::StructFields;
 use crate::dtype::UnionVariants;
@@ -236,15 +234,7 @@ impl TryFrom<ViewedDType> for DType {
                     .ok_or_else(|| vortex_err!("failed to parse union from flatbuffer"))?;
                 let variants =
                     UnionVariants::from_fb(fb_union, vfdt.buffer().clone(), vfdt.session.clone())?;
-
-                let nullability: Nullability = fb_union.nullable().into();
-                vortex_ensure!(
-                    variants.nullability_constraints_satisfied(nullability),
-                    "Union nullability constraint not satisfied: nullability={:?}",
-                    nullability
-                );
-
-                Ok(Self::Union(variants, nullability))
+                Ok(Self::Union(variants))
             }
             fb::Type::Variant => {
                 let fb_variant = fb
@@ -390,7 +380,7 @@ impl WriteFlatBuffer for DType {
                 )
                 .as_union_value()
             }
-            Self::Union(uv, n) => {
+            Self::Union(uv) => {
                 let names = uv
                     .names()
                     .iter()
@@ -412,7 +402,6 @@ impl WriteFlatBuffer for DType {
                         names,
                         dtypes,
                         type_ids,
-                        nullable: (*n).into(),
                     },
                 )
                 .as_union_value()
@@ -583,7 +572,6 @@ mod test {
                 ],
             )
             .unwrap(),
-            Nullability::NonNullable,
         )
     }
 
@@ -593,14 +581,13 @@ mod test {
     }
 
     #[test]
-    fn test_union_round_trip_flatbuffer_with_nullability() {
+    fn test_union_round_trip_flatbuffer_with_nullable_variant() {
         let dtype = DType::Union(
             UnionVariants::new(
                 ["null_variant", "str"].into(),
                 vec![DType::Null, DType::Utf8(Nullability::NonNullable)],
             )
             .unwrap(),
-            Nullability::Nullable,
         );
         roundtrip_dtype(dtype);
     }
@@ -618,7 +605,6 @@ mod test {
                 vec![0, 5, 7],
             )
             .unwrap(),
-            Nullability::NonNullable,
         );
 
         let bytes = dtype.write_flatbuffer_bytes().unwrap();
@@ -631,7 +617,7 @@ mod test {
 
         let deserialized = DType::try_from(view).unwrap();
         assert_eq!(dtype, deserialized);
-        let DType::Union(uv, _) = &deserialized else {
+        let DType::Union(uv) = &deserialized else {
             panic!("Expected Union");
         };
         assert_eq!(uv.type_ids(), &[0, 5, 7]);
@@ -654,7 +640,6 @@ mod test {
                 vec![DType::Utf8(Nullability::NonNullable), struct_with_union],
             )
             .unwrap(),
-            Nullability::NonNullable,
         );
 
         roundtrip_dtype(outer_union);
@@ -710,7 +695,6 @@ mod test {
                 names: Some(names),
                 dtypes: Some(dtypes),
                 type_ids: Some(type_ids),
-                nullable: false,
             },
         );
 

vortex-array/src/dtype/serde/mod.rs

@@ -24,6 +24,7 @@ mod test {
     use crate::dtype::DType;
     use crate::dtype::Nullability;
     use crate::dtype::PType;
+    use crate::dtype::UnionVariants;
     use crate::dtype::serde::DTypeSerde;
     use crate::dtype::test::SESSION;
 
@@ -181,4 +182,24 @@ mod test {
             .unwrap();
         assert_eq!(DType::Variant(Nullability::Nullable), deserialized);
     }
+
+    #[test]
+    fn test_serde_union_dtype_json_roundtrip() {
+        let dtype = DType::Union(
+            UnionVariants::new(["value"].into(), vec![DType::Utf8(Nullability::Nullable)]).unwrap(),
+        );
+
+        let json = serde_json::to_string(&dtype).unwrap();
+        assert_eq!(
+            json,
+            r#"{"Union":{"names":["value"],"dtypes":[{"Utf8":true}],"type_ids":[0]}}"#
+        );
+        let mut deserializer = serde_json::Deserializer::from_str(&json);
+        let deserialized: DType = DTypeSerde::<DType>::new(&SESSION)
+            .deserialize(&mut deserializer)
+            .unwrap();
+
+        assert_eq!(deserialized, dtype);
+        assert_eq!(deserialized.nullability(), Nullability::Nullable);
+    }
 }