From d23573ba614b7512d77e2b7132994c563f11082a Mon Sep 17 00:00:00 2001 From: David Linton Date: Fri, 28 Aug 2026 14:30:58 +0000 Subject: [PATCH] feat(ffi): add vx_scalar_new_extension for extension-typed literals The C API can build scan predicates against primitive, utf8, binary and decimal columns but not extension columns: no scalar constructor can express a value of an extension type, and the compare kernel (correctly) requires both sides of a comparison to share the extension dtype, so a bare storage-typed literal cannot stand in. For date/timestamp columns this leaves FFI consumers unable to push a time window into a scan. Adds vx_scalar_new_extension(dtype, storage, err): builds an extension-typed scalar from an extension dtype and a scalar of its storage dtype (compared ignoring nullability; value copied, arguments not consumed). Validation is delegated to Scalar::try_new, with explicit errors for a non-extension dtype and a storage-dtype mismatch. Header regenerated. Tests cover the date-over-i32 happy path and both rejection paths. Signed-off-by: David Linton --- vortex-ffi/cinclude/vortex.h | 20 ++++++++ vortex-ffi/src/scalar.rs | 90 ++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/vortex-ffi/cinclude/vortex.h b/vortex-ffi/cinclude/vortex.h index c7bc179e7ee..990f21d9cc7 100644 --- a/vortex-ffi/cinclude/vortex.h +++ b/vortex-ffi/cinclude/vortex.h @@ -1356,6 +1356,26 @@ vx_view vx_scalar_get_binary(const vx_scalar *scalar); */ vx_scalar *vx_scalar_new_null(const vx_dtype *dtype, vx_error **err); +/** + * Create an extension-typed scalar from its storage scalar. + * + * "dtype" must be an extension dtype (for example a date or timestamp dtype + * taken from a data source's schema, or built via vx_dtype_from_arrow_schema) + * and "storage" a scalar of that extension type's storage dtype, compared + * ignoring nullability. The storage value is copied; neither argument is + * consumed. + * + * This makes extension columns filterable through the C API: build the + * storage value with a primitive constructor (e.g. vx_scalar_new_i32 for a + * day-precision date), wrap it with the column's dtype, and pass the result + * to vx_expression_literal in a scan predicate. The compare kernel requires + * both sides of a comparison to share the extension dtype, so a bare + * storage-typed literal cannot be used directly. + * + * Returns NULL and sets "err" on error. + */ +vx_scalar *vx_scalar_new_extension(const vx_dtype *dtype, const vx_scalar *storage, vx_error **err); + /** * Create a decimal scalar from a signed i8 unscaled value. * diff --git a/vortex-ffi/src/scalar.rs b/vortex-ffi/src/scalar.rs index 7e45254f584..01492fdd8fa 100644 --- a/vortex-ffi/src/scalar.rs +++ b/vortex-ffi/src/scalar.rs @@ -226,6 +226,50 @@ pub unsafe extern "C-unwind" fn vx_scalar_new_null( }) } +/// Create an extension-typed scalar from its storage scalar. +/// +/// "dtype" must be an extension dtype (for example a date or timestamp dtype +/// taken from a data source's schema, or built via vx_dtype_from_arrow_schema) +/// and "storage" a scalar of that extension type's storage dtype, compared +/// ignoring nullability. The storage value is copied; neither argument is +/// consumed. +/// +/// This makes extension columns filterable through the C API: build the +/// storage value with a primitive constructor (e.g. vx_scalar_new_i32 for a +/// day-precision date), wrap it with the column's dtype, and pass the result +/// to vx_expression_literal in a scan predicate. The compare kernel requires +/// both sides of a comparison to share the extension dtype, so a bare +/// storage-typed literal cannot be used directly. +/// +/// Returns NULL and sets "err" on error. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_scalar_new_extension( + dtype: *const vx_dtype, + storage: *const vx_scalar, + err: *mut *mut vx_error, +) -> *mut vx_scalar { + try_or(err, ptr::null_mut(), || { + let dtype = vx_dtype::as_ref(dtype); + let storage = vx_scalar::as_ref(storage); + let DType::Extension(ext) = dtype else { + vortex_bail!( + "vx_scalar_new_extension: dtype {} is not an extension type", + dtype + ); + }; + vortex_ensure!( + storage.dtype().eq_ignore_nullability(ext.storage_dtype()), + "vx_scalar_new_extension: storage scalar dtype {} does not match extension storage dtype {}", + storage.dtype(), + ext.storage_dtype() + ); + Ok(vx_scalar::new(Scalar::try_new( + dtype.clone(), + storage.value().cloned(), + )?)) + }) +} + macro_rules! scalar_decimal { ($int:ident, $variant:ident) => { paste! { @@ -475,6 +519,7 @@ mod tests { use crate::string::vx_view; use crate::tests::assert_error; use crate::tests::assert_no_error; + use crate::vx_error_free; fn assert_scalar(ptr: *mut vx_scalar, expected: Scalar) { assert!(!ptr.is_null()); @@ -898,4 +943,49 @@ mod tests { vx_scalar_free(s); } } + + #[test] + fn test_scalar_new_extension() { + use vortex::array::extension::datetime::Date; + use vortex::array::extension::datetime::TimeUnit; + + unsafe { + let mut error = ptr::null_mut(); + + // A day-precision date (the Arrow date32 equivalent): an + // extension dtype over i32 storage. + let date_dtype = vx_dtype::new(DType::Extension( + Date::new(TimeUnit::Days, Nullability::NonNullable).erased(), + )); + let storage = vx_scalar_new_i32(20_000, false); + + let date = vx_scalar_new_extension(date_dtype, storage, &raw mut error); + assert_no_error(error); + assert!(!vx_scalar_is_null(date)); + let roundtrip = vx_scalar_dtype(date); + assert!(matches!(vx_dtype::as_ref(roundtrip), DType::Extension(_))); + vx_dtype_free(roundtrip); + vx_scalar_free(date); + + // Mismatched storage dtype is rejected. + let wrong = vx_scalar_new_i64(20_000, false); + let rejected = vx_scalar_new_extension(date_dtype, wrong, &raw mut error); + assert!(rejected.is_null()); + assert!(!error.is_null()); + vx_error_free(error); + error = ptr::null_mut(); + vx_scalar_free(wrong); + + // A non-extension dtype is rejected. + let bool_dtype = vx_dtype_new_bool(false); + let rejected = vx_scalar_new_extension(bool_dtype, storage, &raw mut error); + assert!(rejected.is_null()); + assert!(!error.is_null()); + vx_error_free(error); + + vx_scalar_free(storage); + vx_dtype_free(bool_dtype); + vx_dtype_free(date_dtype); + } + } }