From 3ad7d634025139de37aa6eff75f320a1d4cef155 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 9 Jul 2026 09:14:52 +0100 Subject: [PATCH 1/2] Remove arrow dependency from vortex-array Signed-off-by: Robert Kruszewski --- Cargo.toml | 2 + benchmarks/compress-bench/Cargo.toml | 1 + benchmarks/compress-bench/src/vortex.rs | 1 + benchmarks/datafusion-bench/Cargo.toml | 1 + benchmarks/datafusion-bench/src/main.rs | 1 + encodings/parquet-variant/Cargo.toml | 1 + encodings/parquet-variant/src/array.rs | 14 +- encodings/parquet-variant/src/arrow.rs | 16 +- encodings/parquet-variant/src/kernel.rs | 7 +- encodings/parquet-variant/src/lib.rs | 2 +- encodings/pco/Cargo.toml | 1 + encodings/pco/src/tests.rs | 2 +- encodings/runend/Cargo.toml | 4 - encodings/runend/src/array.rs | 4 +- encodings/runend/src/lib.rs | 4 +- encodings/runend/src/ops.rs | 5 +- encodings/sparse/Cargo.toml | 1 + encodings/sparse/src/canonical.rs | 2 +- vortex-array/Cargo.toml | 12 +- vortex-array/benches/kleene_bool.rs | 2 - .../src/aggregate_fn/fns/is_sorted/mod.rs | 19 +- vortex-array/src/arc_swap_map.rs | 14 +- .../src/arrays/constant/compute/fill_null.rs | 59 ++---- vortex-array/src/arrays/decimal/mod.rs | 14 -- vortex-array/src/canonical.rs | 169 ++---------------- vortex-array/src/dtype/mod.rs | 1 - vortex-array/src/expr/exprs.rs | 12 +- vortex-array/src/extension/uuid/mod.rs | 1 - vortex-array/src/lib.rs | 11 +- vortex-array/src/scalar/mod.rs | 1 - vortex-array/src/scalar_fn/fns/zip/mod.rs | 51 +++--- vortex-arrow/Cargo.toml | 46 +++++ .../benches/to_arrow.rs | 14 +- .../src/arrow => vortex-arrow/src}/convert.rs | 116 ++++++------ .../src/arrow => vortex-arrow/src}/datum.rs | 18 +- .../arrow.rs => vortex-arrow/src/dtype.rs | 118 ++++++------ .../src}/executor/bool.rs | 10 +- .../src}/executor/byte.rs | 40 ++--- .../src}/executor/byte_view.rs | 16 +- .../src}/executor/decimal.rs | 46 ++--- .../src}/executor/dictionary.rs | 51 +++--- .../src}/executor/fixed_size_list.rs | 14 +- .../src}/executor/list.rs | 63 +++---- .../src}/executor/list_view.rs | 44 ++--- .../src}/executor/mod.rs | 50 +++--- .../src}/executor/null.rs | 7 +- .../src}/executor/primitive.rs | 16 +- .../src}/executor/run_end.rs | 109 +++++------ .../src}/executor/struct_.rs | 63 +++---- .../src}/executor/temporal.rs | 24 +-- .../src}/executor/validity.rs | 2 +- .../src/arrow => vortex-arrow/src}/iter.rs | 10 +- .../arrow/mod.rs => vortex-arrow/src/lib.rs | 38 +++- .../arrow => vortex-arrow/src}/null_buffer.rs | 7 +- .../src/run_end_import.rs | 22 +-- .../arrow.rs => vortex-arrow/src/scalar.rs | 151 ++++++++-------- .../src/arrow => vortex-arrow/src}/session.rs | 95 +++++----- .../uuid/arrow.rs => vortex-arrow/src/uuid.rs | 48 ++--- vortex-arrow/tests/canonical.rs | 168 +++++++++++++++++ vortex-bench/Cargo.toml | 1 + vortex-bench/src/conversions.rs | 4 +- .../src/datasets/struct_list_of_ints.rs | 3 +- .../src/spatialbench/datagen/native.rs | 2 +- vortex-bench/src/tpch/tpchgen.rs | 4 +- vortex-cuda/Cargo.toml | 1 + vortex-cuda/src/arrow/mod.rs | 2 +- vortex-datafusion/Cargo.toml | 1 + vortex-datafusion/src/convert/exprs.rs | 4 +- vortex-datafusion/src/convert/scalars.rs | 2 +- vortex-datafusion/src/convert/schema.rs | 2 +- vortex-datafusion/src/lib.rs | 2 +- vortex-datafusion/src/persistent/format.rs | 2 +- vortex-datafusion/src/persistent/opener.rs | 4 +- vortex-datafusion/src/persistent/sink.rs | 2 +- vortex-datafusion/src/persistent/tests.rs | 4 +- vortex-datafusion/src/v2/source.rs | 2 +- vortex-ffi/Cargo.toml | 1 + vortex-ffi/src/array.rs | 2 +- vortex-ffi/src/dtype.rs | 3 +- vortex-ffi/src/scan.rs | 3 +- vortex-geo/Cargo.toml | 1 + vortex-geo/src/extension/linestring.rs | 16 +- vortex-geo/src/extension/mod.rs | 2 +- vortex-geo/src/extension/multilinestring.rs | 16 +- vortex-geo/src/extension/multipoint.rs | 16 +- vortex-geo/src/extension/multipolygon.rs | 16 +- vortex-geo/src/extension/point.rs | 16 +- vortex-geo/src/extension/polygon.rs | 16 +- vortex-geo/src/extension/wkb.rs | 14 +- vortex-geo/src/lib.rs | 2 +- vortex-geo/src/tests/linestring.rs | 2 +- vortex-geo/src/tests/multilinestring.rs | 2 +- vortex-geo/src/tests/multipoint.rs | 2 +- vortex-geo/src/tests/multipolygon.rs | 2 +- vortex-geo/src/tests/point.rs | 2 +- vortex-geo/src/tests/wkb.rs | 2 +- vortex-jni/Cargo.toml | 1 + vortex-jni/src/dtype.rs | 1 + vortex-jni/src/scan.rs | 3 +- vortex-jni/src/writer.rs | 2 +- vortex-json/Cargo.toml | 1 + vortex-json/src/arrow.rs | 16 +- vortex-json/src/lib.rs | 2 +- vortex-layout/Cargo.toml | 1 + vortex-layout/src/scan/arrow.rs | 4 +- vortex-python/Cargo.toml | 1 + vortex-python/src/arrays/from_arrow.rs | 4 +- vortex-python/src/arrays/mod.rs | 2 +- vortex-python/src/dataset.rs | 1 + vortex-python/src/dtype/mod.rs | 5 +- vortex-python/src/file.rs | 1 + vortex-python/src/io.rs | 4 +- vortex-python/src/iter/mod.rs | 3 +- vortex-tensor/Cargo.toml | 1 + vortex-tensor/src/lib.rs | 2 +- vortex-tensor/src/types/vector/arrow.rs | 24 +-- vortex-test/compat-gen/Cargo.toml | 1 + .../fixtures/arrays/datasets/clickbench.rs | 2 +- .../src/fixtures/arrays/datasets/tpch.rs | 2 +- vortex-test/e2e-cuda/src/lib.rs | 2 +- vortex-tui/Cargo.toml | 1 + vortex-tui/src/convert.rs | 4 +- vortex-web/crate/Cargo.toml | 1 + vortex-web/crate/src/wasm.rs | 2 +- vortex/Cargo.toml | 3 +- vortex/src/lib.rs | 13 +- 126 files changed, 1097 insertions(+), 1032 deletions(-) create mode 100644 vortex-arrow/Cargo.toml rename {vortex-array => vortex-arrow}/benches/to_arrow.rs (95%) rename {vortex-array/src/arrow => vortex-arrow/src}/convert.rs (95%) rename {vortex-array/src/arrow => vortex-arrow/src}/datum.rs (94%) rename vortex-array/src/dtype/arrow.rs => vortex-arrow/src/dtype.rs (89%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/bool.rs (83%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/byte.rs (91%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/byte_view.rs (87%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/decimal.rs (92%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/dictionary.rs (87%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/fixed_size_list.rs (86%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/list.rs (90%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/list_view.rs (89%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/mod.rs (88%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/null.rs (86%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/primitive.rs (81%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/run_end.rs (76%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/struct_.rs (89%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/temporal.rs (92%) rename {vortex-array/src/arrow => vortex-arrow/src}/executor/validity.rs (61%) rename {vortex-array/src/arrow => vortex-arrow/src}/iter.rs (86%) rename vortex-array/src/arrow/mod.rs => vortex-arrow/src/lib.rs (64%) rename {vortex-array/src/arrow => vortex-arrow/src}/null_buffer.rs (93%) rename encodings/runend/src/arrow.rs => vortex-arrow/src/run_end_import.rs (96%) rename vortex-array/src/scalar/arrow.rs => vortex-arrow/src/scalar.rs (83%) rename {vortex-array/src/arrow => vortex-arrow/src}/session.rs (94%) rename vortex-array/src/extension/uuid/arrow.rs => vortex-arrow/src/uuid.rs (85%) create mode 100644 vortex-arrow/tests/canonical.rs diff --git a/Cargo.toml b/Cargo.toml index e59c5c2c76c..12e91c7ef20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "vortex-io", "vortex-proto", "vortex-array", + "vortex-arrow", "vortex-row", "vortex-tensor", "vortex-json", @@ -285,6 +286,7 @@ zstd = { version = "0.13.3", default-features = false, features = [ vortex = { version = "0.1.0", path = "./vortex" } vortex-alp = { version = "0.1.0", path = "./encodings/alp", default-features = false } vortex-array = { version = "0.1.0", path = "./vortex-array", default-features = false } +vortex-arrow = { version = "0.1.0", path = "./vortex-arrow", default-features = false } vortex-btrblocks = { version = "0.1.0", path = "./vortex-btrblocks", default-features = false } vortex-buffer = { version = "0.1.0", path = "./vortex-buffer", default-features = false } vortex-bytebool = { version = "0.1.0", path = "./encodings/bytebool", default-features = false } diff --git a/benchmarks/compress-bench/Cargo.toml b/benchmarks/compress-bench/Cargo.toml index baff1daad74..d214af23511 100644 --- a/benchmarks/compress-bench/Cargo.toml +++ b/benchmarks/compress-bench/Cargo.toml @@ -30,6 +30,7 @@ regex = { workspace = true } tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } vortex = { workspace = true } +vortex-arrow = { workspace = true } vortex-bench = { workspace = true } [features] diff --git a/benchmarks/compress-bench/src/vortex.rs b/benchmarks/compress-bench/src/vortex.rs index 87416cdf924..fae0cc44189 100644 --- a/benchmarks/compress-bench/src/vortex.rs +++ b/benchmarks/compress-bench/src/vortex.rs @@ -18,6 +18,7 @@ use vortex::expr::root; use vortex::expr::select; use vortex::file::OpenOptionsSessionExt; use vortex::file::WriteOptionsSessionExt; +use vortex_arrow::ToArrowType; use vortex_bench::Format; use vortex_bench::SESSION; use vortex_bench::compress::Compressor; diff --git a/benchmarks/datafusion-bench/Cargo.toml b/benchmarks/datafusion-bench/Cargo.toml index 2f99d621a85..a195bbaa03f 100644 --- a/benchmarks/datafusion-bench/Cargo.toml +++ b/benchmarks/datafusion-bench/Cargo.toml @@ -39,6 +39,7 @@ parking_lot = { workspace = true } tokio = { workspace = true, features = ["full"] } url = { workspace = true } vortex = { workspace = true, features = ["object_store", "files", "tokio"] } +vortex-arrow = { workspace = true } vortex-bench = { workspace = true } vortex-cuda = { workspace = true, optional = true } vortex-datafusion = { workspace = true } diff --git a/benchmarks/datafusion-bench/src/main.rs b/benchmarks/datafusion-bench/src/main.rs index b8f9ac42df6..67521a8147e 100644 --- a/benchmarks/datafusion-bench/src/main.rs +++ b/benchmarks/datafusion-bench/src/main.rs @@ -29,6 +29,7 @@ use parking_lot::Mutex; use tokio::fs::File; use vortex::io::filesystem::FileSystemRef; use vortex::scan::DataSourceRef; +use vortex_arrow::ToArrowType; use vortex_bench::Benchmark; use vortex_bench::BenchmarkArg; use vortex_bench::CompactionStrategy; diff --git a/encodings/parquet-variant/Cargo.toml b/encodings/parquet-variant/Cargo.toml index d0c0a0d418e..6cdc2b3fa8b 100644 --- a/encodings/parquet-variant/Cargo.toml +++ b/encodings/parquet-variant/Cargo.toml @@ -25,6 +25,7 @@ parquet-variant = { workspace = true } parquet-variant-compute = { workspace = true } prost = { workspace = true } vortex-array = { workspace = true } +vortex-arrow = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-json = { workspace = true } diff --git a/encodings/parquet-variant/src/array.rs b/encodings/parquet-variant/src/array.rs index 7e55357c78f..bc7fd2c22ec 100644 --- a/encodings/parquet-variant/src/array.rs +++ b/encodings/parquet-variant/src/array.rs @@ -24,13 +24,6 @@ use vortex_array::arrays::VariantArray; use vortex_array::arrays::list::ListArrayExt; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::arrays::variant::VariantArrayExt; -#[expect( - deprecated, - reason = "TODO(aduffy): figure out what to do with Parquet Variant" -)] -use vortex_array::arrow::ArrowArrayExecutor; -use vortex_array::arrow::FromArrowArray; -use vortex_array::arrow::to_arrow_null_buffer; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::FieldName; @@ -41,6 +34,13 @@ use vortex_array::smallvec::smallvec; use vortex_array::validity::Validity; use vortex_array::vtable::child_to_validity; use vortex_array::vtable::validity_to_child; +#[expect( + deprecated, + reason = "TODO(aduffy): figure out what to do with Parquet Variant" +)] +use vortex_arrow::ArrowArrayExecutor; +use vortex_arrow::FromArrowArray; +use vortex_arrow::to_arrow_null_buffer; use vortex_buffer::BitBuffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; diff --git a/encodings/parquet-variant/src/arrow.rs b/encodings/parquet-variant/src/arrow.rs index da4e648a138..65efd6c571c 100644 --- a/encodings/parquet-variant/src/arrow.rs +++ b/encodings/parquet-variant/src/arrow.rs @@ -20,14 +20,14 @@ use vortex_array::IntoArray; use vortex_array::VTable; use vortex_array::arrays::Variant; use vortex_array::arrays::variant::VariantArrayExt; -use vortex_array::arrow::ArrowExport; -use vortex_array::arrow::ArrowExportVTable; -use vortex_array::arrow::ArrowImport; -use vortex_array::arrow::ArrowImportVTable; -use vortex_array::arrow::ArrowSession; -use vortex_array::arrow::ArrowSessionExt; -use vortex_array::arrow::to_arrow_null_buffer; use vortex_array::dtype::DType; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportVTable; +use vortex_arrow::ArrowImport; +use vortex_arrow::ArrowImportVTable; +use vortex_arrow::ArrowSession; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::to_arrow_null_buffer; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; @@ -300,11 +300,11 @@ mod tests { use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::VariantArray; - use vortex_array::arrow::ArrowSessionExt; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::validity::Validity; + use vortex_arrow::ArrowSessionExt; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_session::VortexSession; diff --git a/encodings/parquet-variant/src/kernel.rs b/encodings/parquet-variant/src/kernel.rs index bd7d0bfaa7c..c783e6600ac 100644 --- a/encodings/parquet-variant/src/kernel.rs +++ b/encodings/parquet-variant/src/kernel.rs @@ -37,8 +37,6 @@ use vortex_array::arrays::scalar_fn::ExactScalarFn; use vortex_array::arrays::scalar_fn::ScalarFnArrayView; use vortex_array::arrays::slice::SliceExecuteAdaptor; use vortex_array::arrays::slice::SliceKernel; -use vortex_array::arrow::ArrowSessionExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::DType; use vortex_array::kernel::ExecuteParentKernel; use vortex_array::optimizer::kernels::ArrayKernelsExt; @@ -46,6 +44,9 @@ use vortex_array::scalar_fn::ScalarFnVTable; use vortex_array::scalar_fn::fns::variant_get::VariantGet; use vortex_array::scalar_fn::fns::variant_get::VariantPath; use vortex_array::scalar_fn::fns::variant_get::VariantPathElement; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::FromArrowArray; +use vortex_arrow::ToArrowType; use vortex_error::VortexResult; use vortex_error::vortex_ensure_eq; use vortex_error::vortex_err; @@ -318,7 +319,6 @@ mod tests { use vortex_array::arrays::VariantArray; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::arrays::variant::VariantArrayExt; - use vortex_array::arrow::FromArrowArray; use vortex_array::assert_arrays_eq; use vortex_array::assert_nth_scalar_is_null; use vortex_array::dtype::DType as VortexDType; @@ -329,6 +329,7 @@ mod tests { use vortex_array::scalar_fn::fns::variant_get::VariantPath; use vortex_array::scalar_fn::fns::variant_get::VariantPathElement; use vortex_array::validity::Validity; + use vortex_arrow::FromArrowArray; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; diff --git a/encodings/parquet-variant/src/lib.rs b/encodings/parquet-variant/src/lib.rs index c71ea81e734..88125af04fd 100644 --- a/encodings/parquet-variant/src/lib.rs +++ b/encodings/parquet-variant/src/lib.rs @@ -37,8 +37,8 @@ mod vtable; use std::sync::Arc; pub use array::ParquetVariantArrayExt; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::session::ArraySessionExt; +use vortex_arrow::ArrowSessionExt; pub use vortex_json::JsonToVariant; pub use vortex_json::JsonToVariantOptions; pub use vortex_json::ShreddingSpec; diff --git a/encodings/pco/Cargo.toml b/encodings/pco/Cargo.toml index 8d560d5f068..1683d4875e9 100644 --- a/encodings/pco/Cargo.toml +++ b/encodings/pco/Cargo.toml @@ -28,4 +28,5 @@ vortex-session = { workspace = true } [dev-dependencies] rstest = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-arrow = { workspace = true } diff --git a/encodings/pco/src/tests.rs b/encodings/pco/src/tests.rs index 573bd6a0fb4..5cc771601f0 100644 --- a/encodings/pco/src/tests.rs +++ b/encodings/pco/src/tests.rs @@ -9,7 +9,6 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::assert_arrays_eq; use vortex_array::assert_nth_scalar; use vortex_array::dtype::DType; @@ -20,6 +19,7 @@ use vortex_array::serde::SerializedArray; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; use vortex_array::vtable::child_to_validity; +use vortex_arrow::ArrowSessionExt; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; diff --git a/encodings/runend/Cargo.toml b/encodings/runend/Cargo.toml index c7b38d2494f..4d36d0c1548 100644 --- a/encodings/runend/Cargo.toml +++ b/encodings/runend/Cargo.toml @@ -15,7 +15,6 @@ version = { workspace = true } [dependencies] arbitrary = { workspace = true, optional = true } -arrow-array = { workspace = true, optional = true } itertools = { workspace = true } num-traits = { workspace = true } prost = { workspace = true } @@ -29,8 +28,6 @@ vortex-session = { workspace = true } workspace = true [dev-dependencies] -arrow-array = { workspace = true } -arrow-schema = { workspace = true } divan = { workspace = true } itertools = { workspace = true } rand = { workspace = true } @@ -39,7 +36,6 @@ vortex-array = { workspace = true, features = ["_test-harness"] } [features] arbitrary = ["dep:arbitrary", "vortex-array/arbitrary"] -arrow = ["dep:arrow-array"] [[bench]] name = "run_end_null_count" diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 637947bba53..2a9659efdd8 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -323,7 +323,9 @@ impl RunEndData { } } - pub(crate) fn validate_parts( + /// Validate that `ends` and `values` form a well-formed run-end array covering + /// `offset..offset + length`. + pub fn validate_parts( ends: &ArrayRef, values: &ArrayRef, offset: usize, diff --git a/encodings/runend/src/lib.rs b/encodings/runend/src/lib.rs index a0b26b54dab..ed3ff946d31 100644 --- a/encodings/runend/src/lib.rs +++ b/encodings/runend/src/lib.rs @@ -9,14 +9,12 @@ pub use array::*; pub use iter::trimmed_ends_iter; mod array; -#[cfg(feature = "arrow")] -mod arrow; pub mod compress; mod compute; pub mod decompress_bool; mod iter; mod kernel; -mod ops; +pub mod ops; mod rules; #[doc(hidden)] diff --git a/encodings/runend/src/ops.rs b/encodings/runend/src/ops.rs index c929f3d4251..39fb004c3e7 100644 --- a/encodings/runend/src/ops.rs +++ b/encodings/runend/src/ops.rs @@ -31,7 +31,7 @@ impl OperationsVTable for RunEnd { /// /// If the index exists in the array we want to take that position (as we are searching from the right) /// otherwise we want to take the next one -pub(crate) fn find_slice_end_index( +pub fn find_slice_end_index( array: &ArrayRef, index: usize, ctx: &mut ExecutionCtx, @@ -52,7 +52,8 @@ pub(crate) fn find_slice_end_index( }) } -pub(crate) fn find_physical_index( +/// Find the physical index (the run) that contains the logical `index`, given the run `ends`. +pub fn find_physical_index( array: &ArrayRef, index: usize, ctx: &mut ExecutionCtx, diff --git a/encodings/sparse/Cargo.toml b/encodings/sparse/Cargo.toml index acf99a74ccf..56e87745b77 100644 --- a/encodings/sparse/Cargo.toml +++ b/encodings/sparse/Cargo.toml @@ -31,6 +31,7 @@ divan = { workspace = true } itertools = { workspace = true } rstest = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-arrow = { workspace = true } [[bench]] name = "sparse_canonical" diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index afca2b29a18..0be6d0a4162 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -613,7 +613,6 @@ mod test { use vortex_array::arrays::VarBinArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::listview::ListViewArrayExt; - use vortex_array::arrow::ArrowSessionExt; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::DecimalDType; @@ -625,6 +624,7 @@ mod test { use vortex_array::scalar::DecimalValue; use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; + use vortex_arrow::ArrowSessionExt; use vortex_buffer::ByteBuffer; use vortex_buffer::buffer; use vortex_buffer::buffer_mut; diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index 507c7345621..dcd8af6d76a 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -23,12 +23,7 @@ workspace = true arbitrary = { workspace = true, optional = true } arc-swap = { workspace = true } arcref = { workspace = true } -arrow-array = { workspace = true, features = ["ffi"] } arrow-buffer = { workspace = true } -arrow-cast = { workspace = true } -arrow-data = { workspace = true } -arrow-schema = { workspace = true, features = ["canonical_extension_types"] } -arrow-select = { workspace = true } async-lock = { workspace = true } bytes = { workspace = true } cfg-if = { workspace = true } @@ -69,7 +64,7 @@ termtree = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } vortex-array-macros = { workspace = true } -vortex-buffer = { workspace = true, features = ["arrow"] } +vortex-buffer = { workspace = true } vortex-compute = { workspace = true } vortex-error = { workspace = true, features = ["flatbuffers"] } vortex-flatbuffers = { workspace = true, features = ["array", "dtype"] } @@ -87,7 +82,6 @@ _test-harness = ["dep:goldenfile", "dep:rstest", "dep:rstest_reuse"] serde = ["dep:serde", "vortex-buffer/serde", "vortex-mask/serde"] [dev-dependencies] -arrow-cast = { workspace = true } divan = { workspace = true } futures = { workspace = true, features = ["executor"] } insta = { workspace = true } @@ -271,7 +265,3 @@ harness = false [[bench]] name = "slice_dict_primitive" harness = false - -[[bench]] -name = "to_arrow" -harness = false diff --git a/vortex-array/benches/kleene_bool.rs b/vortex-array/benches/kleene_bool.rs index b23a0580a5a..60a0171a9b9 100644 --- a/vortex-array/benches/kleene_bool.rs +++ b/vortex-array/benches/kleene_bool.rs @@ -13,7 +13,6 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; -use vortex_array::arrow::ArrowSession; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; @@ -31,7 +30,6 @@ static SESSION: LazyLock = LazyLock::new(|| { VortexSession::empty() .with::() .with::() - .with::() }); const LEN: usize = 65_536; diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs index 6256dc22327..ada9e6eb2e5 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs @@ -668,16 +668,14 @@ mod tests { // Tests migrated from arrays/decimal/compute/is_sorted.rs #[test] fn test_decimal_is_sorted() -> VortexResult<()> { - use arrow_array::types::Decimal128Type; - use arrow_cast::parse::parse_decimal; - use crate::arrays::DecimalArray; use crate::dtype::DecimalDType; let mut ctx = array_session().create_execution_ctx(); let dtype = DecimalDType::new(19, 2); - let i100 = parse_decimal::("100.00", dtype.precision(), dtype.scale())?; - let i200 = parse_decimal::("200.00", dtype.precision(), dtype.scale())?; + // "100.00" and "200.00" at scale 2. + let i100 = 10_000i128; + let i200 = 20_000i128; let sorted = buffer![i100, i200, i200]; let unsorted = buffer![i200, i100, i200]; @@ -693,17 +691,14 @@ mod tests { #[test] fn test_decimal_is_strict_sorted() -> VortexResult<()> { - use arrow_array::types::Decimal128Type; - use arrow_cast::parse::parse_decimal; - use crate::arrays::DecimalArray; use crate::dtype::DecimalDType; let mut ctx = array_session().create_execution_ctx(); - let dtype = DecimalDType::new(19, 2); - let i100 = parse_decimal::("100.00", dtype.precision(), dtype.scale())?; - let i200 = parse_decimal::("200.00", dtype.precision(), dtype.scale())?; - let i300 = parse_decimal::("300.00", dtype.precision(), dtype.scale())?; + // "100.00", "200.00" and "300.00" at scale 2. + let i100 = 10_000i128; + let i200 = 20_000i128; + let i300 = 30_000i128; let strict_sorted = buffer![i100, i200, i300]; let sorted = buffer![i100, i200, i200]; diff --git a/vortex-array/src/arc_swap_map.rs b/vortex-array/src/arc_swap_map.rs index 73108ae5d57..1788f704feb 100644 --- a/vortex-array/src/arc_swap_map.rs +++ b/vortex-array/src/arc_swap_map.rs @@ -28,7 +28,7 @@ use vortex_utils::aliases::hash_map::HashMap; /// underlying cell: a registry mutated through one clone is observed by all /// others. Session variables rely on this so that encodings registered after a /// session is built remain visible to clones of that session. -pub(crate) struct ArcSwapMap { +pub struct ArcSwapMap { inner: Arc>>, } @@ -56,7 +56,7 @@ impl Debug for ArcSwapMap { impl ArcSwapMap { /// Return the currently published map snapshot. - pub(crate) fn snapshot(&self) -> Arc> { + pub fn snapshot(&self) -> Arc> { self.inner.load_full() } @@ -64,7 +64,7 @@ impl ArcSwapMap { /// /// Every lookup inside `f` observes the same snapshot, which matters when a /// single logical read consults more than one key. - pub(crate) fn read(&self, f: impl FnOnce(&HashMap) -> R) -> R { + pub fn read(&self, f: impl FnOnce(&HashMap) -> R) -> R { f(&self.inner.load()) } @@ -87,7 +87,7 @@ impl ArcSwapMap { impl ArcSwapMap { /// Return a clone of the value stored under `key`, if present. - pub(crate) fn get(&self, key: &Q) -> Option + pub fn get(&self, key: &Q) -> Option where K: Borrow, Q: Eq + Hash + ?Sized, @@ -96,7 +96,7 @@ impl ArcSwapMap { } /// Insert `value` under `key`, replacing any existing value. - pub(crate) fn insert(&self, key: K, value: V) + pub fn insert(&self, key: K, value: V) where K: Clone, { @@ -111,7 +111,7 @@ impl ArcSwapMap> { /// /// Each key maps to an immutable `Arc<[T]>`; appending rebuilds that slice /// copy-on-write so existing readers keep their previous snapshot. - pub(crate) fn extend(&self, key: K, values: &[T]) { + pub fn extend(&self, key: K, values: &[T]) { self.modify(|map| { let merged: Arc<[T]> = match map.get(&key) { Some(existing) => existing.iter().chain(values).cloned().collect(), @@ -123,7 +123,7 @@ impl ArcSwapMap> { /// Append a single `value` to the list stored under `key`, creating it if /// absent. - pub(crate) fn push(&self, key: K, value: T) { + pub fn push(&self, key: K, value: T) { self.extend(key, &[value]); } } diff --git a/vortex-array/src/arrays/constant/compute/fill_null.rs b/vortex-array/src/arrays/constant/compute/fill_null.rs index 1c9c2aaf903..67551a0592a 100644 --- a/vortex-array/src/arrays/constant/compute/fill_null.rs +++ b/vortex-array/src/arrays/constant/compute/fill_null.rs @@ -25,7 +25,7 @@ mod test { use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::ConstantArray; - use crate::arrow::ArrowSessionExt; + use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::Nullability; @@ -43,21 +43,7 @@ mod test { assert!(!actual.dtype().is_nullable()); - let actual_arrow = array_session() - .arrow() - .execute_arrow(actual.clone(), None, &mut ctx) - .unwrap(); - let expected_arrow = array_session() - .arrow() - .execute_arrow(expected.clone(), None, &mut ctx) - .unwrap(); - assert_eq!( - &actual_arrow, - &expected_arrow, - "{}, {}", - actual.display_values(), - expected.display_values() - ); + assert_arrays_eq!(actual, expected, &mut ctx); } #[test] @@ -71,21 +57,7 @@ mod test { assert!(!actual.dtype().is_nullable()); - let actual_arrow = array_session() - .arrow() - .execute_arrow(actual.clone(), None, &mut ctx) - .unwrap(); - let expected_arrow = array_session() - .arrow() - .execute_arrow(expected.clone(), None, &mut ctx) - .unwrap(); - assert_eq!( - &actual_arrow, - &expected_arrow, - "{}, {}", - actual.display_values(), - expected.display_values() - ); + assert_arrays_eq!(actual, expected, &mut ctx); } #[test] @@ -98,26 +70,19 @@ mod test { Some(1.into()), )) .unwrap(); - let expected = ConstantArray::new(Scalar::from(1), 3).into_array(); + let expected = ConstantArray::new( + Scalar::new( + DType::Primitive(PType::I32, Nullability::Nullable), + Some(1.into()), + ), + 3, + ) + .into_array(); assert!(!Scalar::from(1).dtype().is_nullable()); assert!(actual.dtype().is_nullable()); - let actual_arrow = array_session() - .arrow() - .execute_arrow(actual.clone(), None, &mut ctx) - .unwrap(); - let expected_arrow = array_session() - .arrow() - .execute_arrow(expected.clone(), None, &mut ctx) - .unwrap(); - assert_eq!( - &actual_arrow, - &expected_arrow, - "{}, {}", - actual.display_values(), - expected.display_values() - ); + assert_arrays_eq!(actual, expected, &mut ctx); } } diff --git a/vortex-array/src/arrays/decimal/mod.rs b/vortex-array/src/arrays/decimal/mod.rs index e740a75ea91..d8af8529e87 100644 --- a/vortex-array/src/arrays/decimal/mod.rs +++ b/vortex-array/src/arrays/decimal/mod.rs @@ -19,17 +19,3 @@ pub(crate) fn initialize(session: &vortex_session::VortexSession) { mod utils; pub use utils::*; - -#[cfg(test)] -mod tests { - use arrow_array::Decimal128Array; - - #[test] - fn test_decimal() { - // They pass it b/c the DType carries the information. No other way to carry a - // dtype except via the array. - let value = Decimal128Array::new_null(100); - let numeric = value.value(10); - assert_eq!(numeric, 0i128); - } -} diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index 62749db4650..71da52e1300 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -85,14 +85,14 @@ use crate::validity::Validity; /// /// The full list of canonical types and their equivalent Arrow array types are: /// -/// * `NullArray`: [`arrow_array::NullArray`] -/// * `BoolArray`: [`arrow_array::BooleanArray`] -/// * `PrimitiveArray`: [`arrow_array::PrimitiveArray`] -/// * `DecimalArray`: [`arrow_array::Decimal128Array`] and [`arrow_array::Decimal256Array`] -/// * `VarBinViewArray`: [`arrow_array::GenericByteViewArray`] -/// * `ListViewArray`: [`arrow_array::ListViewArray`] -/// * `FixedSizeListArray`: [`arrow_array::FixedSizeListArray`] -/// * `StructArray`: [`arrow_array::StructArray`] +/// * `NullArray`: `arrow_array::NullArray` +/// * `BoolArray`: `arrow_array::BooleanArray` +/// * `PrimitiveArray`: `arrow_array::PrimitiveArray` +/// * `DecimalArray`: `arrow_array::Decimal128Array` and `arrow_array::Decimal256Array` +/// * `VarBinViewArray`: `arrow_array::GenericByteViewArray` +/// * `ListViewArray`: `arrow_array::ListViewArray` +/// * `FixedSizeListArray`: `arrow_array::FixedSizeListArray` +/// * `StructArray`: `arrow_array::StructArray` /// /// Vortex uses a logical type system, unlike Arrow which uses physical encodings for its types. /// As an example, there are at least six valid physical encodings for a `Utf8` array. This can @@ -102,7 +102,7 @@ use crate::validity::Validity; /// variants to hold the data. /// /// To disambiguate, we choose a canonical physical encoding for every Vortex [`DType`], which -/// will correspond to an arrow-rs [`arrow_schema::DataType`]. +/// will correspond to an arrow-rs `arrow_schema::DataType`. /// /// # Views support /// @@ -111,7 +111,7 @@ use crate::validity::Validity; /// fully supported by the Datafusion query engine. We use them as our canonical string encoding /// for all `Utf8` and `Binary` typed arrays in Vortex. They provide considerably faster filter /// execution than the core `StringArray` and `BinaryArray` types, at the expense of potentially -/// needing [garbage collection][arrow_array::GenericByteViewArray::gc] to clear unreferenced items +/// needing garbage collection (`arrow_array::GenericByteViewArray::gc`) to clear unreferenced items /// from memory. /// /// # For Developers @@ -1118,25 +1118,8 @@ impl Matcher for AnyCanonical { #[cfg(test)] mod test { - use std::sync::Arc; use std::sync::LazyLock; - use arrow_array::Array as ArrowArray; - use arrow_array::ArrayRef as ArrowArrayRef; - use arrow_array::ListArray as ArrowListArray; - use arrow_array::PrimitiveArray as ArrowPrimitiveArray; - use arrow_array::StringArray; - use arrow_array::StringViewArray; - use arrow_array::StructArray as ArrowStructArray; - use arrow_array::cast::AsArray; - use arrow_array::types::Int32Type; - use arrow_array::types::Int64Type; - use arrow_array::types::UInt64Type; - use arrow_buffer::NullBufferBuilder; - use arrow_buffer::OffsetBuffer; - use arrow_schema::DataType; - use arrow_schema::Field; - use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_session::VortexSession; @@ -1154,8 +1137,6 @@ mod test { use crate::arrays::VariantArray; use crate::arrays::struct_::StructArrayExt; use crate::arrays::variant::VariantArrayExt; - use crate::arrow::ArrowSessionExt; - use crate::arrow::FromArrowArray; use crate::canonical::StructArray; use crate::dtype::Nullability; use crate::scalar::Scalar; @@ -1207,134 +1188,4 @@ mod test { Ok(()) } - - #[test] - fn test_canonicalize_nested_struct() { - let mut ctx = SESSION.create_execution_ctx(); - // Create a struct array with multiple internal components. - let nested_struct_array = StructArray::from_fields(&[ - ("a", buffer![1u64].into_array()), - ( - "b", - StructArray::from_fields(&[( - "inner_a", - // The nested struct contains a ConstantArray representing the primitive array - // [100i64] - // ConstantArray is not a canonical type, so converting `into_arrow()` should - // map this to the nearest canonical type (PrimitiveArray). - ConstantArray::new(100i64, 1).into_array(), - )]) - .unwrap() - .into_array(), - ), - ]) - .unwrap(); - - let arrow_struct = SESSION - .arrow() - .execute_arrow(nested_struct_array.into_array(), None, &mut ctx) - .unwrap() - .as_any() - .downcast_ref::() - .cloned() - .unwrap(); - - assert!( - arrow_struct - .column(0) - .as_any() - .downcast_ref::>() - .is_some() - ); - - let inner_struct = Arc::clone(arrow_struct.column(1)) - .as_any() - .downcast_ref::() - .cloned() - .unwrap(); - - let inner_a = inner_struct - .column(0) - .as_any() - .downcast_ref::>(); - assert!(inner_a.is_some()); - - assert_eq!( - inner_a.cloned().unwrap(), - ArrowPrimitiveArray::from_iter([100i64]) - ); - } - - #[test] - fn roundtrip_struct() { - let mut ctx = SESSION.create_execution_ctx(); - let mut nulls = NullBufferBuilder::new(6); - nulls.append_n_non_nulls(4); - nulls.append_null(); - nulls.append_non_null(); - let names = Arc::new(StringViewArray::from_iter(vec![ - Some("Joseph"), - None, - Some("Angela"), - Some("Mikhail"), - None, - None, - ])); - let ages = Arc::new(ArrowPrimitiveArray::::from(vec![ - Some(25), - Some(31), - None, - Some(57), - None, - None, - ])); - - let arrow_struct = ArrowStructArray::new( - vec![ - Arc::new(Field::new("name", DataType::Utf8View, true)), - Arc::new(Field::new("age", DataType::Int32, true)), - ] - .into(), - vec![names, ages], - nulls.finish(), - ); - - let vortex_struct = ArrayRef::from_arrow(&arrow_struct, true).unwrap(); - let vortex_struct = SESSION - .arrow() - .execute_arrow(vortex_struct, None, &mut ctx) - .unwrap(); - assert_eq!(&arrow_struct, vortex_struct.as_struct()); - } - - #[test] - fn roundtrip_list() { - let mut ctx = SESSION.create_execution_ctx(); - let names = Arc::new(StringArray::from_iter(vec![ - Some("Joseph"), - Some("Angela"), - Some("Mikhail"), - ])); - - let arrow_list = ArrowListArray::new( - Arc::new(Field::new_list_field(DataType::Utf8, true)), - OffsetBuffer::from_lengths(vec![0, 2, 1]), - names, - None, - ); - let list_data_type = arrow_list.data_type(); - let list_field = Field::new(String::new(), list_data_type.clone(), true); - - let vortex_list = ArrayRef::from_arrow(&arrow_list, true).unwrap(); - - let rt_arrow_list = SESSION - .arrow() - .execute_arrow(vortex_list, Some(&list_field), &mut ctx) - .unwrap(); - - assert_eq!( - (Arc::new(arrow_list.clone()) as ArrowArrayRef).as_ref(), - rt_arrow_list.as_ref() - ); - } } diff --git a/vortex-array/src/dtype/mod.rs b/vortex-array/src/dtype/mod.rs index 6a166a868ff..0160b4c36c7 100644 --- a/vortex-array/src/dtype/mod.rs +++ b/vortex-array/src/dtype/mod.rs @@ -15,7 +15,6 @@ #[cfg(feature = "arbitrary")] mod arbitrary; -pub mod arrow; mod bigint; mod coercion; mod decimal; diff --git a/vortex-array/src/expr/exprs.rs b/vortex-array/src/expr/exprs.rs index 73fa76fce92..da70427a0f7 100644 --- a/vortex-array/src/expr/exprs.rs +++ b/vortex-array/src/expr/exprs.rs @@ -440,7 +440,8 @@ where /// /// ``` /// # use vortex_array::IntoArray; -/// # use vortex_array::arrow::ArrowArrayExecutor; +/// # use vortex_array::arrays::PrimitiveArray; +/// # use vortex_array::builtins::ArrayBuiltins; /// # use vortex_array::{VortexSessionExecute, array_session}; /// # use vortex_buffer::buffer; /// # use vortex_array::expr::{checked_add, lit, root}; @@ -448,13 +449,8 @@ where /// let result = xs.apply(&checked_add(root(), lit(5))).unwrap(); /// /// let mut ctx = array_session().create_execution_ctx(); -/// assert_eq!( -/// &result.execute_arrow(None, &mut ctx).unwrap(), -/// &buffer![6, 7, 8] -/// .into_array() -/// .execute_arrow(None, &mut ctx) -/// .unwrap() -/// ); +/// let result = result.execute::(&mut ctx).unwrap(); +/// assert_eq!(result.as_slice::(), [6, 7, 8]); /// ``` pub fn checked_add(lhs: Expression, rhs: Expression) -> Expression { Binary diff --git a/vortex-array/src/extension/uuid/mod.rs b/vortex-array/src/extension/uuid/mod.rs index 8a178035d2f..e4347c2513c 100644 --- a/vortex-array/src/extension/uuid/mod.rs +++ b/vortex-array/src/extension/uuid/mod.rs @@ -13,7 +13,6 @@ mod metadata; pub use metadata::UuidMetadata; -mod arrow; pub(crate) mod vtable; /// The VTable for the UUID extension type. diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs index 6456c412929..9d3a59eb7a0 100644 --- a/vortex-array/src/lib.rs +++ b/vortex-array/src/lib.rs @@ -95,7 +95,6 @@ use vortex_session::VortexSession; use vortex_session::registry::Context; use crate::aggregate_fn::session::AggregateFnSession; -use crate::arrow::ArrowSession; use crate::dtype::session::DTypeSession; use crate::memory::MemorySession; use crate::optimizer::kernels::KernelSession; @@ -106,10 +105,9 @@ use crate::stats::session::StatsSession; pub mod aggregate_fn; #[doc(hidden)] pub mod aliases; -mod arc_swap_map; +pub mod arc_swap_map; mod array; pub mod arrays; -pub mod arrow; pub mod buffer; pub mod builders; pub mod builtins; @@ -163,11 +161,13 @@ pub fn initialize(session: &VortexSession) { /// Builds a fresh [`VortexSession`] registered with all of vortex-array's built-in session /// variables: arrays, dtypes, scalar functions, stats, optimizer kernels, aggregate functions, -/// Arrow conversion, and memory. +/// and memory. /// /// Each call returns an independent session (with its own registries), so callers may register /// additional encodings or kernels into it without affecting any other session. This does not -/// register file, layout, or runtime state — those live in higher-level crates. +/// register file, layout, or runtime state — those live in higher-level crates. Arrow +/// interoperability lives in the `vortex-arrow` crate, whose session variables are registered +/// lazily on first use (or eagerly by consumers). pub fn array_session() -> VortexSession { VortexSession::empty() .with::() @@ -176,7 +176,6 @@ pub fn array_session() -> VortexSession { .with::() .with::() .with::() - .with::() .with::() } diff --git a/vortex-array/src/scalar/mod.rs b/vortex-array/src/scalar/mod.rs index ff8d2c6134b..dbc1e17d04c 100644 --- a/vortex-array/src/scalar/mod.rs +++ b/vortex-array/src/scalar/mod.rs @@ -15,7 +15,6 @@ #[cfg(feature = "arbitrary")] pub mod arbitrary; -mod arrow; mod cast; mod constructor; diff --git a/vortex-array/src/scalar_fn/fns/zip/mod.rs b/vortex-array/src/scalar_fn/fns/zip/mod.rs index 48449bc8428..40fedbe85eb 100644 --- a/vortex-array/src/scalar_fn/fns/zip/mod.rs +++ b/vortex-array/src/scalar_fn/fns/zip/mod.rs @@ -267,8 +267,6 @@ pub(crate) fn zip_validity( #[cfg(test)] mod tests { - use arrow_array::cast::AsArray; - use arrow_select::zip::zip as arrow_zip; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_mask::Mask; @@ -283,7 +281,7 @@ mod tests { use crate::arrays::Struct; use crate::arrays::StructArray; use crate::arrays::VarBinView; - use crate::arrow::ArrowSessionExt; + use crate::arrays::VarBinViewArray; use crate::assert_arrays_eq; use crate::builders::ArrayBuilder; use crate::builders::BufferGrowthStrategy; @@ -489,35 +487,34 @@ mod tests { let mut ctx = array_session().create_execution_ctx(); let zipped = mask_array - .zip(if_true.clone(), if_false.clone()) + .zip(if_true, if_false) .unwrap() .execute::(&mut ctx) .unwrap(); let zipped = zipped.as_opt::().unwrap(); assert_eq!(zipped.data_buffers().len(), 2); - let mut arrow_ctx = array_session().create_execution_ctx(); - let expected = arrow_zip( - array_session() - .arrow() - .execute_arrow(mask.into_array(), None, &mut arrow_ctx) - .unwrap() - .as_boolean(), - &array_session() - .arrow() - .execute_arrow(if_true, None, &mut arrow_ctx) - .unwrap(), - &array_session() - .arrow() - .execute_arrow(if_false, None, &mut arrow_ctx) - .unwrap(), - ) - .unwrap(); - - let actual = array_session() - .arrow() - .execute_arrow(zipped.array().clone(), None, &mut arrow_ctx) - .unwrap(); - assert_eq!(actual.as_ref(), expected.as_ref()); + let true_value = |i: usize| { + if i.is_multiple_of(2) { + "Hello" + } else { + "Hello this is a long string that won't be inlined." + } + }; + let false_value = |i: usize| { + if i.is_multiple_of(2) { + "Hello2" + } else { + "Hello2 this is a long string that won't be inlined." + } + }; + let expected = VarBinViewArray::from_iter_str((0..200).map(|i| { + if mask.value(i) { + true_value(i) + } else { + false_value(i) + } + })); + assert_arrays_eq!(zipped.array().clone(), expected, &mut ctx); } } diff --git a/vortex-arrow/Cargo.toml b/vortex-arrow/Cargo.toml new file mode 100644 index 00000000000..9d69a06fd7d --- /dev/null +++ b/vortex-arrow/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "vortex-arrow" +authors = { workspace = true } +categories = { workspace = true } +description = "Apache Arrow interoperability for Vortex arrays" +edition = { workspace = true } +homepage = { workspace = true } +include = { workspace = true } +keywords = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true + +[dependencies] +arrow-array = { workspace = true, features = ["ffi"] } +arrow-buffer = { workspace = true } +arrow-cast = { workspace = true } +arrow-data = { workspace = true } +arrow-schema = { workspace = true, features = ["canonical_extension_types"] } +arrow-select = { workspace = true } +itertools = { workspace = true } +num-traits = { workspace = true } +tracing = { workspace = true } +vortex-array = { workspace = true } +vortex-buffer = { workspace = true, features = ["arrow"] } +vortex-error = { workspace = true } +vortex-mask = { workspace = true } +vortex-runend = { workspace = true } +vortex-session = { workspace = true } + +[dev-dependencies] +divan = { workspace = true } +rstest = { workspace = true } +vortex-array = { workspace = true, features = ["_test-harness"] } + +[[bench]] +name = "to_arrow" +harness = false diff --git a/vortex-array/benches/to_arrow.rs b/vortex-arrow/benches/to_arrow.rs similarity index 95% rename from vortex-array/benches/to_arrow.rs rename to vortex-arrow/benches/to_arrow.rs index 99fc0fa4f80..1cec274c28e 100644 --- a/vortex-array/benches/to_arrow.rs +++ b/vortex-arrow/benches/to_arrow.rs @@ -15,17 +15,19 @@ use vortex_array::arrays::DecimalArray; use vortex_array::arrays::ListArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; -#[expect( - deprecated, - reason = "benchmark comparing deprecated method with new one" -)] -use vortex_array::arrow::ArrowArrayExecutor; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::DType; use vortex_array::dtype::DecimalDType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::StructFields; +#[expect( + deprecated, + reason = "benchmark comparing deprecated method with new one" +)] +use vortex_arrow::ArrowArrayExecutor; +use vortex_arrow::ArrowSessionExt; +#[allow(deprecated)] +use vortex_arrow::dtype::ToArrowType as _; use vortex_session::VortexSession; fn main() { diff --git a/vortex-array/src/arrow/convert.rs b/vortex-arrow/src/convert.rs similarity index 95% rename from vortex-array/src/arrow/convert.rs rename to vortex-arrow/src/convert.rs index ee1171a1848..0bab0937a8a 100644 --- a/vortex-array/src/arrow/convert.rs +++ b/vortex-arrow/src/convert.rs @@ -57,6 +57,28 @@ use arrow_buffer::buffer::NullBuffer; use arrow_buffer::buffer::OffsetBuffer; use arrow_schema::DataType; use arrow_schema::TimeUnit as ArrowTimeUnit; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::DecimalArray; +use vortex_array::arrays::DictArray; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::ListArray; +use vortex_array::arrays::ListViewArray; +use vortex_array::arrays::NullArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::StructArray; +use vortex_array::arrays::TemporalArray; +use vortex_array::arrays::VarBinArray; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::dtype::DType; +use vortex_array::dtype::DecimalDType; +use vortex_array::dtype::IntegerPType; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::PType; +use vortex_array::dtype::i256; +use vortex_array::extension::datetime::TimeUnit; +use vortex_array::validity::Validity; use vortex_buffer::Alignment; use vortex_buffer::BitBuffer; use vortex_buffer::Buffer; @@ -68,31 +90,19 @@ use vortex_error::vortex_ensure_eq; use vortex_error::vortex_err; use vortex_error::vortex_panic; -use crate::ArrayRef; -use crate::IntoArray; -use crate::arrays::BoolArray; -use crate::arrays::DecimalArray; -use crate::arrays::DictArray; -use crate::arrays::FixedSizeListArray; -use crate::arrays::ListArray; -use crate::arrays::ListViewArray; -use crate::arrays::NullArray; -use crate::arrays::PrimitiveArray; -use crate::arrays::StructArray; -use crate::arrays::TemporalArray; -use crate::arrays::VarBinArray; -use crate::arrays::VarBinViewArray; -use crate::arrow::FromArrowArray; -use crate::dtype::DType; -use crate::dtype::DecimalDType; -use crate::dtype::IntegerPType; -use crate::dtype::NativePType; -use crate::dtype::PType; -use crate::dtype::i256; -use crate::extension::datetime::TimeUnit; -use crate::validity::Validity; - -impl IntoArray for ArrowBuffer { +use crate::FromArrowArray; +use crate::dtype::FromArrowType; + +/// Zero-copy conversion of Arrow buffers into non-nullable Vortex arrays. +/// +/// This mirrors [`IntoArray`] for Arrow buffer types; a separate trait is required because both +/// [`IntoArray`] and the Arrow buffer types are foreign to this crate. +pub trait IntoVortexArray { + /// Convert this Arrow buffer into a non-nullable Vortex array without copying. + fn into_array(self) -> ArrayRef; +} + +impl IntoVortexArray for ArrowBuffer { fn into_array(self) -> ArrayRef { PrimitiveArray::from_byte_buffer( ByteBuffer::from_arrow_buffer(self, Alignment::of::()), @@ -103,13 +113,13 @@ impl IntoArray for ArrowBuffer { } } -impl IntoArray for BooleanBuffer { +impl IntoVortexArray for BooleanBuffer { fn into_array(self) -> ArrayRef { BoolArray::new(self.into(), Validity::NonNullable).into_array() } } -impl IntoArray for ScalarBuffer +impl IntoVortexArray for ScalarBuffer where T: ArrowNativeType + NativePType, { @@ -122,7 +132,7 @@ where } } -impl IntoArray for OffsetBuffer +impl IntoVortexArray for OffsetBuffer where O: IntegerPType + OffsetSizeTrait, { @@ -257,10 +267,14 @@ where Ok(match value.data_type() { DataType::Timestamp(time_unit, tz) => { - TemporalArray::new_timestamp(arr, time_unit.into(), tz.clone()).into() + TemporalArray::new_timestamp(arr, TimeUnit::from_arrow(time_unit), tz.clone()).into() + } + DataType::Time32(time_unit) => { + TemporalArray::new_time(arr, TimeUnit::from_arrow(time_unit)).into() + } + DataType::Time64(time_unit) => { + TemporalArray::new_time(arr, TimeUnit::from_arrow(time_unit)).into() } - DataType::Time32(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(), - DataType::Time64(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(), DataType::Date32 => TemporalArray::new_date(arr, TimeUnit::Days).into(), DataType::Date64 => TemporalArray::new_date(arr, TimeUnit::Milliseconds).into(), DataType::Duration(_) => unimplemented!(), @@ -693,26 +707,26 @@ mod tests { use arrow_schema::Fields; use arrow_schema::Schema; use rstest::rstest; - - use crate::ArrayRef; - use crate::IntoArray; - use crate::arrays::Decimal; - use crate::arrays::FixedSizeList; - use crate::arrays::List; - use crate::arrays::ListView; - use crate::arrays::Primitive; - use crate::arrays::Struct; - use crate::arrays::VarBinView; - use crate::arrays::fixed_size_list::FixedSizeListArrayExt; - use crate::arrays::list::ListArrayExt; - use crate::arrays::listview::ListViewArrayExt; - use crate::arrays::struct_::StructArrayExt; - use crate::arrow::FromArrowArray as _; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::extension::datetime::TimeUnit; - use crate::extension::datetime::Timestamp; + use vortex_array::ArrayRef; + use vortex_array::arrays::Decimal; + use vortex_array::arrays::FixedSizeList; + use vortex_array::arrays::List; + use vortex_array::arrays::ListView; + use vortex_array::arrays::Primitive; + use vortex_array::arrays::Struct; + use vortex_array::arrays::VarBinView; + use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; + use vortex_array::arrays::list::ListArrayExt; + use vortex_array::arrays::listview::ListViewArrayExt; + use vortex_array::arrays::struct_::StructArrayExt; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::extension::datetime::TimeUnit; + use vortex_array::extension::datetime::Timestamp; + + use crate::FromArrowArray as _; + use crate::IntoVortexArray as _; #[rstest] #[case::i8( diff --git a/vortex-array/src/arrow/datum.rs b/vortex-arrow/src/datum.rs similarity index 94% rename from vortex-array/src/arrow/datum.rs rename to vortex-arrow/src/datum.rs index 03e1b8dc24b..5019642c452 100644 --- a/vortex-array/src/arrow/datum.rs +++ b/vortex-arrow/src/datum.rs @@ -6,19 +6,19 @@ use arrow_array::ArrayRef as ArrowArrayRef; use arrow_array::Datum as ArrowDatum; use arrow_schema::DataType; use arrow_schema::Field; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::Constant; +use vortex_array::arrays::ConstantArray; +use vortex_array::legacy_session; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_panic; -use crate::ArrayRef; -use crate::IntoArray; -use crate::VortexSessionExecute; -use crate::arrays::Constant; -use crate::arrays::ConstantArray; -use crate::arrow::ArrowSessionExt; -use crate::arrow::FromArrowArray; -use crate::executor::ExecutionCtx; -use crate::legacy_session; +use crate::ArrowSessionExt; +use crate::FromArrowArray; /// A wrapper around a generic Arrow array that can be used as a Datum in Arrow compute. #[derive(Debug)] diff --git a/vortex-array/src/dtype/arrow.rs b/vortex-arrow/src/dtype.rs similarity index 89% rename from vortex-array/src/dtype/arrow.rs rename to vortex-arrow/src/dtype.rs index 9d381b6eab2..a87c931f3df 100644 --- a/vortex-array/src/dtype/arrow.rs +++ b/vortex-arrow/src/dtype.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Convert between Vortex [`crate::dtype::DType`] and Apache Arrow [`arrow_schema::DataType`]. +//! Convert between Vortex [`vortex_array::dtype::DType`] and Apache Arrow [`arrow_schema::DataType`]. //! //! Apache Arrow's type system includes physical information, which could lead to ambiguities as //! Vortex treats encodings as separate from logical types. @@ -23,26 +23,24 @@ use arrow_schema::Schema; use arrow_schema::SchemaBuilder; use arrow_schema::SchemaRef; use arrow_schema::TimeUnit as ArrowTimeUnit; -use vortex_error::VortexError; +use vortex_array::dtype::DType; +use vortex_array::dtype::DecimalDType; +use vortex_array::dtype::FieldName; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::dtype::StructFields; +use vortex_array::extension::datetime::AnyTemporal; +use vortex_array::extension::datetime::Date; +use vortex_array::extension::datetime::TemporalMetadata; +use vortex_array::extension::datetime::Time; +use vortex_array::extension::datetime::TimeUnit; +use vortex_array::extension::datetime::Timestamp; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; use vortex_error::vortex_panic; -use crate::dtype::DType; -use crate::dtype::DecimalDType; -use crate::dtype::FieldName; -use crate::dtype::Nullability; -use crate::dtype::PType; -use crate::dtype::StructFields; -use crate::extension::datetime::AnyTemporal; -use crate::extension::datetime::Date; -use crate::extension::datetime::TemporalMetadata; -use crate::extension::datetime::Time; -use crate::extension::datetime::TimeUnit; -use crate::extension::datetime::Timestamp; - /// Trait for converting Arrow types to Vortex types. pub trait FromArrowType: Sized { /// Convert the Arrow type to a Vortex type. @@ -93,14 +91,14 @@ impl TryFromArrowType<&DataType> for DecimalDType { } } -impl From<&ArrowTimeUnit> for TimeUnit { - fn from(value: &ArrowTimeUnit) -> Self { - (*value).into() +impl FromArrowType<&ArrowTimeUnit> for TimeUnit { + fn from_arrow(value: &ArrowTimeUnit) -> Self { + Self::from_arrow(*value) } } -impl From for TimeUnit { - fn from(value: ArrowTimeUnit) -> Self { +impl FromArrowType for TimeUnit { + fn from_arrow(value: ArrowTimeUnit) -> Self { match value { ArrowTimeUnit::Second => Self::Seconds, ArrowTimeUnit::Millisecond => Self::Milliseconds, @@ -110,18 +108,19 @@ impl From for TimeUnit { } } -impl TryFrom for ArrowTimeUnit { - type Error = VortexError; - - fn try_from(value: TimeUnit) -> VortexResult { - Ok(match value { - TimeUnit::Seconds => Self::Second, - TimeUnit::Milliseconds => Self::Millisecond, - TimeUnit::Microseconds => Self::Microsecond, - TimeUnit::Nanoseconds => Self::Nanosecond, - _ => vortex_bail!("Cannot convert {value} to Arrow TimeUnit"), - }) - } +/// Convert a Vortex [`TimeUnit`] to an Arrow [`ArrowTimeUnit`]. +/// +/// # Errors +/// +/// Returns an error for units with no Arrow equivalent (e.g. [`TimeUnit::Days`]). +pub fn to_arrow_time_unit(value: TimeUnit) -> VortexResult { + Ok(match value { + TimeUnit::Seconds => ArrowTimeUnit::Second, + TimeUnit::Milliseconds => ArrowTimeUnit::Millisecond, + TimeUnit::Microseconds => ArrowTimeUnit::Microsecond, + TimeUnit::Nanoseconds => ArrowTimeUnit::Nanosecond, + _ => vortex_bail!("Cannot convert {value} to Arrow TimeUnit"), + }) } impl FromArrowType for DType { @@ -204,13 +203,14 @@ impl TryFromArrowType<(&DataType, Nullability)> for DType { DType::Extension(Date::new(TimeUnit::Milliseconds, nullability).erased()) } DataType::Time32(unit) => { - DType::Extension(Time::new(unit.into(), nullability).erased()) + DType::Extension(Time::new(TimeUnit::from_arrow(unit), nullability).erased()) } DataType::Time64(unit) => { - DType::Extension(Time::new(unit.into(), nullability).erased()) + DType::Extension(Time::new(TimeUnit::from_arrow(unit), nullability).erased()) } DataType::Timestamp(unit, tz) => DType::Extension( - Timestamp::new_with_tz(unit.into(), tz.clone(), nullability).erased(), + Timestamp::new_with_tz(TimeUnit::from_arrow(unit), tz.clone(), nullability) + .erased(), ), DataType::List(e) | DataType::LargeList(e) @@ -255,14 +255,32 @@ impl TryFromArrowType<&Field> for DType { } } -impl DType { +/// Extension trait converting Vortex [`DType`]s into Arrow schemas and data types. +/// +/// This mirrors inherent methods that lived on [`DType`] before Arrow interoperability moved +/// into this crate. +pub trait ToArrowType { /// Convert a Vortex [`DType`] into an Arrow [`Schema`]. /// /// **Prefer `ArrowSession::to_arrow_schema`.** This method is not plugin-aware and /// strips any `ARROW:extension:name` metadata for non-builtin extensions (only /// `arrow.parquet.variant` is special-cased here). Use the session method when you /// need round-trippable extension metadata. - pub fn to_arrow_schema(&self) -> VortexResult { + fn to_arrow_schema(&self) -> VortexResult; + + /// Returns the Arrow [`DataType`] that best corresponds to this Vortex [`DType`]. + /// + /// **Prefer `ArrowSession::to_arrow_datatype` (or `to_arrow_field`).** This method + /// has no awareness of registered Arrow extension plugins, so any [`DType::Extension`] + /// outside the builtin temporal set will fail or silently lose its + /// `ARROW:extension:name` metadata. The session methods recurse through containers + /// and dispatch plugins at every extension node. + #[deprecated(note = "Use `ArrowSession::to_arrow_datatype` instead")] + fn to_arrow_dtype(&self) -> VortexResult; +} + +impl ToArrowType for DType { + fn to_arrow_schema(&self) -> VortexResult { let DType::Struct(struct_dtype, nullable) = self else { vortex_bail!("only DType::Struct can be converted to arrow schema"); }; @@ -295,15 +313,7 @@ impl DType { Ok(builder.finish()) } - /// Returns the Arrow [`DataType`] that best corresponds to this Vortex [`DType`]. - /// - /// **Prefer `ArrowSession::to_arrow_data_type` (or `to_arrow_field`).** This method - /// has no awareness of registered Arrow extension plugins, so any [`DType::Extension`] - /// outside the builtin temporal set will fail or silently lose its - /// `ARROW:extension:name` metadata. The session methods recurse through containers - /// and dispatch plugins at every extension node. - #[deprecated(note = "Use `ArrowSession::to_arrow_datatype` instead")] - pub fn to_arrow_dtype(&self) -> VortexResult { + fn to_arrow_dtype(&self) -> VortexResult { to_data_type_naive(self) } } @@ -382,7 +392,7 @@ pub(crate) fn to_data_type_naive(dtype: &DType) -> VortexResult { if let Some(temporal) = ext_dtype.metadata_opt::() { return Ok(match temporal { TemporalMetadata::Timestamp(unit, tz) => { - DataType::Timestamp(ArrowTimeUnit::try_from(*unit)?, tz.clone()) + DataType::Timestamp(to_arrow_time_unit(*unit)?, tz.clone()) } TemporalMetadata::Date(unit) => match unit { TimeUnit::Days => DataType::Date32, @@ -425,14 +435,14 @@ mod test { use arrow_schema::Schema; use rstest::fixture; use rstest::rstest; + use vortex_array::dtype::DType; + use vortex_array::dtype::FieldName; + use vortex_array::dtype::FieldNames; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::dtype::StructFields; use super::*; - use crate::dtype::DType; - use crate::dtype::FieldName; - use crate::dtype::FieldNames; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::dtype::StructFields; #[test] fn test_dtype_conversion_success() { @@ -489,7 +499,7 @@ mod test { #[case(39, DataType::Decimal256(39, 0))] #[case(76, DataType::Decimal256(76, 0))] fn test_decimal_dtype_to_arrow(#[case] precision: u8, #[case] expected: DataType) { - use crate::dtype::DecimalDType; + use vortex_array::dtype::DecimalDType; let dtype = DType::Decimal(DecimalDType::new(precision, 0), Nullability::NonNullable); assert_eq!(dtype.to_arrow_dtype().unwrap(), expected); diff --git a/vortex-array/src/arrow/executor/bool.rs b/vortex-arrow/src/executor/bool.rs similarity index 83% rename from vortex-array/src/arrow/executor/bool.rs rename to vortex-arrow/src/executor/bool.rs index 68f2451cd77..64bffa804f2 100644 --- a/vortex-array/src/arrow/executor/bool.rs +++ b/vortex-arrow/src/executor/bool.rs @@ -5,13 +5,13 @@ use std::sync::Arc; use arrow_array::ArrayRef as ArrowArrayRef; use arrow_array::BooleanArray as ArrowBooleanArray; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::bool::BoolArrayExt; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::arrays::BoolArray; -use crate::arrays::bool::BoolArrayExt; -use crate::arrow::null_buffer::to_null_buffer; +use crate::null_buffer::to_null_buffer; /// Convert a canonical BoolArray directly to Arrow. pub fn canonical_bool_to_arrow( diff --git a/vortex-array/src/arrow/executor/byte.rs b/vortex-arrow/src/executor/byte.rs similarity index 91% rename from vortex-array/src/arrow/executor/byte.rs rename to vortex-arrow/src/executor/byte.rs index 83c1c681284..ae40ee01c26 100644 --- a/vortex-array/src/arrow/executor/byte.rs +++ b/vortex-arrow/src/executor/byte.rs @@ -8,22 +8,22 @@ use arrow_array::GenericByteArray; use arrow_array::types::BinaryViewType; use arrow_array::types::ByteArrayType; use arrow_array::types::StringViewType; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::VarBin; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::varbin::VarBinArrayExt; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; use vortex_error::VortexError; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::Canonical; -use crate::ExecutionCtx; -use crate::array::ArrayView; -use crate::arrays::VarBin; -use crate::arrays::VarBinViewArray; -use crate::arrays::varbin::VarBinArrayExt; -use crate::arrow::byte_view::execute_varbinview_to_arrow; -use crate::arrow::executor::validity::to_arrow_null_buffer; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::NativePType; -use crate::dtype::Nullability; +use crate::byte_view::execute_varbinview_to_arrow; +use crate::executor::validity::to_arrow_null_buffer; /// Convert a Vortex array into an Arrow GenericBinaryArray. pub(super) fn to_arrow_byte_array( @@ -81,16 +81,16 @@ mod tests { use arrow_schema::DataType; use arrow_schema::Field; use rstest::rstest; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; use vortex_error::VortexResult; use vortex_mask::Mask; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::array_session; - use crate::arrow::ArrowSessionExt; - use crate::arrow::executor::byte::VarBinViewArray; - use crate::dtype::DType; - use crate::dtype::Nullability; + use crate::ArrowSessionExt; + use crate::executor::byte::VarBinViewArray; fn make_utf8_array() -> VarBinViewArray { VarBinViewArray::from_iter_str(["hello", "world", "this is a longer string for testing"]) diff --git a/vortex-array/src/arrow/executor/byte_view.rs b/vortex-arrow/src/executor/byte_view.rs similarity index 87% rename from vortex-array/src/arrow/executor/byte_view.rs rename to vortex-arrow/src/executor/byte_view.rs index 4e17b7f2376..ffddc49a4dd 100644 --- a/vortex-array/src/arrow/executor/byte_view.rs +++ b/vortex-arrow/src/executor/byte_view.rs @@ -7,16 +7,16 @@ use arrow_array::ArrayRef as ArrowArrayRef; use arrow_array::GenericByteViewArray; use arrow_array::types::ByteViewType; use arrow_buffer::ScalarBuffer; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::arrays::VarBinViewArray; -use crate::arrow::null_buffer::to_null_buffer; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::Nullability; -use crate::dtype::arrow::FromArrowType; +use crate::dtype::FromArrowType; +use crate::null_buffer::to_null_buffer; /// Convert a canonical VarBinViewArray directly to Arrow. pub fn canonical_varbinview_to_arrow( diff --git a/vortex-array/src/arrow/executor/decimal.rs b/vortex-arrow/src/executor/decimal.rs similarity index 92% rename from vortex-array/src/arrow/executor/decimal.rs rename to vortex-arrow/src/executor/decimal.rs index a9aa9fdd2f5..d5eabc77145 100644 --- a/vortex-array/src/arrow/executor/decimal.rs +++ b/vortex-arrow/src/executor/decimal.rs @@ -13,15 +13,15 @@ use arrow_schema::DataType; use itertools::Itertools; use num_traits::AsPrimitive; use num_traits::ToPrimitive; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::DecimalArray; +use vortex_array::dtype::DecimalType; use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_error::vortex_err; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::arrays::DecimalArray; -use crate::arrow::null_buffer::to_null_buffer; -use crate::dtype::DecimalType; +use crate::null_buffer::to_null_buffer; pub(super) fn to_arrow_decimal( array: ArrayRef, @@ -72,7 +72,7 @@ fn to_arrow_decimal32(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResu }) .process_results(|iter| Buffer::from_trusted_len_iter(iter))?, DecimalType::I256 => array - .buffer::() + .buffer::() .into_iter() .map(|x| { x.to_i32() @@ -116,7 +116,7 @@ fn to_arrow_decimal64(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResu }) .process_results(|iter| Buffer::from_trusted_len_iter(iter))?, DecimalType::I256 => array - .buffer::() + .buffer::() .into_iter() .map(|x| { x.to_i64() @@ -155,7 +155,7 @@ fn to_arrow_decimal128(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexRes } DecimalType::I128 => array.buffer::(), DecimalType::I256 => array - .buffer::() + .buffer::() .into_iter() .map(|x| { x.to_i128() @@ -196,7 +196,7 @@ fn to_arrow_decimal256(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexRes array .buffer::() .into_iter() - .map(|x| crate::dtype::i256::from_i128(x).into()), + .map(|x| vortex_array::dtype::i256::from_i128(x).into()), ), DecimalType::I256 => { Buffer::::from_byte_buffer(array.buffer_handle().clone().into_host_sync()) @@ -219,19 +219,19 @@ mod tests { use arrow_buffer::i256; use arrow_schema::DataType; use rstest::rstest; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::builders::ArrayBuilder; + use vortex_array::builders::DecimalBuilder; + use vortex_array::dtype::DecimalDType; + use vortex_array::dtype::NativeDecimalType; + use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; - use crate::VortexSessionExecute; - use crate::array::IntoArray; - use crate::array_session; - use crate::arrow::ArrowArrayExecutor; - use crate::arrow::executor::decimal::DecimalArray; - use crate::builders::ArrayBuilder; - use crate::builders::DecimalBuilder; - use crate::dtype::DecimalDType; - use crate::dtype::NativeDecimalType; - use crate::validity::Validity; + use crate::ArrowArrayExecutor; + use crate::executor::decimal::DecimalArray; #[test] fn decimal_to_arrow() -> VortexResult<()> { @@ -260,7 +260,7 @@ mod tests { #[case(0i32)] #[case(0i64)] #[case(0i128)] - #[case(crate::dtype::i256::ZERO)] + #[case(vortex_array::dtype::i256::ZERO)] fn test_to_arrow_decimal128( #[case] _decimal_type: T, ) -> VortexResult<()> { @@ -288,7 +288,7 @@ mod tests { #[case(0i32)] #[case(0i64)] #[case(0i128)] - #[case(crate::dtype::i256::ZERO)] + #[case(vortex_array::dtype::i256::ZERO)] fn test_to_arrow_decimal32(#[case] _decimal_type: T) -> VortexResult<()> { use arrow_array::Decimal32Array; @@ -316,7 +316,7 @@ mod tests { #[case(0i32)] #[case(0i64)] #[case(0i128)] - #[case(crate::dtype::i256::ZERO)] + #[case(vortex_array::dtype::i256::ZERO)] fn test_to_arrow_decimal64(#[case] _decimal_type: T) -> VortexResult<()> { use arrow_array::Decimal64Array; @@ -344,7 +344,7 @@ mod tests { #[case(0i32)] #[case(0i64)] #[case(0i128)] - #[case(crate::dtype::i256::ZERO)] + #[case(vortex_array::dtype::i256::ZERO)] fn test_to_arrow_decimal256( #[case] _decimal_type: T, ) -> VortexResult<()> { diff --git a/vortex-array/src/arrow/executor/dictionary.rs b/vortex-arrow/src/executor/dictionary.rs similarity index 87% rename from vortex-array/src/arrow/executor/dictionary.rs rename to vortex-arrow/src/executor/dictionary.rs index 97d25c3f7cc..f5be1076a4b 100644 --- a/vortex-array/src/arrow/executor/dictionary.rs +++ b/vortex-arrow/src/executor/dictionary.rs @@ -10,19 +10,19 @@ use arrow_array::cast::AsArray; use arrow_array::new_null_array; use arrow_array::types::*; use arrow_schema::DataType; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::Constant; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::Dict; +use vortex_array::arrays::DictArray; +use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::Constant; -use crate::arrays::ConstantArray; -use crate::arrays::Dict; -use crate::arrays::DictArray; -use crate::arrays::dict::DictArraySlotsExt; -use crate::arrow::ArrowArrayExecutor; +use crate::ArrowArrayExecutor; pub(super) fn to_arrow_dictionary( array: ArrayRef, @@ -145,30 +145,33 @@ mod tests { use arrow_array::types::UInt32Type; use arrow_schema::DataType; use rstest::rstest; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::VarBinViewArray; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability::Nullable; + use vortex_array::scalar::Scalar; use vortex_buffer::buffer; use vortex_error::VortexResult; - use crate::IntoArray; - use crate::array_session; - use crate::arrays::PrimitiveArray; - use crate::arrays::VarBinViewArray; - use crate::arrow::ArrowArrayExecutor; - use crate::arrow::executor::dictionary::ConstantArray; - use crate::arrow::executor::dictionary::DictArray; - use crate::dtype::DType; - use crate::dtype::Nullability::Nullable; - use crate::executor::VortexSessionExecute; - use crate::scalar::Scalar; + use crate::ArrowArrayExecutor; + use crate::executor::dictionary::ConstantArray; + use crate::executor::dictionary::DictArray; fn dict_type(codes: DataType, values: DataType) -> DataType { DataType::Dictionary(Box::new(codes), Box::new(values)) } - fn execute(array: crate::ArrayRef, dt: &DataType) -> VortexResult { + fn execute( + array: vortex_array::ArrayRef, + dt: &DataType, + ) -> VortexResult { array.execute_arrow(Some(dt), &mut array_session().create_execution_ctx()) } - fn dict_basic_input() -> crate::ArrayRef { + fn dict_basic_input() -> vortex_array::ArrayRef { DictArray::try_new( buffer![0u8, 1, 0].into_array(), VarBinViewArray::from_iter_str(["a", "b"]).into_array(), @@ -177,7 +180,7 @@ mod tests { .into_array() } - fn dict_with_null_codes_input() -> crate::ArrayRef { + fn dict_with_null_codes_input() -> vortex_array::ArrayRef { DictArray::try_new( PrimitiveArray::from_option_iter(vec![Some(0u8), None, Some(1)]).into_array(), VarBinViewArray::from_iter_str(["a", "b"]).into_array(), @@ -213,7 +216,7 @@ mod tests { Arc::new(vec![Some("a"), None, Some("a"), Some("b"), Some("a")].into_iter().collect::>()) as arrow_array::ArrayRef, )] fn to_arrow_dictionary( - #[case] input: crate::ArrayRef, + #[case] input: vortex_array::ArrayRef, #[case] target_type: DataType, #[case] expected: arrow_array::ArrayRef, ) -> VortexResult<()> { diff --git a/vortex-array/src/arrow/executor/fixed_size_list.rs b/vortex-arrow/src/executor/fixed_size_list.rs similarity index 86% rename from vortex-array/src/arrow/executor/fixed_size_list.rs rename to vortex-arrow/src/executor/fixed_size_list.rs index caeb5804da2..72ce7c57ff0 100644 --- a/vortex-array/src/arrow/executor/fixed_size_list.rs +++ b/vortex-arrow/src/executor/fixed_size_list.rs @@ -4,16 +4,16 @@ use std::sync::Arc; use arrow_schema::FieldRef; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::FixedSizeList; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_error::VortexResult; use vortex_error::vortex_ensure; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::arrays::FixedSizeList; -use crate::arrays::FixedSizeListArray; -use crate::arrays::fixed_size_list::FixedSizeListArrayExt; -use crate::arrow::executor::validity::to_arrow_null_buffer; -use crate::arrow::session::ArrowSessionExt; +use crate::executor::validity::to_arrow_null_buffer; +use crate::session::ArrowSessionExt; pub(super) fn to_arrow_fixed_list( array: ArrayRef, diff --git a/vortex-array/src/arrow/executor/list.rs b/vortex-arrow/src/executor/list.rs similarity index 90% rename from vortex-array/src/arrow/executor/list.rs rename to vortex-arrow/src/executor/list.rs index 7bf7a16f496..0b0d7f625b5 100644 --- a/vortex-array/src/arrow/executor/list.rs +++ b/vortex-arrow/src/executor/list.rs @@ -8,30 +8,30 @@ use arrow_array::GenericListArray; use arrow_array::OffsetSizeTrait; use arrow_buffer::OffsetBuffer; use arrow_schema::FieldRef; +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::Chunked; +use vortex_array::arrays::List; +use vortex_array::arrays::ListArray; +use vortex_array::arrays::ListView; +use vortex_array::arrays::ListViewArray; +use vortex_array::arrays::chunked::ChunkedArrayExt; +use vortex_array::arrays::list::ListArrayExt; +use vortex_array::arrays::listview::ListViewArrayExt; +use vortex_array::arrays::listview::ListViewDataParts; +use vortex_array::arrays::listview::ListViewRebuildMode; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; -use crate::ArrayRef; -use crate::Canonical; -use crate::ExecutionCtx; -use crate::arrays::Chunked; -use crate::arrays::List; -use crate::arrays::ListArray; -use crate::arrays::ListView; -use crate::arrays::ListViewArray; -use crate::arrays::chunked::ChunkedArrayExt; -use crate::arrays::list::ListArrayExt; -use crate::arrays::listview::ListViewArrayExt; -use crate::arrays::listview::ListViewDataParts; -use crate::arrays::listview::ListViewRebuildMode; -use crate::arrow::executor::validity::to_arrow_null_buffer; -use crate::arrow::session::ArrowSessionExt; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::NativePType; -use crate::dtype::Nullability; +use crate::executor::validity::to_arrow_null_buffer; +use crate::session::ArrowSessionExt; #[allow(rustdoc::broken_intra_doc_links)] /// Convert a Vortex VarBinArray into an Arrow [`GenericListArray`](arrow_array:array::GenericListArray). @@ -210,22 +210,22 @@ mod tests { use arrow_array::Int32Array; use arrow_schema::DataType; use arrow_schema::Field; + use vortex_array::Canonical; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability::NonNullable; + use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::Canonical; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrays::PrimitiveArray; - use crate::arrow::ArrowArrayExecutor; - use crate::arrow::executor::list::ListViewArray; - use crate::dtype::DType; - use crate::dtype::Nullability::NonNullable; - use crate::validity::Validity; + use crate::ArrowArrayExecutor; + use crate::executor::list::ListViewArray; /// A shared session for these list-executor tests, used to create execution contexts. - static SESSION: LazyLock = LazyLock::new(crate::array_session); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_to_arrow_list_i32() -> VortexResult<()> { @@ -368,7 +368,10 @@ mod tests { fn test_to_arrow_list_empty_zctl() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); let dtype = DType::List( - Arc::new(DType::Primitive(crate::dtype::PType::I32, NonNullable)), + Arc::new(DType::Primitive( + vortex_array::dtype::PType::I32, + NonNullable, + )), NonNullable, ); let list_array = unsafe { diff --git a/vortex-array/src/arrow/executor/list_view.rs b/vortex-arrow/src/executor/list_view.rs similarity index 89% rename from vortex-array/src/arrow/executor/list_view.rs rename to vortex-arrow/src/executor/list_view.rs index 7d5666b3762..da242eb9719 100644 --- a/vortex-array/src/arrow/executor/list_view.rs +++ b/vortex-arrow/src/executor/list_view.rs @@ -6,24 +6,24 @@ use std::sync::Arc; use arrow_array::GenericListViewArray; use arrow_array::OffsetSizeTrait; use arrow_schema::FieldRef; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::ListViewArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::listview::DEFAULT_REBUILD_DENSITY_THRESHOLD; +use vortex_array::arrays::listview::DEFAULT_TRIM_ELEMENTS_THRESHOLD; +use vortex_array::arrays::listview::ListViewArrayExt; +use vortex_array::arrays::listview::ListViewDataParts; +use vortex_array::arrays::listview::ListViewRebuildMode; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::IntegerPType; +use vortex_array::dtype::Nullability::NonNullable; use vortex_error::VortexResult; use vortex_error::vortex_ensure; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::arrays::ListViewArray; -use crate::arrays::PrimitiveArray; -use crate::arrays::listview::DEFAULT_REBUILD_DENSITY_THRESHOLD; -use crate::arrays::listview::DEFAULT_TRIM_ELEMENTS_THRESHOLD; -use crate::arrays::listview::ListViewArrayExt; -use crate::arrays::listview::ListViewDataParts; -use crate::arrays::listview::ListViewRebuildMode; -use crate::arrow::executor::validity::to_arrow_null_buffer; -use crate::arrow::session::ArrowSessionExt; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::IntegerPType; -use crate::dtype::Nullability::NonNullable; +use crate::executor::validity::to_arrow_null_buffer; +use crate::session::ArrowSessionExt; pub(super) fn to_arrow_list_view( array: ArrayRef, @@ -115,19 +115,19 @@ mod tests { use arrow_array::GenericListViewArray; use arrow_schema::DataType; use arrow_schema::Field; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::arrow::ArrowArrayExecutor; - use crate::arrow::executor::list_view::ListViewArray; - use crate::arrow::executor::list_view::PrimitiveArray; - use crate::validity::Validity; + use crate::ArrowArrayExecutor; + use crate::executor::list_view::ListViewArray; + use crate::executor::list_view::PrimitiveArray; /// A shared session for these list-view-executor tests, used to create execution contexts. - static SESSION: LazyLock = LazyLock::new(crate::array_session); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn trims_zero_copy_with_significant_trailing_waste() -> VortexResult<()> { diff --git a/vortex-array/src/arrow/executor/mod.rs b/vortex-arrow/src/executor/mod.rs similarity index 88% rename from vortex-array/src/arrow/executor/mod.rs rename to vortex-arrow/src/executor/mod.rs index 49bd87bcbdf..875577788b4 100644 --- a/vortex-array/src/arrow/executor/mod.rs +++ b/vortex-arrow/src/executor/mod.rs @@ -30,35 +30,35 @@ use arrow_schema::Field; use arrow_schema::FieldRef; use arrow_schema::Schema; use itertools::Itertools; -use vortex_array::dtype::arrow::to_data_type_naive; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::List; +use vortex_array::arrays::VarBin; +use vortex_array::arrays::list::ListArrayExt; +use vortex_array::arrays::varbin::VarBinArrayExt; +use vortex_array::dtype::DType; +use vortex_array::dtype::PType; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; -use crate::ArrayRef; -use crate::arrays::List; -use crate::arrays::VarBin; -use crate::arrays::list::ListArrayExt; -use crate::arrays::varbin::VarBinArrayExt; -use crate::arrow::executor::bool::to_arrow_bool; -use crate::arrow::executor::byte::to_arrow_byte_array; -use crate::arrow::executor::byte_view::to_arrow_byte_view; -use crate::arrow::executor::decimal::to_arrow_decimal; -use crate::arrow::executor::dictionary::to_arrow_dictionary; -use crate::arrow::executor::fixed_size_list::to_arrow_fixed_list; -use crate::arrow::executor::list::to_arrow_list; -use crate::arrow::executor::list_view::to_arrow_list_view; -use crate::arrow::executor::null::to_arrow_null; -use crate::arrow::executor::primitive::to_arrow_primitive; -use crate::arrow::executor::run_end::to_arrow_run_end; -use crate::arrow::executor::struct_::to_arrow_struct; -use crate::arrow::executor::temporal::to_arrow_date; -use crate::arrow::executor::temporal::to_arrow_time; -use crate::arrow::executor::temporal::to_arrow_timestamp; -use crate::arrow::session::ArrowSessionExt; -use crate::dtype::DType; -use crate::dtype::PType; -use crate::executor::ExecutionCtx; +use crate::dtype::to_data_type_naive; +use crate::executor::bool::to_arrow_bool; +use crate::executor::byte::to_arrow_byte_array; +use crate::executor::byte_view::to_arrow_byte_view; +use crate::executor::decimal::to_arrow_decimal; +use crate::executor::dictionary::to_arrow_dictionary; +use crate::executor::fixed_size_list::to_arrow_fixed_list; +use crate::executor::list::to_arrow_list; +use crate::executor::list_view::to_arrow_list_view; +use crate::executor::null::to_arrow_null; +use crate::executor::primitive::to_arrow_primitive; +use crate::executor::run_end::to_arrow_run_end; +use crate::executor::struct_::to_arrow_struct; +use crate::executor::temporal::to_arrow_date; +use crate::executor::temporal::to_arrow_time; +use crate::executor::temporal::to_arrow_timestamp; +use crate::session::ArrowSessionExt; /// Trait for executing a Vortex array to produce an Arrow array. #[deprecated(note = "Use an `ArrowSession` to perform conversions to/from Arrow arrays")] diff --git a/vortex-array/src/arrow/executor/null.rs b/vortex-arrow/src/executor/null.rs similarity index 86% rename from vortex-array/src/arrow/executor/null.rs rename to vortex-arrow/src/executor/null.rs index 9d2b5ef6fc4..c5ffc991006 100644 --- a/vortex-array/src/arrow/executor/null.rs +++ b/vortex-arrow/src/executor/null.rs @@ -5,12 +5,11 @@ use std::sync::Arc; use arrow_array::ArrayRef as ArrowArrayRef; use arrow_array::NullArray as ArrowNullArray; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::NullArray; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::arrays::NullArray; - /// Convert a canonical NullArray directly to Arrow. pub fn canonical_null_to_arrow(array: &NullArray) -> ArrowArrayRef { Arc::new(ArrowNullArray::new(array.len())) diff --git a/vortex-array/src/arrow/executor/primitive.rs b/vortex-arrow/src/executor/primitive.rs similarity index 81% rename from vortex-array/src/arrow/executor/primitive.rs rename to vortex-arrow/src/executor/primitive.rs index 0d41176972b..6cea08c0a72 100644 --- a/vortex-array/src/arrow/executor/primitive.rs +++ b/vortex-arrow/src/executor/primitive.rs @@ -6,16 +6,16 @@ use std::sync::Arc; use arrow_array::ArrayRef as ArrowArrayRef; use arrow_array::ArrowPrimitiveType; use arrow_array::PrimitiveArray as ArrowPrimitiveArray; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::arrays::PrimitiveArray; -use crate::arrow::null_buffer::to_null_buffer; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::NativePType; -use crate::dtype::Nullability; +use crate::null_buffer::to_null_buffer; /// Convert a canonical PrimitiveArray directly to Arrow. pub fn canonical_primitive_to_arrow( diff --git a/vortex-array/src/arrow/executor/run_end.rs b/vortex-arrow/src/executor/run_end.rs similarity index 76% rename from vortex-array/src/arrow/executor/run_end.rs rename to vortex-arrow/src/executor/run_end.rs index 0632f871f29..b6913c1ea1c 100644 --- a/vortex-array/src/arrow/executor/run_end.rs +++ b/vortex-arrow/src/executor/run_end.rs @@ -11,35 +11,20 @@ use arrow_array::types::*; use arrow_buffer::ArrowNativeType; use arrow_schema::DataType; use arrow_schema::Field; -use prost::Message; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::Constant; +use vortex_array::arrays::ConstantArray; use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; use vortex_error::vortex_err; +use vortex_runend::RunEnd; +use vortex_runend::RunEndArray; +use vortex_runend::RunEndArrayExt; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::Constant; -use crate::arrays::ConstantArray; -use crate::arrow::ArrowArrayExecutor; -use crate::session::ArraySessionExt; - -/// The encoding ID used by `vortex-runend`. We match on this string to avoid a crate dependency. -const VORTEX_RUNEND_ID: &str = "vortex.runend"; - -/// Mirror of `RunEndMetadata` from the `vortex-runend` crate. Same prost field tags so we can -/// decode the serialized metadata without depending on that crate. -#[derive(Clone, prost::Message)] -struct RunEndMetadata { - #[prost(int32, tag = "1")] - pub ends_ptype: i32, - #[prost(uint64, tag = "2")] - pub num_runs: u64, - #[prost(uint64, tag = "3")] - pub offset: u64, -} +use crate::ArrowArrayExecutor; pub(super) fn to_arrow_run_end( array: ArrayRef, @@ -57,47 +42,32 @@ pub(super) fn to_arrow_run_end( // Execute to unwrap any wrapper VTables (Slice, Filter, etc.) which may // reveal a RunEndArray. let array = array.execute::(ctx)?; - if array.encoding_id().as_ref() == VORTEX_RUNEND_ID { - // NOTE(ngates): while this module still lives in vortex-array, we cannot depend on the - // vortex-runend crate. Therefore, we match on the encoding ID string and extract the children - // and metadata directly. - return run_end_to_arrow(array, ends_type, values_type, ctx); + match array.try_downcast::() { + Ok(run_end) => run_end_to_arrow(run_end, ends_type, values_type, ctx), + Err(array) => { + // Fallback: canonicalize to flat Arrow, then cast to REE. + let flat = array.execute_arrow(Some(values_type.data_type()), ctx)?; + let ree_type = DataType::RunEndEncoded( + Arc::new(Field::new("run_ends", ends_type.clone(), false)), + Arc::new(values_type.clone()), + ); + arrow_cast::cast(&flat, &ree_type).map_err(VortexError::from) + } } - - // Fallback: canonicalize to flat Arrow, then cast to REE. - let flat = array.execute_arrow(Some(values_type.data_type()), ctx)?; - let ree_type = DataType::RunEndEncoded( - Arc::new(Field::new("run_ends", ends_type.clone(), false)), - Arc::new(values_type.clone()), - ); - arrow_cast::cast(&flat, &ree_type).map_err(VortexError::from) } fn run_end_to_arrow( - array: ArrayRef, + array: RunEndArray, ends_type: &DataType, values_type: &Field, ctx: &mut ExecutionCtx, ) -> VortexResult { let length = array.len(); - let metadata_bytes = ctx - .session() - .array_serialize(&array)? - .ok_or_else(|| vortex_err!("RunEndArray missing metadata"))?; - let metadata = RunEndMetadata::decode(&*metadata_bytes) - .map_err(|e| vortex_err!("Failed to decode RunEndMetadata: {e}"))?; - let offset = usize::try_from(metadata.offset) - .map_err(|_| vortex_err!("RunEndArray offset {} overflows usize", metadata.offset))?; - - let children = array.children(); - vortex_ensure!( - children.len() == 2, - "Expected RunEndArray to have 2 children, got {}", - children.len() - ); - - let arrow_ends = children[0].clone().execute_arrow(Some(ends_type), ctx)?; - let arrow_values = children[1] + let offset = array.offset(); + + let arrow_ends = array.ends().clone().execute_arrow(Some(ends_type), ctx)?; + let arrow_values = array + .values() .clone() .execute_arrow(Some(values_type.data_type()), ctx)?; @@ -204,21 +174,21 @@ mod tests { use arrow_schema::DataType; use arrow_schema::Field; use rstest::rstest; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability::Nullable; + use vortex_array::dtype::PType; + use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_session::VortexSession; - use crate::IntoArray; - use crate::arrays::PrimitiveArray; - use crate::arrow::ArrowArrayExecutor; - use crate::arrow::executor::run_end::ConstantArray; - use crate::dtype::DType; - use crate::dtype::Nullability::Nullable; - use crate::dtype::PType; - use crate::executor::VortexSessionExecute; - use crate::scalar::Scalar; + use crate::ArrowArrayExecutor; + use crate::executor::run_end::ConstantArray; - static SESSION: LazyLock = LazyLock::new(crate::array_session); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn ree_type(ends: DataType, values_dtype: DataType) -> DataType { DataType::RunEndEncoded( @@ -227,7 +197,10 @@ mod tests { ) } - fn execute(array: crate::ArrayRef, dt: &DataType) -> VortexResult { + fn execute( + array: vortex_array::ArrayRef, + dt: &DataType, + ) -> VortexResult { array.execute_arrow(Some(dt), &mut SESSION.create_execution_ctx()) } @@ -279,7 +252,7 @@ mod tests { ), )] fn constant_to_ree( - #[case] input: crate::ArrayRef, + #[case] input: vortex_array::ArrayRef, #[case] target_type: DataType, #[case] expected: arrow_array::ArrayRef, ) -> VortexResult<()> { diff --git a/vortex-array/src/arrow/executor/struct_.rs b/vortex-arrow/src/executor/struct_.rs similarity index 89% rename from vortex-array/src/arrow/executor/struct_.rs rename to vortex-arrow/src/executor/struct_.rs index d3b7bcda2dc..e2ce2700d4a 100644 --- a/vortex-array/src/arrow/executor/struct_.rs +++ b/vortex-arrow/src/executor/struct_.rs @@ -9,27 +9,27 @@ use arrow_buffer::NullBuffer; use arrow_schema::Field; use arrow_schema::Fields; use itertools::Itertools; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::Chunked; +use vortex_array::arrays::ScalarFn; +use vortex_array::arrays::Struct; +use vortex_array::arrays::StructArray; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::struct_::StructDataParts; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::FieldNames; +use vortex_array::dtype::StructFields; +use vortex_array::scalar_fn::fns::pack::Pack; use vortex_error::VortexResult; use vortex_error::vortex_ensure; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::Chunked; -use crate::arrays::ScalarFn; -use crate::arrays::Struct; -use crate::arrays::StructArray; -use crate::arrays::scalar_fn::ScalarFnArrayExt; -use crate::arrays::struct_::StructDataParts; -use crate::arrow::ArrowArrayExecutor; -use crate::arrow::executor::validity::to_arrow_null_buffer; -use crate::arrow::session::ArrowSessionExt; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::FieldNames; -use crate::dtype::StructFields; -use crate::dtype::arrow::FromArrowType; -use crate::scalar_fn::fns::pack::Pack; +use crate::ArrowArrayExecutor; +use crate::dtype::FromArrowType; +use crate::executor::validity::to_arrow_null_buffer; +use crate::session::ArrowSessionExt; pub(super) fn to_arrow_struct( array: ArrayRef, @@ -91,7 +91,7 @@ pub(super) fn to_arrow_struct( // We apply a cast to ensure we push down casting where possible into the struct fields. array.cast(DType::Struct( vx_fields, - crate::dtype::Nullability::Nullable, + vortex_array::dtype::Nullability::Nullable, ))? } else { array @@ -204,20 +204,21 @@ mod tests { use arrow_buffer::NullBuffer; use arrow_schema::DataType; use arrow_schema::Field; + use vortex_array as array; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::arrays; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::StructArray; + use vortex_array::dtype::FieldNames; + use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; - use crate::IntoArray; - use crate::VortexSessionExecute; - use crate::array; - use crate::array_session; - use crate::arrays; - use crate::arrays::PrimitiveArray; - use crate::arrays::StructArray; - use crate::arrow::ArrowArrayExecutor; - use crate::arrow::FromArrowArray; - use crate::dtype::FieldNames; - use crate::validity::Validity; + use crate::ArrowArrayExecutor; + use crate::FromArrowArray; + use crate::dtype::to_data_type_naive; #[test] fn struct_nullable_non_null_to_arrow() -> VortexResult<()> { @@ -328,7 +329,7 @@ mod tests { ), ])?); - let arrow_dtype = array.dtype().to_arrow_dtype()?; + let arrow_dtype = to_data_type_naive(array.dtype())?; assert_eq!( &array .into_array() diff --git a/vortex-array/src/arrow/executor/temporal.rs b/vortex-arrow/src/executor/temporal.rs similarity index 92% rename from vortex-array/src/arrow/executor/temporal.rs rename to vortex-arrow/src/executor/temporal.rs index bed70f8baec..8af9aa0b2c4 100644 --- a/vortex-array/src/arrow/executor/temporal.rs +++ b/vortex-arrow/src/executor/temporal.rs @@ -26,20 +26,20 @@ use arrow_array::types::TimestampNanosecondType; use arrow_array::types::TimestampSecondType; use arrow_schema::DataType; use arrow_schema::TimeUnit as ArrowTimeUnit; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; +use vortex_array::extension::datetime::AnyTemporal; +use vortex_array::extension::datetime::TemporalMetadata; +use vortex_array::extension::datetime::TimeUnit; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::arrays::PrimitiveArray; -use crate::arrow::null_buffer::to_null_buffer; -use crate::builtins::ArrayBuiltins; -use crate::dtype::DType; -use crate::dtype::NativePType; -use crate::dtype::Nullability; -use crate::extension::datetime::AnyTemporal; -use crate::extension::datetime::TemporalMetadata; -use crate::extension::datetime::TimeUnit; +use crate::null_buffer::to_null_buffer; pub(super) fn to_arrow_date( array: ArrayRef, @@ -168,7 +168,7 @@ fn validate_temporal_extension(array: &ArrayRef, target: &DataType) -> VortexRes Ok(()) } (TemporalMetadata::Timestamp(unit, src_tz), DataType::Timestamp(arrow_unit, tgt_tz)) => { - let src_arrow_unit = ArrowTimeUnit::try_from(*unit)?; + let src_arrow_unit = crate::dtype::to_arrow_time_unit(*unit)?; if src_arrow_unit != *arrow_unit { vortex_bail!( "Cannot convert Timestamp({unit}) to Arrow Timestamp({arrow_unit:?}): unit mismatch" diff --git a/vortex-array/src/arrow/executor/validity.rs b/vortex-arrow/src/executor/validity.rs similarity index 61% rename from vortex-array/src/arrow/executor/validity.rs rename to vortex-arrow/src/executor/validity.rs index fa848dfc79a..ab8a88b5ee3 100644 --- a/vortex-array/src/arrow/executor/validity.rs +++ b/vortex-arrow/src/executor/validity.rs @@ -1,4 +1,4 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -pub(super) use crate::arrow::null_buffer::to_arrow_null_buffer; +pub(super) use crate::null_buffer::to_arrow_null_buffer; diff --git a/vortex-array/src/arrow/iter.rs b/vortex-arrow/src/iter.rs similarity index 86% rename from vortex-array/src/arrow/iter.rs rename to vortex-arrow/src/iter.rs index f05bb0a4e7b..0d7ebe5c82a 100644 --- a/vortex-array/src/arrow/iter.rs +++ b/vortex-arrow/src/iter.rs @@ -2,14 +2,14 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use arrow_array::ffi_stream; +use vortex_array::ArrayRef; +use vortex_array::dtype::DType; +use vortex_array::iter::ArrayIterator; use vortex_error::VortexError; use vortex_error::VortexResult; -use crate::ArrayRef; -use crate::arrow::FromArrowArray; -use crate::dtype::DType; -use crate::dtype::arrow::FromArrowType; -use crate::iter::ArrayIterator; +use crate::FromArrowArray; +use crate::dtype::FromArrowType; /// An adapter for converting an `ArrowArrayStreamReader` into a Vortex `ArrayStream`. pub struct ArrowArrayStreamAdapter { diff --git a/vortex-array/src/arrow/mod.rs b/vortex-arrow/src/lib.rs similarity index 64% rename from vortex-array/src/arrow/mod.rs rename to vortex-arrow/src/lib.rs index feda2a5f4d8..73fa4c724ce 100644 --- a/vortex-array/src/arrow/mod.rs +++ b/vortex-arrow/src/lib.rs @@ -1,38 +1,62 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Utilities to work with `Arrow` data and types. +//! Apache Arrow interoperability for Vortex. +//! +//! This crate owns every conversion between Vortex and Arrow: importing Arrow arrays, record +//! batches, schemas, and fields into Vortex ([`FromArrowArray`], [`FromArrowType`], +//! [`TryFromArrowType`]), and executing Vortex arrays into Arrow ([`ArrowSession::execute_arrow`] +//! and the [`ArrowArrayExecutor`] convenience trait). +//! +//! The [`ArrowSession`] session variable is created lazily on first use of +//! [`ArrowSessionExt::arrow`], so no explicit registration is required for plain conversions. use arrow_array::ArrayRef as ArrowArrayRef; use arrow_schema::DataType; +use vortex_array::ArrayRef; +use vortex_array::VortexSessionExecute; +use vortex_array::legacy_session; use vortex_error::VortexResult; +use vortex_session::VortexSession; mod convert; mod datum; +pub mod dtype; mod executor; mod iter; mod null_buffer; +mod run_end_import; +mod scalar; mod session; +mod uuid; +pub use convert::IntoVortexArray; pub(crate) use convert::nulls; pub use datum::*; +pub use dtype::FromArrowType; +pub use dtype::ToArrowType; +pub use dtype::TryFromArrowType; pub use executor::*; pub use iter::*; pub use null_buffer::to_arrow_null_buffer; pub use null_buffer::to_null_buffer; +pub use scalar::ToArrowDatum; pub use session::*; -use crate::ArrayRef; -use crate::VortexSessionExecute; -use crate::legacy_session; +/// Register vortex-arrow's session extensions into `session`. +/// +/// This eagerly registers the [`ArrowSession`], which is otherwise created lazily on first use. +pub fn initialize(session: &VortexSession) { + session.register(ArrowSession::default()); +} /// Construct a Vortex array from an Arrow array (or other Arrow container) of type `A`. /// /// Implementations reuse the underlying Arrow buffers without copying wherever the Arrow and /// Vortex memory layouts allow it. pub trait FromArrowArray { - /// Convert `array` into a Vortex array whose [`DType`](crate::dtype::DType) has the requested - /// `nullable` [`Nullability`](crate::dtype::Nullability). + /// Convert `array` into a Vortex array whose [`DType`](vortex_array::dtype::DType) has the requested + /// `nullable` [`Nullability`](vortex_array::dtype::Nullability). /// /// An Arrow array can carry a validity (null) buffer regardless of whether its schema declares /// the field nullable, so the desired nullability is supplied separately by the caller @@ -64,7 +88,7 @@ pub trait IntoArrowArray { #[allow(deprecated)] impl IntoArrowArray for ArrayRef { - /// Convert this [`crate::ArrayRef`] into an Arrow [`crate::ArrayRef`] by using the array's + /// Convert this [`vortex_array::ArrayRef`] into an Arrow [`ArrowArrayRef`] by using the array's /// preferred (cheapest) Arrow [`DataType`]. #[allow(clippy::disallowed_methods)] fn into_arrow_preferred(self) -> VortexResult { diff --git a/vortex-array/src/arrow/null_buffer.rs b/vortex-arrow/src/null_buffer.rs similarity index 93% rename from vortex-array/src/arrow/null_buffer.rs rename to vortex-arrow/src/null_buffer.rs index 484dd5e55bc..b50adb30cd2 100644 --- a/vortex-array/src/arrow/null_buffer.rs +++ b/vortex-arrow/src/null_buffer.rs @@ -3,12 +3,11 @@ use arrow_buffer::BooleanBuffer; use arrow_buffer::NullBuffer; +use vortex_array::ExecutionCtx; +use vortex_array::validity::Validity; use vortex_error::VortexResult; use vortex_mask::Mask; -use crate::ExecutionCtx; -use crate::validity::Validity; - /// Converts a [`Validity`] to an Arrow [`NullBuffer`], executing the validity array if needed. pub fn to_arrow_null_buffer( validity: Validity, @@ -38,7 +37,7 @@ mod tests { use vortex_buffer::BitBuffer; use vortex_mask::Mask; - use crate::arrow::null_buffer::to_null_buffer; + use crate::null_buffer::to_null_buffer; #[test] fn test_mask_to_null_buffer() { diff --git a/encodings/runend/src/arrow.rs b/vortex-arrow/src/run_end_import.rs similarity index 96% rename from encodings/runend/src/arrow.rs rename to vortex-arrow/src/run_end_import.rs index 1f58e21bef2..69deba71fdf 100644 --- a/encodings/runend/src/arrow.rs +++ b/vortex-arrow/src/run_end_import.rs @@ -8,16 +8,16 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::NativePType; use vortex_array::legacy_session; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; +use vortex_runend::RunEndData; +use vortex_runend::ops::find_physical_index; +use vortex_runend::ops::find_slice_end_index; -use crate::RunEndData; -use crate::ops::find_physical_index; -use crate::ops::find_slice_end_index; +use crate::FromArrowArray; impl FromArrowArray<&RunArray> for RunEndData where @@ -80,8 +80,6 @@ mod tests { use vortex_array::VortexSessionExecute as _; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; - use vortex_array::arrow::ArrowSessionExt; - use vortex_array::arrow::FromArrowArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; @@ -91,16 +89,18 @@ mod tests { use vortex_buffer::Buffer; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_runend::RunEnd; + use vortex_runend::RunEndArray; + use vortex_runend::ops::find_physical_index; + use vortex_runend::ops::find_slice_end_index; use vortex_session::VortexSession; - use crate::RunEnd; - use crate::RunEndArray; - use crate::ops::find_physical_index; - use crate::ops::find_slice_end_index; + use crate::ArrowSessionExt; + use crate::FromArrowArray; static SESSION: LazyLock = LazyLock::new(|| { let session = vortex_array::array_session(); - crate::initialize(&session); + vortex_runend::initialize(&session); session }); diff --git a/vortex-array/src/scalar/arrow.rs b/vortex-arrow/src/scalar.rs similarity index 83% rename from vortex-array/src/scalar/arrow.rs rename to vortex-arrow/src/scalar.rs index cca6749cb19..aeac56f43d3 100644 --- a/vortex-array/src/scalar/arrow.rs +++ b/vortex-arrow/src/scalar.rs @@ -7,24 +7,23 @@ use std::sync::Arc; use arrow_array::Scalar as ArrowScalar; use arrow_array::*; +use vortex_array::dtype::DType; +use vortex_array::dtype::PType; +use vortex_array::extension::datetime::AnyTemporal; +use vortex_array::extension::datetime::TemporalMetadata; +use vortex_array::extension::datetime::TimeUnit; +use vortex_array::scalar::BinaryScalar; +use vortex_array::scalar::BoolScalar; +use vortex_array::scalar::DecimalScalar; +use vortex_array::scalar::DecimalValue; +use vortex_array::scalar::ExtScalar; +use vortex_array::scalar::PrimitiveScalar; +use vortex_array::scalar::Scalar; +use vortex_array::scalar::Utf8Scalar; use vortex_error::VortexError; use vortex_error::vortex_bail; use vortex_error::vortex_err; -use crate::dtype::DType; -use crate::dtype::PType; -use crate::extension::datetime::AnyTemporal; -use crate::extension::datetime::TemporalMetadata; -use crate::extension::datetime::TimeUnit; -use crate::scalar::BinaryScalar; -use crate::scalar::BoolScalar; -use crate::scalar::DecimalScalar; -use crate::scalar::DecimalValue; -use crate::scalar::ExtScalar; -use crate::scalar::PrimitiveScalar; -use crate::scalar::Scalar; -use crate::scalar::Utf8Scalar; - /// Arrow represents scalars as single-element arrays. This constant is the length of those arrays. const SCALAR_ARRAY_LEN: usize = 1; @@ -50,10 +49,18 @@ macro_rules! timestamp_to_arrow_scalar { }}; } -impl TryFrom<&Scalar> for Arc { - type Error = VortexError; +/// Convert a Vortex [`Scalar`] into an Arrow [`Datum`] (a single-element Arrow array). +/// +/// This mirrors a `TryFrom<&Scalar> for Arc` conversion; a separate trait is +/// required because both `Scalar` and `Datum` are foreign to this crate. +pub trait ToArrowDatum { + /// Convert this scalar to an Arrow [`Datum`]. + fn to_arrow_datum(&self) -> Result, VortexError>; +} - fn try_from(value: &Scalar) -> Result, Self::Error> { +impl ToArrowDatum for Scalar { + fn to_arrow_datum(&self) -> Result, VortexError> { + let value = self; match value.dtype() { DType::Null => Ok(Arc::new(NullArray::new(SCALAR_ARRAY_LEN))), DType::Bool(_) => bool_to_arrow(value.as_bool()), @@ -192,149 +199,149 @@ fn extension_to_arrow(scalar: ExtScalar<'_>) -> Result, VortexErr mod tests { use std::sync::Arc; - use arrow_array::Datum; use rstest::rstest; + use vortex_array::dtype::DType; + use vortex_array::dtype::DecimalDType; + use vortex_array::dtype::FieldDType; + use vortex_array::dtype::NativeDType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::dtype::StructFields; + use vortex_array::dtype::extension::ExtDType; + use vortex_array::dtype::extension::ExtId; + use vortex_array::dtype::extension::ExtVTable; + use vortex_array::dtype::i256; + use vortex_array::extension::datetime::Date; + use vortex_array::extension::datetime::Time; + use vortex_array::extension::datetime::TimeUnit; + use vortex_array::extension::datetime::Timestamp; + use vortex_array::extension::datetime::TimestampOptions; + use vortex_array::scalar::DecimalValue; + use vortex_array::scalar::Scalar; + use vortex_array::scalar::ScalarValue; use vortex_error::VortexResult; use vortex_error::vortex_bail; - use crate::dtype::DType; - use crate::dtype::DecimalDType; - use crate::dtype::FieldDType; - use crate::dtype::NativeDType; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::dtype::StructFields; - use crate::dtype::extension::ExtDType; - use crate::dtype::extension::ExtId; - use crate::dtype::extension::ExtVTable; - use crate::dtype::i256; - use crate::extension::datetime::Date; - use crate::extension::datetime::Time; - use crate::extension::datetime::TimeUnit; - use crate::extension::datetime::Timestamp; - use crate::extension::datetime::TimestampOptions; - use crate::scalar::DecimalValue; - use crate::scalar::Scalar; - use crate::scalar::ScalarValue; + use super::ToArrowDatum; #[test] fn test_null_scalar_to_arrow() { let scalar = Scalar::null(DType::Null); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_bool_scalar_to_arrow() { let scalar = Scalar::bool(true, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_null_bool_scalar_to_arrow() { let scalar = Scalar::null(bool::dtype().as_nullable()); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_primitive_u8_to_arrow() { let scalar = Scalar::primitive(42u8, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_primitive_u16_to_arrow() { let scalar = Scalar::primitive(1000u16, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_primitive_u32_to_arrow() { let scalar = Scalar::primitive(100000u32, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_primitive_u64_to_arrow() { let scalar = Scalar::primitive(10000000000u64, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_primitive_i8_to_arrow() { let scalar = Scalar::primitive(-42i8, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_primitive_i16_to_arrow() { let scalar = Scalar::primitive(-1000i16, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_primitive_i32_to_arrow() { let scalar = Scalar::primitive(-100000i32, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_primitive_i64_to_arrow() { let scalar = Scalar::primitive(-10000000000i64, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_primitive_f16_to_arrow() { - use crate::dtype::half::f16; + use vortex_array::dtype::half::f16; let scalar = Scalar::primitive(f16::from_f32(1.234), Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_primitive_f32_to_arrow() { let scalar = Scalar::primitive(1.234f32, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_primitive_f64_to_arrow() { let scalar = Scalar::primitive(1.234567890123f64, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_null_primitive_to_arrow() { let scalar = Scalar::null(i32::dtype().as_nullable()); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_utf8_scalar_to_arrow() { let scalar = Scalar::utf8("hello world".to_string(), Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_null_utf8_scalar_to_arrow() { let scalar = Scalar::null(String::dtype().as_nullable()); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } @@ -342,14 +349,14 @@ mod tests { fn test_binary_scalar_to_arrow() { let data = vec![1u8, 2, 3, 4, 5]; let scalar = Scalar::binary(data, Nullability::NonNullable); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } #[test] fn test_null_binary_scalar_to_arrow() { let scalar = Scalar::null(DType::Binary(Nullability::Nullable)); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } @@ -363,35 +370,35 @@ mod tests { decimal_dtype, Nullability::NonNullable, ); - assert!(Arc::::try_from(&scalar_i8).is_ok()); + assert!(scalar_i8.to_arrow_datum().is_ok()); let scalar_i16 = Scalar::decimal( DecimalValue::I16(10000), decimal_dtype, Nullability::NonNullable, ); - assert!(Arc::::try_from(&scalar_i16).is_ok()); + assert!(scalar_i16.to_arrow_datum().is_ok()); let scalar_i32 = Scalar::decimal( DecimalValue::I32(99999), decimal_dtype, Nullability::NonNullable, ); - assert!(Arc::::try_from(&scalar_i32).is_ok()); + assert!(scalar_i32.to_arrow_datum().is_ok()); let scalar_i64 = Scalar::decimal( DecimalValue::I64(99999), decimal_dtype, Nullability::NonNullable, ); - assert!(Arc::::try_from(&scalar_i64).is_ok()); + assert!(scalar_i64.to_arrow_datum().is_ok()); let scalar_i128 = Scalar::decimal( DecimalValue::I128(99999), decimal_dtype, Nullability::NonNullable, ); - assert!(Arc::::try_from(&scalar_i128).is_ok()); + assert!(scalar_i128.to_arrow_datum().is_ok()); // Test i256 @@ -401,14 +408,14 @@ mod tests { decimal_dtype, Nullability::NonNullable, ); - assert!(Arc::::try_from(&scalar_i256).is_ok()); + assert!(scalar_i256.to_arrow_datum().is_ok()); } #[test] fn test_null_decimal_to_arrow() { let decimal_dtype = DecimalDType::new(10, 2); let scalar = Scalar::null(DType::Decimal(decimal_dtype, Nullability::Nullable)); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } @@ -427,7 +434,7 @@ mod tests { struct_dtype, vec![Scalar::primitive(42i32, Nullability::NonNullable)], ); - Arc::::try_from(&struct_scalar).unwrap(); + struct_scalar.to_arrow_datum().unwrap(); } #[test] @@ -443,7 +450,7 @@ mod tests { Nullability::NonNullable, ); - Arc::::try_from(&list_scalar).unwrap(); + list_scalar.to_arrow_datum().unwrap(); } #[test] @@ -485,7 +492,7 @@ mod tests { Scalar::primitive(42i32, Nullability::NonNullable), ); - Arc::::try_from(&scalar).unwrap(); + scalar.to_arrow_datum().unwrap(); } #[rstest] @@ -510,7 +517,7 @@ mod tests { }, ); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } @@ -534,7 +541,7 @@ mod tests { }, ); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } @@ -552,7 +559,7 @@ mod tests { Scalar::primitive(value, Nullability::NonNullable), ); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } @@ -576,7 +583,7 @@ mod tests { Scalar::primitive(value, Nullability::NonNullable), ); - let result = Arc::::try_from(&scalar); + let result = scalar.to_arrow_datum(); assert!(result.is_ok()); } @@ -587,7 +594,7 @@ mod tests { Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable)), ); - let _result = Arc::::try_from(&scalar).unwrap(); + let _result = scalar.to_arrow_datum().unwrap(); } #[test] diff --git a/vortex-array/src/arrow/session.rs b/vortex-arrow/src/session.rs similarity index 94% rename from vortex-array/src/arrow/session.rs rename to vortex-arrow/src/session.rs index d3f43443d5d..240b354b2d5 100644 --- a/vortex-array/src/arrow/session.rs +++ b/vortex-arrow/src/session.rs @@ -3,8 +3,8 @@ //! Plugin layer for moving Arrow extension types in and out of Vortex. //! -//! Vortex's canonical Arrow conversion (see [`crate::dtype::arrow`] and the executor in -//! [`crate::arrow::executor`]) handles every non-extension Arrow type and the builtin temporal +//! Vortex's canonical Arrow conversion (see [`crate::dtype`] and the executor in +//! [`crate::executor`]) handles every non-extension Arrow type and the builtin temporal //! extensions. The plugins registered here cover the remaining case: **Arrow extension types**. //! //! * An [`ArrowExportVTable`] is dispatched purely by the **target Arrow extension Id** — @@ -36,6 +36,20 @@ use arrow_schema::extension::EXTENSION_TYPE_NAME_KEY; use arrow_schema::extension::ExtensionType; use tracing::debug; use tracing::trace; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arc_swap_map::ArcSwapMap; +use vortex_array::arrays::StructArray; +use vortex_array::dtype::DType; +use vortex_array::dtype::FieldName; +use vortex_array::dtype::FieldNames; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::StructFields; +use vortex_array::dtype::extension::ExtId; +use vortex_array::extension::datetime::AnyTemporal; +use vortex_array::extension::uuid::Uuid; +use vortex_array::validity::Validity; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -44,26 +58,13 @@ use vortex_session::SessionGuard; use vortex_session::SessionVar; use vortex_session::registry::Id; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arc_swap_map::ArcSwapMap; -use crate::arrays::StructArray; -use crate::arrow::FromArrowArray; -use crate::arrow::convert::nulls; -use crate::arrow::convert::remove_nulls; -use crate::arrow::executor::execute_arrow_naive; -use crate::dtype::DType; -use crate::dtype::FieldName; -use crate::dtype::FieldNames; -use crate::dtype::Nullability; -use crate::dtype::StructFields; -use crate::dtype::arrow::TryFromArrowType; -use crate::dtype::arrow::to_data_type_naive; -use crate::dtype::extension::ExtId; -use crate::extension::datetime::AnyTemporal; -use crate::extension::uuid::Uuid; -use crate::validity::Validity; +use crate::FromArrowArray; +use crate::IntoVortexArray; +use crate::convert::nulls; +use crate::convert::remove_nulls; +use crate::dtype::TryFromArrowType; +use crate::dtype::to_data_type_naive; +use crate::executor::execute_arrow_naive; /// Outcome of a successful call to [`ArrowExportVTable::execute_arrow`]. /// @@ -537,7 +538,10 @@ impl ArrowSession { .from_arrow_array(ArrowArrayRef::clone(list.values()), elem_field.as_ref())?; let offsets = list.offsets().clone().into_array(); let validity = nulls(list.nulls(), field.is_nullable())?; - Ok(crate::arrays::ListArray::try_new(elements, offsets, validity)?.into_array()) + Ok( + vortex_array::arrays::ListArray::try_new(elements, offsets, validity)? + .into_array(), + ) } DataType::LargeList(elem_field) => { let list = array.as_list::(); @@ -545,14 +549,17 @@ impl ArrowSession { .from_arrow_array(ArrowArrayRef::clone(list.values()), elem_field.as_ref())?; let offsets = list.offsets().clone().into_array(); let validity = nulls(list.nulls(), field.is_nullable())?; - Ok(crate::arrays::ListArray::try_new(elements, offsets, validity)?.into_array()) + Ok( + vortex_array::arrays::ListArray::try_new(elements, offsets, validity)? + .into_array(), + ) } DataType::FixedSizeList(elem_field, list_size) => { let fsl = array.as_fixed_size_list(); let elements = self.from_arrow_array(ArrowArrayRef::clone(fsl.values()), elem_field.as_ref())?; let validity = nulls(fsl.nulls(), field.is_nullable())?; - Ok(crate::arrays::FixedSizeListArray::try_new( + Ok(vortex_array::arrays::FixedSizeListArray::try_new( elements, *list_size as u32, validity, @@ -567,10 +574,10 @@ impl ArrowSession { let offsets = list.offsets().clone().into_array(); let sizes = list.sizes().clone().into_array(); let validity = nulls(list.nulls(), field.is_nullable())?; - Ok( - crate::arrays::ListViewArray::try_new(elements, offsets, sizes, validity)? - .into_array(), - ) + Ok(vortex_array::arrays::ListViewArray::try_new( + elements, offsets, sizes, validity, + )? + .into_array()) } DataType::LargeListView(elem_field) => { let list = array.as_list_view::(); @@ -579,10 +586,10 @@ impl ArrowSession { let offsets = list.offsets().clone().into_array(); let sizes = list.sizes().clone().into_array(); let validity = nulls(list.nulls(), field.is_nullable())?; - Ok( - crate::arrays::ListViewArray::try_new(elements, offsets, sizes, validity)? - .into_array(), - ) + Ok(vortex_array::arrays::ListViewArray::try_new( + elements, offsets, sizes, validity, + )? + .into_array()) } _ => ArrayRef::from_arrow(array.as_ref(), field.is_nullable()), } @@ -631,20 +638,20 @@ mod tests { use arrow_schema::DataType; use arrow_schema::Field; use arrow_schema::extension::Uuid as ArrowUuid; + use vortex_array::VortexSessionExecute; + use vortex_array::array_session; + use vortex_array::dtype::DType; + use vortex_array::dtype::FieldName; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::dtype::StructFields; + use vortex_array::dtype::extension::ExtDType; + use vortex_array::dtype::extension::ExtVTable; + use vortex_array::extension::uuid::Uuid; + use vortex_array::extension::uuid::UuidMetadata; use vortex_error::VortexResult; use super::*; - use crate::VortexSessionExecute; - use crate::array_session; - use crate::dtype::DType; - use crate::dtype::FieldName; - use crate::dtype::Nullability; - use crate::dtype::PType; - use crate::dtype::StructFields; - use crate::dtype::extension::ExtDType; - use crate::dtype::extension::ExtVTable; - use crate::extension::uuid::Uuid; - use crate::extension::uuid::UuidMetadata; fn uuid_dtype(nullable: bool) -> DType { let storage = DType::FixedSizeList( diff --git a/vortex-array/src/extension/uuid/arrow.rs b/vortex-arrow/src/uuid.rs similarity index 85% rename from vortex-array/src/extension/uuid/arrow.rs rename to vortex-arrow/src/uuid.rs index cd4b97ea60a..5901f56389b 100644 --- a/vortex-array/src/extension/uuid/arrow.rs +++ b/vortex-arrow/src/uuid.rs @@ -18,8 +18,22 @@ use arrow_schema::DataType; use arrow_schema::Field; use arrow_schema::extension::ExtensionType; use arrow_schema::extension::Uuid as ArrowUuid; -use vortex_array::arrow::ArrowSession; -use vortex_array::arrow::has_valid_extension_type; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::buffer::BufferHandle; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::dtype::extension::ExtDType; +use vortex_array::dtype::extension::ExtVTable; +use vortex_array::extension::uuid::Uuid; +use vortex_array::extension::uuid::UuidMetadata; +use vortex_array::validity::Validity; use vortex_buffer::Alignment; use vortex_buffer::Buffer; use vortex_error::VortexExpect; @@ -27,28 +41,14 @@ use vortex_error::VortexResult; use vortex_session::registry::CachedId; use vortex_session::registry::Id; -use crate::ArrayRef; -use crate::ExecutionCtx; -use crate::IntoArray; -use crate::arrays::ExtensionArray; -use crate::arrays::FixedSizeListArray; -use crate::arrays::PrimitiveArray; -use crate::arrays::extension::ExtensionArrayExt; -use crate::arrow::ArrowExport; -use crate::arrow::ArrowExportVTable; -use crate::arrow::ArrowImport; -use crate::arrow::ArrowImportVTable; -use crate::arrow::ArrowSessionExt; -use crate::arrow::nulls; -use crate::buffer::BufferHandle; -use crate::dtype::DType; -use crate::dtype::Nullability; -use crate::dtype::PType; -use crate::dtype::extension::ExtDType; -use crate::dtype::extension::ExtVTable; -use crate::extension::uuid::Uuid; -use crate::extension::uuid::UuidMetadata; -use crate::validity::Validity; +use crate::ArrowExport; +use crate::ArrowExportVTable; +use crate::ArrowImport; +use crate::ArrowImportVTable; +use crate::ArrowSession; +use crate::ArrowSessionExt; +use crate::nulls; +use crate::session::has_valid_extension_type; const UUID_BYTE_LEN: i32 = 16; diff --git a/vortex-arrow/tests/canonical.rs b/vortex-arrow/tests/canonical.rs new file mode 100644 index 00000000000..f67e0b73115 --- /dev/null +++ b/vortex-arrow/tests/canonical.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#![expect(clippy::tests_outside_test_module)] + +//! Round-trip tests between canonical Vortex arrays and Arrow, moved from +//! `vortex-array/src/canonical.rs` when Arrow interoperability moved into this crate. + +use std::sync::Arc; +use std::sync::LazyLock; + +use arrow_array::Array as ArrowArray; +use arrow_array::ArrayRef as ArrowArrayRef; +use arrow_array::ListArray as ArrowListArray; +use arrow_array::PrimitiveArray as ArrowPrimitiveArray; +use arrow_array::StringArray; +use arrow_array::StringViewArray; +use arrow_array::StructArray as ArrowStructArray; +use arrow_array::cast::AsArray; +use arrow_array::types::Int32Type; +use arrow_array::types::Int64Type; +use arrow_array::types::UInt64Type; +use arrow_buffer::NullBufferBuilder; +use arrow_buffer::OffsetBuffer; +use arrow_schema::DataType; +use arrow_schema::Field; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::StructArray; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::FromArrowArray; +use vortex_buffer::buffer; +use vortex_session::VortexSession; + +/// A shared session for these canonical tests, used to create execution contexts. +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); + +#[test] +fn test_canonicalize_nested_struct() { + let mut ctx = SESSION.create_execution_ctx(); + // Create a struct array with multiple internal components. + let nested_struct_array = StructArray::from_fields(&[ + ("a", buffer![1u64].into_array()), + ( + "b", + StructArray::from_fields(&[( + "inner_a", + // The nested struct contains a ConstantArray representing the primitive array + // [100i64] + // ConstantArray is not a canonical type, so converting `into_arrow()` should + // map this to the nearest canonical type (PrimitiveArray). + ConstantArray::new(100i64, 1).into_array(), + )]) + .unwrap() + .into_array(), + ), + ]) + .unwrap(); + + let arrow_struct = SESSION + .arrow() + .execute_arrow(nested_struct_array.into_array(), None, &mut ctx) + .unwrap() + .as_any() + .downcast_ref::() + .cloned() + .unwrap(); + + assert!( + arrow_struct + .column(0) + .as_any() + .downcast_ref::>() + .is_some() + ); + + let inner_struct = Arc::clone(arrow_struct.column(1)) + .as_any() + .downcast_ref::() + .cloned() + .unwrap(); + + let inner_a = inner_struct + .column(0) + .as_any() + .downcast_ref::>(); + assert!(inner_a.is_some()); + + assert_eq!( + inner_a.cloned().unwrap(), + ArrowPrimitiveArray::from_iter([100i64]) + ); +} + +#[test] +fn roundtrip_struct() { + let mut ctx = SESSION.create_execution_ctx(); + let mut nulls = NullBufferBuilder::new(6); + nulls.append_n_non_nulls(4); + nulls.append_null(); + nulls.append_non_null(); + let names = Arc::new(StringViewArray::from_iter(vec![ + Some("Joseph"), + None, + Some("Angela"), + Some("Mikhail"), + None, + None, + ])); + let ages = Arc::new(ArrowPrimitiveArray::::from(vec![ + Some(25), + Some(31), + None, + Some(57), + None, + None, + ])); + + let arrow_struct = ArrowStructArray::new( + vec![ + Arc::new(Field::new("name", DataType::Utf8View, true)), + Arc::new(Field::new("age", DataType::Int32, true)), + ] + .into(), + vec![names, ages], + nulls.finish(), + ); + + let vortex_struct = ArrayRef::from_arrow(&arrow_struct, true).unwrap(); + let vortex_struct = SESSION + .arrow() + .execute_arrow(vortex_struct, None, &mut ctx) + .unwrap(); + assert_eq!(&arrow_struct, vortex_struct.as_struct()); +} + +#[test] +fn roundtrip_list() { + let mut ctx = SESSION.create_execution_ctx(); + let names = Arc::new(StringArray::from_iter(vec![ + Some("Joseph"), + Some("Angela"), + Some("Mikhail"), + ])); + + let arrow_list = ArrowListArray::new( + Arc::new(Field::new_list_field(DataType::Utf8, true)), + OffsetBuffer::from_lengths(vec![0, 2, 1]), + names, + None, + ); + let list_data_type = arrow_list.data_type(); + let list_field = Field::new(String::new(), list_data_type.clone(), true); + + let vortex_list = ArrayRef::from_arrow(&arrow_list, true).unwrap(); + + let rt_arrow_list = SESSION + .arrow() + .execute_arrow(vortex_list, Some(&list_field), &mut ctx) + .unwrap(); + + assert_eq!( + (Arc::new(arrow_list.clone()) as ArrowArrayRef).as_ref(), + rt_arrow_list.as_ref() + ); +} diff --git a/vortex-bench/Cargo.toml b/vortex-bench/Cargo.toml index 069f33a57e9..ff604fdb48a 100644 --- a/vortex-bench/Cargo.toml +++ b/vortex-bench/Cargo.toml @@ -23,6 +23,7 @@ vortex = { workspace = true, features = [ "tokio", "zstd", ] } +vortex-arrow = { workspace = true } vortex-geo = { workspace = true } vortex-tensor = { workspace = true } # TODO(connor): In the future, this might be inside vortex. diff --git a/vortex-bench/src/conversions.rs b/vortex-bench/src/conversions.rs index b7c367ef8f6..92064c25785 100644 --- a/vortex-bench/src/conversions.rs +++ b/vortex-bench/src/conversions.rs @@ -32,7 +32,6 @@ use vortex::array::arrays::Struct; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinViewArray; use vortex::array::arrays::struct_::StructArrayExt; -use vortex::array::arrow::FromArrowArray; use vortex::array::builders::builder_with_capacity; use vortex::array::stream::ArrayStreamAdapter; use vortex::array::stream::ArrayStreamExt; @@ -40,7 +39,6 @@ use vortex::compressor::BtrBlocksCompressorBuilder; use vortex::dtype::DType; use vortex::dtype::FieldPath; use vortex::dtype::StructFields; -use vortex::dtype::arrow::FromArrowType; use vortex::dtype::extension::ExtDType; use vortex::dtype::extension::ExtDTypeRef; use vortex::error::VortexResult; @@ -54,6 +52,8 @@ use vortex::layout::layouts::compressed::CompressingStrategy; use vortex::layout::layouts::flat::writer::FlatLayoutStrategy; use vortex::session::VortexSession; use vortex::utils::aliases::hash_set::HashSet; +use vortex_arrow::FromArrowArray; +use vortex_arrow::FromArrowType; use vortex_geo::extension::GeoMetadata; use vortex_geo::extension::WellKnownBinary; use wkb::Endianness; diff --git a/vortex-bench/src/datasets/struct_list_of_ints.rs b/vortex-bench/src/datasets/struct_list_of_ints.rs index fe9aa991681..164ad4534fc 100644 --- a/vortex-bench/src/datasets/struct_list_of_ints.rs +++ b/vortex-bench/src/datasets/struct_list_of_ints.rs @@ -24,9 +24,10 @@ use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::chunked::ChunkedArrayExt; use vortex::array::arrays::listview::recursive_list_from_list_view; -use vortex::array::arrow::ArrowSessionExt; use vortex::array::validity::Validity; use vortex::dtype::FieldNames; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::ToArrowType; use crate::IdempotentPath; use crate::SESSION; diff --git a/vortex-bench/src/spatialbench/datagen/native.rs b/vortex-bench/src/spatialbench/datagen/native.rs index 1d600a8b8cf..2e03b61b39f 100644 --- a/vortex-bench/src/spatialbench/datagen/native.rs +++ b/vortex-bench/src/spatialbench/datagen/native.rs @@ -32,8 +32,8 @@ use tokio::fs::File as TokioFile; use vortex::array::ArrayRef; use vortex::array::IntoArray; use vortex::array::arrays::ChunkedArray; -use vortex::array::arrow::ArrowSessionExt; use vortex::file::WriteOptionsSessionExt; +use vortex_arrow::ArrowSessionExt; use super::table::GeometryKind; use super::table::Table; diff --git a/vortex-bench/src/tpch/tpchgen.rs b/vortex-bench/src/tpch/tpchgen.rs index 5dbeb380a7e..79b91200e48 100644 --- a/vortex-bench/src/tpch/tpchgen.rs +++ b/vortex-bench/src/tpch/tpchgen.rs @@ -32,12 +32,12 @@ use tpchgen::generators::SupplierGenerator; use tpchgen_arrow::RecordBatchIterator; use tracing::info; use vortex::array::ArrayRef; -use vortex::array::arrow::FromArrowArray; use vortex::array::stream::ArrayStreamAdapter; use vortex::dtype::DType; -use vortex::dtype::arrow::FromArrowType; use vortex::error::VortexExpect; use vortex::file::WriteOptionsSessionExt; +use vortex_arrow::FromArrowArray; +use vortex_arrow::FromArrowType; use crate::CompactionStrategy; use crate::Format; diff --git a/vortex-cuda/Cargo.toml b/vortex-cuda/Cargo.toml index 6ad2662e416..be6faa77e7b 100644 --- a/vortex-cuda/Cargo.toml +++ b/vortex-cuda/Cargo.toml @@ -38,6 +38,7 @@ tokio = { workspace = true, features = ["fs"] } tracing = { workspace = true, features = ["std", "attributes"] } vortex = { workspace = true, features = ["zstd"] } vortex-array = { workspace = true, features = ["cudarc"] } +vortex-arrow = { workspace = true } vortex-cub = { path = "cub" } vortex-cuda-macros = { workspace = true } vortex-error = { workspace = true, features = ["object_store"] } diff --git a/vortex-cuda/src/arrow/mod.rs b/vortex-cuda/src/arrow/mod.rs index 9545ba6abee..aff582159b9 100644 --- a/vortex-cuda/src/arrow/mod.rs +++ b/vortex-cuda/src/arrow/mod.rs @@ -41,7 +41,6 @@ use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex::array::arrays::list::ListArrayExt; use vortex::array::arrays::listview::ListViewArrayExt; use vortex::array::arrays::struct_::StructArrayExt; -use vortex::array::arrow::ArrowSessionExt; use vortex::array::buffer::BufferHandle; use vortex::array::stream::SendableArrayStream; use vortex::dtype::DType; @@ -55,6 +54,7 @@ use vortex::error::vortex_err; use vortex::io::runtime::BlockingRuntime; use vortex::io::runtime::current::CurrentThreadRuntime; use vortex::session::VortexSession; +use vortex_arrow::ArrowSessionExt; use crate::CudaBufferExt; use crate::CudaExecutionCtx; diff --git a/vortex-datafusion/Cargo.toml b/vortex-datafusion/Cargo.toml index 72ebb3362b4..49fb22d4f59 100644 --- a/vortex-datafusion/Cargo.toml +++ b/vortex-datafusion/Cargo.toml @@ -37,6 +37,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "fs"] } tokio-stream = { workspace = true } tracing = { workspace = true, features = ["std", "attributes"] } vortex = { workspace = true, features = ["object_store", "tokio", "files"] } +vortex-arrow = { workspace = true } vortex-utils = { workspace = true, features = ["dashmap"] } [dev-dependencies] diff --git a/vortex-datafusion/src/convert/exprs.rs b/vortex-datafusion/src/convert/exprs.rs index 862c56bc3cc..847a020b887 100644 --- a/vortex-datafusion/src/convert/exprs.rs +++ b/vortex-datafusion/src/convert/exprs.rs @@ -24,7 +24,6 @@ use datafusion_physical_expr_common::physical_expr::is_dynamic_physical_expr; use datafusion_physical_plan::expressions as df_expr; use itertools::Itertools; use vortex::VortexSessionDefault; -use vortex::array::arrow::ArrowSessionExt; use vortex::dtype::Nullability; use vortex::expr::Expression; use vortex::expr::and_collect; @@ -47,6 +46,7 @@ use vortex::scalar_fn::fns::like::Like; use vortex::scalar_fn::fns::like::LikeOptions; use vortex::scalar_fn::fns::operators::Operator; use vortex::session::VortexSession; +use vortex_arrow::ArrowSessionExt; use crate::convert::FromDataFusion; @@ -1192,8 +1192,8 @@ mod tests { use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::VortexSessionExecute as _; - use vortex::array::arrow::FromArrowArray; use vortex::session::VortexSession; + use vortex_arrow::FromArrowArray; // Create test data let values = Arc::new(Int32Array::from(vec![1, 5, 10, 15, 20])); diff --git a/vortex-datafusion/src/convert/scalars.rs b/vortex-datafusion/src/convert/scalars.rs index e26aa005adc..e72a1f9ace3 100644 --- a/vortex-datafusion/src/convert/scalars.rs +++ b/vortex-datafusion/src/convert/scalars.rs @@ -14,7 +14,6 @@ use vortex::dtype::DecimalDType; use vortex::dtype::NativeDecimalType; use vortex::dtype::Nullability; use vortex::dtype::PType; -use vortex::dtype::arrow::FromArrowType; use vortex::dtype::half::f16; use vortex::dtype::i256; use vortex::error::VortexExpect; @@ -27,6 +26,7 @@ use vortex::extension::datetime::TemporalMetadata; use vortex::extension::datetime::TimeUnit; use vortex::scalar::DecimalValue; use vortex::scalar::Scalar; +use vortex_arrow::FromArrowType; use crate::convert::FromDataFusion; use crate::convert::TryToDataFusion; diff --git a/vortex-datafusion/src/convert/schema.rs b/vortex-datafusion/src/convert/schema.rs index ecfaa72d01e..2ec17758593 100644 --- a/vortex-datafusion/src/convert/schema.rs +++ b/vortex-datafusion/src/convert/schema.rs @@ -6,8 +6,8 @@ use arrow_schema::Field; use arrow_schema::Schema; use datafusion_common::Result as DFResult; use datafusion_common::exec_datafusion_err; -use vortex::array::arrow::ArrowSession; use vortex::dtype::DType; +use vortex_arrow::ArrowSession; /// Calculate the physical Arrow schema for a Vortex file given its DType and the expected logical schema. /// diff --git a/vortex-datafusion/src/lib.rs b/vortex-datafusion/src/lib.rs index 64992849383..fd430403e55 100644 --- a/vortex-datafusion/src/lib.rs +++ b/vortex-datafusion/src/lib.rs @@ -141,11 +141,11 @@ mod common_tests { use url::Url; use vortex::VortexSessionDefault; use vortex::array::ArrayRef; - use vortex::array::arrow::FromArrowArray; use vortex::file::WriteOptionsSessionExt; use vortex::io::VortexWrite; use vortex::io::object_store::ObjectStoreWrite; use vortex::session::VortexSession; + use vortex_arrow::FromArrowArray; use crate::VortexFormatFactory; use crate::VortexTableOptions; diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index c139fe2767f..779b74b212e 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -44,7 +44,6 @@ use futures::stream; use object_store::ObjectMeta; use object_store::ObjectStore; use vortex::VortexSessionDefault; -use vortex::array::arrow::ArrowSessionExt; use vortex::array::memory::MemorySessionExt; use vortex::dtype::DType; use vortex::dtype::Nullability; @@ -63,6 +62,7 @@ use vortex::io::session::RuntimeSessionExt; use vortex::scalar::Scalar; use vortex::scalar::ScalarValue as VortexScalarValue; use vortex::session::VortexSession; +use vortex_arrow::ArrowSessionExt; use super::cache::CachedVortexMetadata; use super::sink::VortexSink; diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index d50b003f1dc..b1c999656f6 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -37,7 +37,6 @@ use itertools::Itertools; use object_store::path::Path; use tracing::Instrument; use vortex::array::VortexSessionExecute; -use vortex::array::arrow::ArrowSessionExt; use vortex::dtype::FieldMask; use vortex::error::VortexError; use vortex::error::VortexExpect; @@ -49,6 +48,7 @@ use vortex::layout::scan::split_by::SplitBy; use vortex::metrics::Label; use vortex::metrics::MetricsRegistry; use vortex::session::VortexSession; +use vortex_arrow::ArrowSessionExt; use vortex_utils::aliases::dash_map::DashMap; use vortex_utils::aliases::dash_map::Entry; @@ -596,7 +596,6 @@ mod tests { use rstest::rstest; use vortex::VortexSessionDefault; use vortex::array::ArrayRef; - use vortex::array::arrow::FromArrowArray; use vortex::buffer::Buffer; use vortex::file::WriteOptionsSessionExt; use vortex::io::VortexWrite; @@ -604,6 +603,7 @@ mod tests { use vortex::metrics::DefaultMetricsRegistry; use vortex::scan::selection::Selection; use vortex::session::VortexSession; + use vortex_arrow::FromArrowArray; use super::*; use crate::VortexAccessPlan; diff --git a/vortex-datafusion/src/persistent/sink.rs b/vortex-datafusion/src/persistent/sink.rs index 828d55e97e3..32be0c6873b 100644 --- a/vortex-datafusion/src/persistent/sink.rs +++ b/vortex-datafusion/src/persistent/sink.rs @@ -25,13 +25,13 @@ use futures::StreamExt; use object_store::ObjectStore; use object_store::path::Path; use tokio_stream::wrappers::ReceiverStream; -use vortex::array::arrow::ArrowSessionExt; use vortex::array::stream::ArrayStreamAdapter; use vortex::file::WriteOptionsSessionExt; use vortex::file::WriteSummary; use vortex::io::VortexWrite; use vortex::io::object_store::ObjectStoreWrite; use vortex::session::VortexSession; +use vortex_arrow::ArrowSessionExt; /// Implements [`DataSink`] for writing Vortex files. pub struct VortexSink { diff --git a/vortex-datafusion/src/persistent/tests.rs b/vortex-datafusion/src/persistent/tests.rs index 220a1477b13..41d8141165f 100644 --- a/vortex-datafusion/src/persistent/tests.rs +++ b/vortex-datafusion/src/persistent/tests.rs @@ -494,7 +494,7 @@ async fn arrow_uuid_extension_roundtrip() -> anyhow::Result<()> { use datafusion::arrow::array::FixedSizeBinaryArray; use datafusion::arrow::array::RecordBatch; use datafusion::assert_batches_sorted_eq; - use vortex::array::arrow::ArrowSessionExt; + use vortex_arrow::ArrowSessionExt; let ctx = TestSessionContext::default(); // Default vortex session has importer/exporter for Arrow UUID @@ -561,7 +561,7 @@ async fn arrow_uuid_extension_roundtrip_nested_struct() -> anyhow::Result<()> { use datafusion::arrow::array::RecordBatch; use datafusion::arrow::array::StructArray as ArrowStructArray; use datafusion::assert_batches_sorted_eq; - use vortex::array::arrow::ArrowSessionExt; + use vortex_arrow::ArrowSessionExt; let ctx = TestSessionContext::default(); let session = VortexSession::default(); diff --git a/vortex-datafusion/src/v2/source.rs b/vortex-datafusion/src/v2/source.rs index 82fae9bf56d..325f8eae92f 100644 --- a/vortex-datafusion/src/v2/source.rs +++ b/vortex-datafusion/src/v2/source.rs @@ -98,7 +98,6 @@ use futures::StreamExt; use futures::TryStreamExt; use futures::future::try_join_all; use vortex::array::VortexSessionExecute; -use vortex::array::arrow::ArrowSessionExt; use vortex::dtype::DType; use vortex::dtype::FieldPath; use vortex::dtype::Nullability; @@ -115,6 +114,7 @@ use vortex::io::session::RuntimeSessionExt; use vortex::scan::DataSourceRef; use vortex::scan::ScanRequest; use vortex::session::VortexSession; +use vortex_arrow::ArrowSessionExt; use vortex_utils::parallelism::get_available_parallelism; use crate::convert::exprs::DefaultExpressionConvertor; diff --git a/vortex-ffi/Cargo.toml b/vortex-ffi/Cargo.toml index 61b1c4071b4..bf9d6c1c342 100644 --- a/vortex-ffi/Cargo.toml +++ b/vortex-ffi/Cargo.toml @@ -30,6 +30,7 @@ paste = { workspace = true } tracing = { workspace = true, features = ["std", "log"] } tracing-subscriber = { workspace = true, features = ["env-filter"] } vortex = { workspace = true, features = ["object_store"] } +vortex-arrow = { workspace = true } [dev-dependencies] rand = { workspace = true } diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index 61a8ab00a08..3eda7732190 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -24,7 +24,6 @@ use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinView; use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::arrays::struct_::StructArrayExt; -use vortex::array::arrow::FromArrowArray; use vortex::array::legacy_session; use vortex::array::validity::Validity; use vortex::buffer::Buffer; @@ -36,6 +35,7 @@ use vortex::error::vortex_bail; use vortex::error::vortex_ensure; use vortex::error::vortex_err; use vortex::error::vortex_panic; +use vortex_arrow::FromArrowArray; use crate::arc_wrapper; use crate::dtype::vx_dtype; diff --git a/vortex-ffi/src/dtype.rs b/vortex-ffi/src/dtype.rs index fa1f32c46ac..baeab5fd280 100644 --- a/vortex-ffi/src/dtype.rs +++ b/vortex-ffi/src/dtype.rs @@ -9,7 +9,6 @@ use arrow_array::ffi::FFI_ArrowSchema; use arrow_schema::Schema; use vortex::dtype::DType; use vortex::dtype::DecimalDType; -use vortex::dtype::arrow::FromArrowType; use vortex::error::VortexExpect; use vortex::error::vortex_ensure; use vortex::error::vortex_panic; @@ -17,6 +16,8 @@ use vortex::extension::datetime::AnyTemporal; use vortex::extension::datetime::Date; use vortex::extension::datetime::Time; use vortex::extension::datetime::Timestamp; +use vortex_arrow::FromArrowType; +use vortex_arrow::ToArrowType; use crate::arc_wrapper; use crate::error::try_or; diff --git a/vortex-ffi/src/scan.rs b/vortex-ffi/src/scan.rs index fdf8615d065..b21f654634a 100644 --- a/vortex-ffi/src/scan.rs +++ b/vortex-ffi/src/scan.rs @@ -18,7 +18,6 @@ use futures::StreamExt; use vortex::array::ArrayRef; use vortex::array::ExecutionCtx; use vortex::array::VortexSessionExecute; -use vortex::array::arrow::ArrowSessionExt; use vortex::array::expr::stats::Precision; use vortex::array::stream::SendableArrayStream; use vortex::buffer::Buffer; @@ -33,6 +32,8 @@ use vortex::scan::Partition; use vortex::scan::PartitionStream; use vortex::scan::ScanRequest; use vortex::scan::selection::Selection; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::ToArrowType; use crate::RUNTIME; use crate::array::vx_array; diff --git a/vortex-geo/Cargo.toml b/vortex-geo/Cargo.toml index 2f0583b49e6..bfb6a639742 100644 --- a/vortex-geo/Cargo.toml +++ b/vortex-geo/Cargo.toml @@ -23,6 +23,7 @@ geoarrow = { workspace = true } geoarrow-cast = { workspace = true } prost = { workspace = true } vortex-array = { workspace = true } +vortex-arrow = { workspace = true } vortex-error = { workspace = true } vortex-session = { workspace = true } wkb = { workspace = true } diff --git a/vortex-geo/src/extension/linestring.rs b/vortex-geo/src/extension/linestring.rs index f7d8fcff1de..274d20f28ba 100644 --- a/vortex-geo/src/extension/linestring.rs +++ b/vortex-geo/src/extension/linestring.rs @@ -24,20 +24,20 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrow::ArrowExport; -use vortex_array::arrow::ArrowExportVTable; -use vortex_array::arrow::ArrowImport; -use vortex_array::arrow::ArrowImportVTable; -use vortex_array::arrow::ArrowSession; -use vortex_array::arrow::ArrowSessionExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::dtype::arrow::FromArrowType; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtId; use vortex_array::dtype::extension::ExtVTable; use vortex_array::scalar::ScalarValue; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportVTable; +use vortex_arrow::ArrowImport; +use vortex_arrow::ArrowImportVTable; +use vortex_arrow::ArrowSession; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::FromArrowArray; +use vortex_arrow::FromArrowType; use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_bail; diff --git a/vortex-geo/src/extension/mod.rs b/vortex-geo/src/extension/mod.rs index 9da09020dbf..73f6b6317b0 100644 --- a/vortex-geo/src/extension/mod.rs +++ b/vortex-geo/src/extension/mod.rs @@ -43,10 +43,10 @@ use vortex_array::IntoArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtVTable; use vortex_array::scalar::Scalar; +use vortex_arrow::FromArrowArray; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; diff --git a/vortex-geo/src/extension/multilinestring.rs b/vortex-geo/src/extension/multilinestring.rs index edfb0d3554c..6ba4c4c5324 100644 --- a/vortex-geo/src/extension/multilinestring.rs +++ b/vortex-geo/src/extension/multilinestring.rs @@ -25,20 +25,20 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrow::ArrowExport; -use vortex_array::arrow::ArrowExportVTable; -use vortex_array::arrow::ArrowImport; -use vortex_array::arrow::ArrowImportVTable; -use vortex_array::arrow::ArrowSession; -use vortex_array::arrow::ArrowSessionExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::dtype::arrow::FromArrowType; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtId; use vortex_array::dtype::extension::ExtVTable; use vortex_array::scalar::ScalarValue; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportVTable; +use vortex_arrow::ArrowImport; +use vortex_arrow::ArrowImportVTable; +use vortex_arrow::ArrowSession; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::FromArrowArray; +use vortex_arrow::FromArrowType; use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_bail; diff --git a/vortex-geo/src/extension/multipoint.rs b/vortex-geo/src/extension/multipoint.rs index 3e8778744e5..7d8e429dcc0 100644 --- a/vortex-geo/src/extension/multipoint.rs +++ b/vortex-geo/src/extension/multipoint.rs @@ -25,20 +25,20 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrow::ArrowExport; -use vortex_array::arrow::ArrowExportVTable; -use vortex_array::arrow::ArrowImport; -use vortex_array::arrow::ArrowImportVTable; -use vortex_array::arrow::ArrowSession; -use vortex_array::arrow::ArrowSessionExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::dtype::arrow::FromArrowType; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtId; use vortex_array::dtype::extension::ExtVTable; use vortex_array::scalar::ScalarValue; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportVTable; +use vortex_arrow::ArrowImport; +use vortex_arrow::ArrowImportVTable; +use vortex_arrow::ArrowSession; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::FromArrowArray; +use vortex_arrow::FromArrowType; use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_bail; diff --git a/vortex-geo/src/extension/multipolygon.rs b/vortex-geo/src/extension/multipolygon.rs index 67dac438970..524e470749c 100644 --- a/vortex-geo/src/extension/multipolygon.rs +++ b/vortex-geo/src/extension/multipolygon.rs @@ -24,20 +24,20 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrow::ArrowExport; -use vortex_array::arrow::ArrowExportVTable; -use vortex_array::arrow::ArrowImport; -use vortex_array::arrow::ArrowImportVTable; -use vortex_array::arrow::ArrowSession; -use vortex_array::arrow::ArrowSessionExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::dtype::arrow::FromArrowType; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtId; use vortex_array::dtype::extension::ExtVTable; use vortex_array::scalar::ScalarValue; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportVTable; +use vortex_arrow::ArrowImport; +use vortex_arrow::ArrowImportVTable; +use vortex_arrow::ArrowSession; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::FromArrowArray; +use vortex_arrow::FromArrowType; use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_bail; diff --git a/vortex-geo/src/extension/point.rs b/vortex-geo/src/extension/point.rs index 6371105ca25..cfebecee461 100644 --- a/vortex-geo/src/extension/point.rs +++ b/vortex-geo/src/extension/point.rs @@ -23,20 +23,20 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrow::ArrowExport; -use vortex_array::arrow::ArrowExportVTable; -use vortex_array::arrow::ArrowImport; -use vortex_array::arrow::ArrowImportVTable; -use vortex_array::arrow::ArrowSession; -use vortex_array::arrow::ArrowSessionExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::DType; -use vortex_array::dtype::arrow::FromArrowType; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtId; use vortex_array::dtype::extension::ExtVTable; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportVTable; +use vortex_arrow::ArrowImport; +use vortex_arrow::ArrowImportVTable; +use vortex_arrow::ArrowSession; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::FromArrowArray; +use vortex_arrow::FromArrowType; use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_ensure; diff --git a/vortex-geo/src/extension/polygon.rs b/vortex-geo/src/extension/polygon.rs index aeda30931a2..9a74c3ce7b3 100644 --- a/vortex-geo/src/extension/polygon.rs +++ b/vortex-geo/src/extension/polygon.rs @@ -24,20 +24,20 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrow::ArrowExport; -use vortex_array::arrow::ArrowExportVTable; -use vortex_array::arrow::ArrowImport; -use vortex_array::arrow::ArrowImportVTable; -use vortex_array::arrow::ArrowSession; -use vortex_array::arrow::ArrowSessionExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::dtype::arrow::FromArrowType; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtId; use vortex_array::dtype::extension::ExtVTable; use vortex_array::scalar::ScalarValue; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportVTable; +use vortex_arrow::ArrowImport; +use vortex_arrow::ArrowImportVTable; +use vortex_arrow::ArrowSession; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::FromArrowArray; +use vortex_arrow::FromArrowType; use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_bail; diff --git a/vortex-geo/src/extension/wkb.rs b/vortex-geo/src/extension/wkb.rs index a45f1b309a1..fb8c443a7b3 100644 --- a/vortex-geo/src/extension/wkb.rs +++ b/vortex-geo/src/extension/wkb.rs @@ -19,18 +19,18 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrow::ArrowExport; -use vortex_array::arrow::ArrowExportVTable; -use vortex_array::arrow::ArrowImport; -use vortex_array::arrow::ArrowImportVTable; -use vortex_array::arrow::ArrowSession; -use vortex_array::arrow::ArrowSessionExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::DType; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtId; use vortex_array::dtype::extension::ExtVTable; use vortex_array::scalar::ScalarValue; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportVTable; +use vortex_arrow::ArrowImport; +use vortex_arrow::ArrowImportVTable; +use vortex_arrow::ArrowSession; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::FromArrowArray; use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_ensure; diff --git a/vortex-geo/src/lib.rs b/vortex-geo/src/lib.rs index c2637ecac55..4ddb8c25c3b 100644 --- a/vortex-geo/src/lib.rs +++ b/vortex-geo/src/lib.rs @@ -3,9 +3,9 @@ use std::sync::Arc; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::session::DTypeSessionExt; use vortex_array::scalar_fn::session::ScalarFnSessionExt; +use vortex_arrow::ArrowSessionExt; use vortex_session::VortexSession; use crate::extension::LineString; diff --git a/vortex-geo/src/tests/linestring.rs b/vortex-geo/src/tests/linestring.rs index a1a9b3a4845..46e2be94647 100644 --- a/vortex-geo/src/tests/linestring.rs +++ b/vortex-geo/src/tests/linestring.rs @@ -13,9 +13,9 @@ use geoarrow::datatypes::Crs; use geoarrow::datatypes::Dimension as GeoArrowDimension; use geoarrow::datatypes::LineStringType; use geoarrow::datatypes::Metadata; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; +use vortex_arrow::ArrowSessionExt; use vortex_error::VortexResult; use super::SESSION; diff --git a/vortex-geo/src/tests/multilinestring.rs b/vortex-geo/src/tests/multilinestring.rs index cf5b5204681..694c77cdf3b 100644 --- a/vortex-geo/src/tests/multilinestring.rs +++ b/vortex-geo/src/tests/multilinestring.rs @@ -13,9 +13,9 @@ use geoarrow::datatypes::Crs; use geoarrow::datatypes::Dimension as GeoArrowDimension; use geoarrow::datatypes::Metadata; use geoarrow::datatypes::MultiLineStringType; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; +use vortex_arrow::ArrowSessionExt; use vortex_error::VortexResult; use super::SESSION; diff --git a/vortex-geo/src/tests/multipoint.rs b/vortex-geo/src/tests/multipoint.rs index c1de946f0ba..366dac73fc2 100644 --- a/vortex-geo/src/tests/multipoint.rs +++ b/vortex-geo/src/tests/multipoint.rs @@ -13,9 +13,9 @@ use geoarrow::datatypes::Crs; use geoarrow::datatypes::Dimension as GeoArrowDimension; use geoarrow::datatypes::Metadata; use geoarrow::datatypes::MultiPointType; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; +use vortex_arrow::ArrowSessionExt; use vortex_error::VortexResult; use super::SESSION; diff --git a/vortex-geo/src/tests/multipolygon.rs b/vortex-geo/src/tests/multipolygon.rs index 38f2543a96a..407661f6a43 100644 --- a/vortex-geo/src/tests/multipolygon.rs +++ b/vortex-geo/src/tests/multipolygon.rs @@ -13,9 +13,9 @@ use geoarrow::datatypes::Crs; use geoarrow::datatypes::Dimension as GeoArrowDimension; use geoarrow::datatypes::Metadata; use geoarrow::datatypes::MultiPolygonType; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; +use vortex_arrow::ArrowSessionExt; use vortex_error::VortexResult; use super::SESSION; diff --git a/vortex-geo/src/tests/point.rs b/vortex-geo/src/tests/point.rs index ff74b01ba01..61a2e98dbe7 100644 --- a/vortex-geo/src/tests/point.rs +++ b/vortex-geo/src/tests/point.rs @@ -20,9 +20,9 @@ use geoarrow::datatypes::Dimension as GeoArrowDimension; use geoarrow::datatypes::Metadata; use geoarrow::datatypes::PointType; use vortex_array::VortexSessionExecute; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; +use vortex_arrow::ArrowSessionExt; use vortex_error::VortexResult; use vortex_error::vortex_err; diff --git a/vortex-geo/src/tests/wkb.rs b/vortex-geo/src/tests/wkb.rs index 4ef3d6e2ba8..d61c48bdc32 100644 --- a/vortex-geo/src/tests/wkb.rs +++ b/vortex-geo/src/tests/wkb.rs @@ -27,11 +27,11 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::varbin::builder::VarBinBuilder; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtVTable; +use vortex_arrow::ArrowSessionExt; use vortex_error::VortexResult; use vortex_error::vortex_err; use wkb::writer::WriteOptions; diff --git a/vortex-jni/Cargo.toml b/vortex-jni/Cargo.toml index e6aef3d8756..767770db46f 100644 --- a/vortex-jni/Cargo.toml +++ b/vortex-jni/Cargo.toml @@ -30,6 +30,7 @@ tracing = { workspace = true, features = ["std", "log"] } tracing-subscriber = { workspace = true, features = ["env-filter"] } url = { workspace = true } vortex = { workspace = true, features = ["object_store", "files"] } +vortex-arrow = { workspace = true } vortex-parquet-variant = { workspace = true } [dev-dependencies] diff --git a/vortex-jni/src/dtype.rs b/vortex-jni/src/dtype.rs index 5a0b54d2c14..8969d7b573a 100644 --- a/vortex-jni/src/dtype.rs +++ b/vortex-jni/src/dtype.rs @@ -13,6 +13,7 @@ use arrow_schema::Fields; use arrow_schema::Schema; use vortex::dtype::DType; use vortex::error::VortexResult; +use vortex_arrow::ToArrowType; /// Export a Vortex [`DType`] to the Arrow C Data Interface struct at `schema_addr`. Views /// (Utf8View/BinaryView) are downgraded to regular Utf8/Binary so Spark and other consumers diff --git a/vortex-jni/src/scan.rs b/vortex-jni/src/scan.rs index 7f923af1ac9..87819681686 100644 --- a/vortex-jni/src/scan.rs +++ b/vortex-jni/src/scan.rs @@ -29,7 +29,6 @@ use jni::objects::JLongArray; use jni::sys::jboolean; use jni::sys::jlong; use vortex::array::VortexSessionExecute; -use vortex::array::arrow::ArrowSessionExt; use vortex::array::stream::SendableArrayStream; use vortex::buffer::Buffer; use vortex::error::VortexResult; @@ -44,6 +43,8 @@ use vortex::scan::PartitionRef; use vortex::scan::PartitionStream; use vortex::scan::ScanRequest; use vortex::scan::selection::Selection; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::ToArrowType; use crate::POOL; use crate::RUNTIME; diff --git a/vortex-jni/src/writer.rs b/vortex-jni/src/writer.rs index 78e7c268f19..5b098b4c320 100644 --- a/vortex-jni/src/writer.rs +++ b/vortex-jni/src/writer.rs @@ -29,7 +29,6 @@ use object_store::ObjectStore; use object_store::path::Path as ObjectStorePath; use vortex::array::ArrayRef; use vortex::array::VTable; -use vortex::array::arrow::ArrowSessionExt; use vortex::array::stream::ArrayStreamAdapter; use vortex::dtype::DType; use vortex::dtype::Field as DTypeField; @@ -48,6 +47,7 @@ use vortex::io::runtime::Task; use vortex::io::session::RuntimeSessionExt; use vortex::session::VortexSession; use vortex::utils::aliases::hash_map::HashMap; +use vortex_arrow::ArrowSessionExt; use vortex_parquet_variant::ParquetVariant; use crate::RUNTIME; diff --git a/vortex-json/Cargo.toml b/vortex-json/Cargo.toml index 71eb015b5ee..b839c0ac76c 100644 --- a/vortex-json/Cargo.toml +++ b/vortex-json/Cargo.toml @@ -21,6 +21,7 @@ arrow-array = { workspace = true } arrow-schema = { workspace = true, features = ["canonical_extension_types"] } prost = { workspace = true } vortex-array = { workspace = true, default-features = false } +vortex-arrow = { workspace = true } vortex-error = { workspace = true, default-features = false } vortex-proto = { workspace = true, features = ["expr"] } vortex-session = { workspace = true } diff --git a/vortex-json/src/arrow.rs b/vortex-json/src/arrow.rs index 9db94d86743..987d6e16347 100644 --- a/vortex-json/src/arrow.rs +++ b/vortex-json/src/arrow.rs @@ -13,16 +13,16 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrow::ArrowExport; -use vortex_array::arrow::ArrowExportVTable; -use vortex_array::arrow::ArrowImport; -use vortex_array::arrow::ArrowImportVTable; -use vortex_array::arrow::ArrowSession; -use vortex_array::arrow::ArrowSessionExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::DType; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtVTable; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportVTable; +use vortex_arrow::ArrowImport; +use vortex_arrow::ArrowImportVTable; +use vortex_arrow::ArrowSession; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::FromArrowArray; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_session::registry::CachedId; @@ -156,9 +156,9 @@ mod tests { use vortex_array::VortexSessionExecute; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::VarBinArray; - use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::Nullability; use vortex_array::dtype::extension::ExtDType; + use vortex_arrow::ArrowSessionExt; use vortex_error::VortexExpect; use vortex_error::VortexResult; diff --git a/vortex-json/src/lib.rs b/vortex-json/src/lib.rs index 21f86ca9a7c..7a6dc8a780e 100644 --- a/vortex-json/src/lib.rs +++ b/vortex-json/src/lib.rs @@ -20,9 +20,9 @@ pub use json_to_variant::JsonToVariant; pub use json_to_variant::JsonToVariantOptions; pub use json_to_variant::ShreddingSpec; pub use json_to_variant::json_to_variant; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::session::DTypeSessionExt; use vortex_array::scalar_fn::session::ScalarFnSessionExt; +use vortex_arrow::ArrowSessionExt; use vortex_session::VortexSession; /// Register JSON extension support with a session. diff --git a/vortex-layout/Cargo.toml b/vortex-layout/Cargo.toml index daa96d4e138..f772b9ab639 100644 --- a/vortex-layout/Cargo.toml +++ b/vortex-layout/Cargo.toml @@ -40,6 +40,7 @@ termtree = { workspace = true } tokio = { workspace = true, features = ["rt"], optional = true } tracing = { workspace = true } vortex-array = { workspace = true } +vortex-arrow = { workspace = true } vortex-btrblocks = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } diff --git a/vortex-layout/src/scan/arrow.rs b/vortex-layout/src/scan/arrow.rs index 485c2685934..663b29d9501 100644 --- a/vortex-layout/src/scan/arrow.rs +++ b/vortex-layout/src/scan/arrow.rs @@ -14,7 +14,7 @@ use futures::TryStreamExt; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::VortexSessionExecute; -use vortex_array::arrow::ArrowSessionExt; +use vortex_arrow::ArrowSessionExt; use vortex_error::VortexResult; use vortex_io::runtime::BlockingRuntime; @@ -128,7 +128,7 @@ mod tests { use arrow_schema::Schema; use vortex_array::ArrayRef; use vortex_array::VortexSessionExecute; - use vortex_array::arrow::FromArrowArray; + use vortex_arrow::FromArrowArray; use vortex_error::VortexResult; use super::*; diff --git a/vortex-python/Cargo.toml b/vortex-python/Cargo.toml index 491bff86e6a..a87548d69a9 100644 --- a/vortex-python/Cargo.toml +++ b/vortex-python/Cargo.toml @@ -52,6 +52,7 @@ pyo3-object_store = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } url = { workspace = true } vortex = { workspace = true, features = ["object_store"] } +vortex-arrow = { workspace = true } vortex-python-abi = { path = "../vortex-python-abi" } vortex-tui = { workspace = true, optional = true } diff --git a/vortex-python/src/arrays/from_arrow.rs b/vortex-python/src/arrays/from_arrow.rs index e8c9ca04d60..3964742626e 100644 --- a/vortex-python/src/arrays/from_arrow.rs +++ b/vortex-python/src/arrays/from_arrow.rs @@ -13,11 +13,11 @@ use pyo3::prelude::*; use vortex::array::ArrayRef; use vortex::array::IntoArray; use vortex::array::arrays::ChunkedArray; -use vortex::array::arrow::FromArrowArray; use vortex::dtype::DType; -use vortex::dtype::arrow::TryFromArrowType; use vortex::error::VortexError; use vortex::error::VortexResult; +use vortex_arrow::FromArrowArray; +use vortex_arrow::TryFromArrowType; use crate::arrays::PyArrayRef; use crate::arrow::FromPyArrow; diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index e22e8d95955..ad6575fdbe0 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -39,7 +39,6 @@ use vortex::array::arrays::BoolArray; use vortex::array::arrays::Chunked; use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::arrays::chunked::ChunkedArrayExt; -use vortex::array::arrow::ArrowSessionExt; use vortex::array::buffer::BufferHandle; use vortex::array::builtins::ArrayBuiltins; use vortex::array::match_each_integer_ptype; @@ -52,6 +51,7 @@ use vortex::flatbuffers::WriteFlatBufferExt; use vortex::ipc::messages::EncoderMessage; use vortex::ipc::messages::MessageEncoder; use vortex::scalar_fn::fns::operators::Operator; +use vortex_arrow::ArrowSessionExt; use vortex_python_abi::BUFFER_EXPORT_CAPSULE_NAME; use vortex_python_abi::VORTEX_BUFFER_EXPORT_VERSION; use vortex_python_abi::VORTEX_BUFFER_HOST; diff --git a/vortex-python/src/dataset.rs b/vortex-python/src/dataset.rs index acb8285e5a0..504ed05aeee 100644 --- a/vortex-python/src/dataset.rs +++ b/vortex-python/src/dataset.rs @@ -25,6 +25,7 @@ use vortex::file::OpenOptionsSessionExt; use vortex::file::VortexFile; use vortex::io::runtime::BlockingRuntime; use vortex::layout::scan::split_by::SplitBy; +use vortex_arrow::ToArrowType; use crate::RUNTIME; use crate::arrays::PyArrayRef; diff --git a/vortex-python/src/dtype/mod.rs b/vortex-python/src/dtype/mod.rs index bea8f7d7fda..ce18ceb6d83 100644 --- a/vortex-python/src/dtype/mod.rs +++ b/vortex-python/src/dtype/mod.rs @@ -33,9 +33,10 @@ use pyo3::pyclass; use pyo3::pymethods; use pyo3::types::PyType; use pyo3::wrap_pyfunction; -use vortex::array::arrow::ArrowSessionExt; use vortex::dtype::DType; -use vortex::dtype::arrow::TryFromArrowType; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::ToArrowType; +use vortex_arrow::TryFromArrowType; use crate::arrow::FromPyArrow; use crate::arrow::ToPyArrow; diff --git a/vortex-python/src/file.rs b/vortex-python/src/file.rs index f6bd1ed9cda..9550067b956 100644 --- a/vortex-python/src/file.rs +++ b/vortex-python/src/file.rs @@ -27,6 +27,7 @@ use vortex::io::runtime::BlockingRuntime; use vortex::layout::scan::scan_builder::ScanBuilder; use vortex::layout::scan::split_by::SplitBy; use vortex::layout::segments::MokaSegmentCache; +use vortex_arrow::ToArrowType; use crate::RUNTIME; use crate::arrays::PyArrayRef; diff --git a/vortex-python/src/io.rs b/vortex-python/src/io.rs index a8d61edfc18..182ab7fbe74 100644 --- a/vortex-python/src/io.rs +++ b/vortex-python/src/io.rs @@ -12,13 +12,11 @@ use pyo3_object_store::PyObjectStore; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::IntoArray; -use vortex::array::arrow::FromArrowArray; use vortex::array::iter::ArrayIterator; use vortex::array::iter::ArrayIteratorAdapter; use vortex::array::iter::ArrayIteratorExt; use vortex::compressor::BtrBlocksCompressorBuilder; use vortex::dtype::DType; -use vortex::dtype::arrow::TryFromArrowType; use vortex::error::VortexError; use vortex::error::VortexResult; use vortex::file::WriteOptionsSessionExt; @@ -26,6 +24,8 @@ use vortex::file::WriteStrategyBuilder; use vortex::io::VortexWrite; use vortex::io::object_store::ObjectStoreWrite; use vortex::io::runtime::BlockingRuntime; +use vortex_arrow::FromArrowArray; +use vortex_arrow::TryFromArrowType; use crate::PyVortex; use crate::RUNTIME; diff --git a/vortex-python/src/iter/mod.rs b/vortex-python/src/iter/mod.rs index e03bd36d609..757f9910ad6 100644 --- a/vortex-python/src/iter/mod.rs +++ b/vortex-python/src/iter/mod.rs @@ -21,11 +21,12 @@ use pyo3::types::PyIterator; use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::VortexSessionExecute; -use vortex::array::arrow::ArrowSessionExt; use vortex::array::iter::ArrayIterator; use vortex::array::iter::ArrayIteratorAdapter; use vortex::array::iter::ArrayIteratorExt; use vortex::dtype::DType; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::ToArrowType; use crate::arrays::PyArrayRef; use crate::arrow::IntoPyArrow; diff --git a/vortex-tensor/Cargo.toml b/vortex-tensor/Cargo.toml index 7429d71521d..a6782eea309 100644 --- a/vortex-tensor/Cargo.toml +++ b/vortex-tensor/Cargo.toml @@ -18,6 +18,7 @@ workspace = true [dependencies] vortex-array = { workspace = true } +vortex-arrow = { workspace = true } vortex-buffer = { workspace = true } vortex-compressor = { workspace = true } vortex-error = { workspace = true } diff --git a/vortex-tensor/src/lib.rs b/vortex-tensor/src/lib.rs index 17b13b1ba1e..6826f3ad191 100644 --- a/vortex-tensor/src/lib.rs +++ b/vortex-tensor/src/lib.rs @@ -13,10 +13,10 @@ use std::sync::Arc; use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin; -use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::session::DTypeSessionExt; use vortex_array::scalar_fn::session::ScalarFnSessionExt; use vortex_array::session::ArraySessionExt; +use vortex_arrow::ArrowSessionExt; use vortex_session::VortexSession; use crate::scalar_fns::cosine_similarity::CosineSimilarity; diff --git a/vortex-tensor/src/types/vector/arrow.rs b/vortex-tensor/src/types/vector/arrow.rs index a011dc50c49..05c1890e7e1 100644 --- a/vortex-tensor/src/types/vector/arrow.rs +++ b/vortex-tensor/src/types/vector/arrow.rs @@ -20,17 +20,17 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrow::ArrowExport; -use vortex_array::arrow::ArrowExportVTable; -use vortex_array::arrow::ArrowImport; -use vortex_array::arrow::ArrowImportVTable; -use vortex_array::arrow::ArrowSession; -use vortex_array::arrow::ArrowSessionExt; -use vortex_array::arrow::FromArrowArray; use vortex_array::dtype::DType; -use vortex_array::dtype::arrow::FromArrowType; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtVTable; +use vortex_arrow::ArrowExport; +use vortex_arrow::ArrowExportVTable; +use vortex_arrow::ArrowImport; +use vortex_arrow::ArrowImportVTable; +use vortex_arrow::ArrowSession; +use vortex_arrow::ArrowSessionExt; +use vortex_arrow::FromArrowArray; +use vortex_arrow::FromArrowType; use vortex_error::VortexResult; use vortex_session::registry::CachedId; use vortex_session::registry::Id; @@ -179,10 +179,6 @@ mod tests { use vortex_array::VortexSessionExecute; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; - use vortex_array::arrow::ArrowExport; - use vortex_array::arrow::ArrowImport; - use vortex_array::arrow::ArrowSession; - use vortex_array::arrow::ArrowSessionExt; use vortex_array::dtype::DType; use vortex_array::dtype::FieldName; use vortex_array::dtype::Nullability; @@ -190,6 +186,10 @@ mod tests { use vortex_array::dtype::StructFields; use vortex_array::dtype::extension::ExtDType; use vortex_array::validity::Validity; + use vortex_arrow::ArrowExport; + use vortex_arrow::ArrowImport; + use vortex_arrow::ArrowSession; + use vortex_arrow::ArrowSessionExt; use vortex_buffer::buffer; use super::*; diff --git a/vortex-test/compat-gen/Cargo.toml b/vortex-test/compat-gen/Cargo.toml index dd8bec4703a..4fbd90a6273 100644 --- a/vortex-test/compat-gen/Cargo.toml +++ b/vortex-test/compat-gen/Cargo.toml @@ -23,6 +23,7 @@ path = "src/main.rs" # Vortex crates vortex = { workspace = true, features = ["files", "tokio", "zstd"] } vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-arrow = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } vortex-session = { workspace = true } diff --git a/vortex-test/compat-gen/src/fixtures/arrays/datasets/clickbench.rs b/vortex-test/compat-gen/src/fixtures/arrays/datasets/clickbench.rs index 7cae6944463..12d86c41241 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/datasets/clickbench.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/datasets/clickbench.rs @@ -11,7 +11,7 @@ use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::arrays::ChunkedArray; -use vortex_array::arrow::FromArrowArray; +use vortex_arrow::FromArrowArray; use vortex_error::VortexResult; use vortex_error::vortex_err; diff --git a/vortex-test/compat-gen/src/fixtures/arrays/datasets/tpch.rs b/vortex-test/compat-gen/src/fixtures/arrays/datasets/tpch.rs index 9d831bef143..59e97a7e587 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/datasets/tpch.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/datasets/tpch.rs @@ -8,7 +8,7 @@ use tpchgen_arrow::RecordBatchIterator; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::arrays::ChunkedArray; -use vortex_array::arrow::FromArrowArray; +use vortex_arrow::FromArrowArray; use vortex_error::VortexResult; use crate::fixtures::DatasetFixture; diff --git a/vortex-test/e2e-cuda/src/lib.rs b/vortex-test/e2e-cuda/src/lib.rs index 493d7c7d23c..1c6530ce612 100644 --- a/vortex-test/e2e-cuda/src/lib.rs +++ b/vortex-test/e2e-cuda/src/lib.rs @@ -45,9 +45,9 @@ use vortex::array::arrays::StructArray; use vortex::array::arrays::TemporalArray; use vortex::array::arrays::VarBinViewArray; use vortex::array::arrays::varbinview::BinaryView; -use vortex::array::arrow::ArrowSessionExt; use vortex::array::stream::ArrayStreamExt; use vortex::array::validity::Validity; +use vortex::arrow::ArrowSessionExt; use vortex::buffer::Buffer; use vortex::buffer::ByteBuffer; use vortex::dtype::DType; diff --git a/vortex-tui/Cargo.toml b/vortex-tui/Cargo.toml index 2c890758945..34b90ecfc05 100644 --- a/vortex-tui/Cargo.toml +++ b/vortex-tui/Cargo.toml @@ -55,6 +55,7 @@ taffy = { workspace = true } vortex = { version = "0.1.0", path = "../vortex", default-features = false, features = [ "files", ] } +vortex-arrow = { workspace = true } # Native-only dependencies (gated behind "native" feature) arrow-array = { workspace = true, optional = true } diff --git a/vortex-tui/src/convert.rs b/vortex-tui/src/convert.rs index f6798413422..17fd5dcbc96 100644 --- a/vortex-tui/src/convert.rs +++ b/vortex-tui/src/convert.rs @@ -13,16 +13,16 @@ use parquet::arrow::ParquetRecordBatchStreamBuilder; use tokio::fs::File; use tokio::io::AsyncWriteExt; use vortex::array::ArrayRef; -use vortex::array::arrow::FromArrowArray; use vortex::array::stream::ArrayStreamAdapter; use vortex::compressor::BtrBlocksCompressorBuilder; use vortex::dtype::DType; -use vortex::dtype::arrow::FromArrowType; use vortex::error::VortexExpect; use vortex::error::vortex_err; use vortex::file::WriteOptionsSessionExt; use vortex::file::WriteStrategyBuilder; use vortex::session::VortexSession; +use vortex_arrow::FromArrowArray; +use vortex_arrow::FromArrowType; /// Compression strategy to use when converting Parquet files to Vortex format. #[derive(Clone, Copy, Debug, Default, ValueEnum)] diff --git a/vortex-web/crate/Cargo.toml b/vortex-web/crate/Cargo.toml index 1b65fe832eb..b4de2003c28 100644 --- a/vortex-web/crate/Cargo.toml +++ b/vortex-web/crate/Cargo.toml @@ -19,6 +19,7 @@ serde_json = { workspace = true } vortex = { path = "../../vortex", default-features = false, features = [ "files", ] } +vortex-arrow = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] futures = { workspace = true } diff --git a/vortex-web/crate/src/wasm.rs b/vortex-web/crate/src/wasm.rs index 04abe8ee6d3..0323a98c271 100644 --- a/vortex-web/crate/src/wasm.rs +++ b/vortex-web/crate/src/wasm.rs @@ -20,7 +20,6 @@ use futures::future::BoxFuture; use serde::Serialize; use vortex::array::ArrayRef; use vortex::array::VortexSessionExecute; -use vortex::array::arrow::ArrowSessionExt; use vortex::array::buffer::BufferHandle; use vortex::array::dtype::DType; use vortex::array::serde::SerializedArray; @@ -40,6 +39,7 @@ use vortex::layout::layouts::flat::Flat; use vortex::layout::scan::scan_builder::ScanBuilder; use vortex::session::VortexSession; use vortex::session::registry::ReadContext; +use vortex_arrow::ArrowSessionExt; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index 510ec5747f7..2511893d09e 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -25,6 +25,7 @@ workspace = true [dependencies] vortex-alp = { workspace = true } vortex-array = { workspace = true } +vortex-arrow = { workspace = true } vortex-btrblocks = { workspace = true } vortex-buffer = { workspace = true } vortex-bytebool = { workspace = true } @@ -42,7 +43,7 @@ vortex-mask = { workspace = true } vortex-metrics = { workspace = true } vortex-pco = { workspace = true } vortex-proto = { workspace = true, default-features = true } -vortex-runend = { workspace = true, features = ["arrow"] } +vortex-runend = { workspace = true } vortex-scan = { workspace = true } vortex-sequence = { workspace = true } vortex-session = { workspace = true } diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 764c1f3d5fc..2703020906c 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -99,7 +99,6 @@ // vortex::compute is deprecated and will be ported over to expressions. pub use vortex_array::aggregate_fn; use vortex_array::aggregate_fn::session::AggregateFnSession; -use vortex_array::arrow::ArrowSession; pub use vortex_array::compute; use vortex_array::dtype::session::DTypeSession; // vortex::expr is in the process of having its dependencies inverted, and will eventually be @@ -127,6 +126,12 @@ pub mod array { // twice. } +/// Arrow interoperability: conversion between Vortex and Apache Arrow arrays, types, and +/// schemas. +pub mod arrow { + pub use vortex_arrow::*; +} + /// Aligned buffers and byte buffers used by arrays, layouts, IPC, and file IO. pub mod buffer { pub use vortex_buffer::*; @@ -297,9 +302,9 @@ impl VortexSessionDefault for VortexSession { .with::() .with::() .with::() - .with::() .with::() .with::(); + vortex_arrow::initialize(&session); // `MultiFileSession` holds a `moka` cache whose clock reads `std::time::Instant::now()` // when constructed. `Instant` is unsupported on `wasm32` and panics with "time not @@ -355,9 +360,9 @@ mod test { use arrow_array::RecordBatchReader; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use vortex::array::arrays::ChunkedArray; - use vortex::array::arrow::FromArrowArray; + use vortex::arrow::FromArrowArray; + use vortex::arrow::FromArrowType; use vortex::dtype::DType; - use vortex::dtype::arrow::FromArrowType; let reader = ParquetRecordBatchReaderBuilder::try_new(File::open( "../docs/_static/example.parquet", From 82b8d9512f890412970a782eb1d1b631b9f867ba Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 13 Jul 2026 14:13:10 +0100 Subject: [PATCH 2/2] nicer Signed-off-by: Robert Kruszewski --- Cargo.lock | 49 +++++++++++++++++++++++++++++++------ vortex-arrow/src/session.rs | 38 +++++++++------------------- 2 files changed, 54 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d75d5661a98..881744baa91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1559,6 +1559,7 @@ dependencies = [ "tokio", "tracing", "vortex", + "vortex-arrow", "vortex-bench", ] @@ -2125,6 +2126,7 @@ dependencies = [ "tokio", "url", "vortex", + "vortex-arrow", "vortex-bench", "vortex-cuda", "vortex-datafusion", @@ -9605,6 +9607,7 @@ dependencies = [ "vortex", "vortex-alp", "vortex-array", + "vortex-arrow", "vortex-btrblocks", "vortex-buffer", "vortex-bytebool", @@ -9660,12 +9663,7 @@ dependencies = [ "arbitrary", "arc-swap", "arcref", - "arrow-array 58.3.0", "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", "async-lock", "bytes", "cfg-if", @@ -9728,6 +9726,29 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "vortex-arrow" +version = "0.1.0" +dependencies = [ + "arrow-array 58.3.0", + "arrow-buffer 58.3.0", + "arrow-cast 58.3.0", + "arrow-data 58.3.0", + "arrow-schema 58.3.0", + "arrow-select 58.3.0", + "codspeed-divan-compat", + "itertools 0.14.0", + "num-traits", + "rstest", + "tracing", + "vortex-array", + "vortex-buffer", + "vortex-error", + "vortex-mask", + "vortex-runend", + "vortex-session", +] + [[package]] name = "vortex-bench" version = "0.1.0" @@ -9775,6 +9796,7 @@ dependencies = [ "url", "uuid", "vortex", + "vortex-arrow", "vortex-geo", "vortex-tensor", "wkb", @@ -9862,6 +9884,7 @@ dependencies = [ "tpchgen-arrow", "vortex", "vortex-array", + "vortex-arrow", "vortex-buffer", "vortex-error", "vortex-session", @@ -9935,6 +9958,7 @@ dependencies = [ "tracing", "vortex", "vortex-array", + "vortex-arrow", "vortex-cub", "vortex-cuda", "vortex-cuda-macros", @@ -9996,6 +10020,7 @@ dependencies = [ "tracing", "url", "vortex", + "vortex-arrow", "vortex-utils", ] @@ -10118,6 +10143,7 @@ dependencies = [ "tracing-subscriber", "vortex", "vortex-array", + "vortex-arrow", ] [[package]] @@ -10234,6 +10260,7 @@ dependencies = [ "prost 0.14.4", "rstest", "vortex-array", + "vortex-arrow", "vortex-error", "vortex-session", "wkb", @@ -10305,6 +10332,7 @@ dependencies = [ "tracing-subscriber", "url", "vortex", + "vortex-arrow", "vortex-parquet-variant", ] @@ -10316,6 +10344,7 @@ dependencies = [ "arrow-schema 58.3.0", "prost 0.14.4", "vortex-array", + "vortex-arrow", "vortex-error", "vortex-proto", "vortex-session", @@ -10351,6 +10380,7 @@ dependencies = [ "tokio", "tracing", "vortex-array", + "vortex-arrow", "vortex-btrblocks", "vortex-buffer", "vortex-error", @@ -10427,6 +10457,7 @@ dependencies = [ "rstest", "tokio", "vortex-array", + "vortex-arrow", "vortex-buffer", "vortex-error", "vortex-file", @@ -10446,6 +10477,7 @@ dependencies = [ "prost 0.14.4", "rstest", "vortex-array", + "vortex-arrow", "vortex-buffer", "vortex-error", "vortex-mask", @@ -10481,6 +10513,7 @@ dependencies = [ "url", "vortex", "vortex-array", + "vortex-arrow", "vortex-python-abi", "vortex-tui", ] @@ -10525,8 +10558,6 @@ name = "vortex-runend" version = "0.1.0" dependencies = [ "arbitrary", - "arrow-array 58.3.0", - "arrow-schema 58.3.0", "codspeed-divan-compat", "itertools 0.14.0", "num-traits", @@ -10592,6 +10623,7 @@ dependencies = [ "prost 0.14.4", "rstest", "vortex-array", + "vortex-arrow", "vortex-buffer", "vortex-error", "vortex-mask", @@ -10632,6 +10664,7 @@ dependencies = [ "prost 0.14.4", "rstest", "vortex-array", + "vortex-arrow", "vortex-buffer", "vortex-compressor", "vortex-error", @@ -10675,6 +10708,7 @@ dependencies = [ "taffy", "tokio", "vortex", + "vortex-arrow", "vortex-datafusion", "wasm-bindgen", "wasm-bindgen-futures", @@ -10702,6 +10736,7 @@ dependencies = [ "serde", "serde_json", "vortex", + "vortex-arrow", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", diff --git a/vortex-arrow/src/session.rs b/vortex-arrow/src/session.rs index 240b354b2d5..745e8afeb47 100644 --- a/vortex-arrow/src/session.rs +++ b/vortex-arrow/src/session.rs @@ -40,6 +40,9 @@ use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arc_swap_map::ArcSwapMap; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::ListArray; +use vortex_array::arrays::ListViewArray; use vortex_array::arrays::StructArray; use vortex_array::dtype::DType; use vortex_array::dtype::FieldName; @@ -297,9 +300,7 @@ impl ArrowSession { /// extensions are preserved via [`Self::to_arrow_field`]. pub fn to_arrow_schema(&self, dtype: &DType) -> VortexResult { let DType::Struct(struct_dtype, _) = dtype else { - vortex_error::vortex_bail!( - "to_arrow_schema requires a top-level struct dtype, got {dtype}" - ); + vortex_bail!("to_arrow_schema requires a top-level struct dtype, got {dtype}"); }; let mut fields = Vec::with_capacity(struct_dtype.names().len()); for (name, field_dtype) in struct_dtype.names().iter().zip(struct_dtype.fields()) { @@ -538,10 +539,7 @@ impl ArrowSession { .from_arrow_array(ArrowArrayRef::clone(list.values()), elem_field.as_ref())?; let offsets = list.offsets().clone().into_array(); let validity = nulls(list.nulls(), field.is_nullable())?; - Ok( - vortex_array::arrays::ListArray::try_new(elements, offsets, validity)? - .into_array(), - ) + Ok(ListArray::try_new(elements, offsets, validity)?.into_array()) } DataType::LargeList(elem_field) => { let list = array.as_list::(); @@ -549,23 +547,17 @@ impl ArrowSession { .from_arrow_array(ArrowArrayRef::clone(list.values()), elem_field.as_ref())?; let offsets = list.offsets().clone().into_array(); let validity = nulls(list.nulls(), field.is_nullable())?; - Ok( - vortex_array::arrays::ListArray::try_new(elements, offsets, validity)? - .into_array(), - ) + Ok(ListArray::try_new(elements, offsets, validity)?.into_array()) } DataType::FixedSizeList(elem_field, list_size) => { let fsl = array.as_fixed_size_list(); let elements = self.from_arrow_array(ArrowArrayRef::clone(fsl.values()), elem_field.as_ref())?; let validity = nulls(fsl.nulls(), field.is_nullable())?; - Ok(vortex_array::arrays::FixedSizeListArray::try_new( - elements, - *list_size as u32, - validity, - fsl.len(), - )? - .into_array()) + Ok( + FixedSizeListArray::try_new(elements, *list_size as u32, validity, fsl.len())? + .into_array(), + ) } DataType::ListView(elem_field) => { let list = array.as_list_view::(); @@ -574,10 +566,7 @@ impl ArrowSession { let offsets = list.offsets().clone().into_array(); let sizes = list.sizes().clone().into_array(); let validity = nulls(list.nulls(), field.is_nullable())?; - Ok(vortex_array::arrays::ListViewArray::try_new( - elements, offsets, sizes, validity, - )? - .into_array()) + Ok(ListViewArray::try_new(elements, offsets, sizes, validity)?.into_array()) } DataType::LargeListView(elem_field) => { let list = array.as_list_view::(); @@ -586,10 +575,7 @@ impl ArrowSession { let offsets = list.offsets().clone().into_array(); let sizes = list.sizes().clone().into_array(); let validity = nulls(list.nulls(), field.is_nullable())?; - Ok(vortex_array::arrays::ListViewArray::try_new( - elements, offsets, sizes, validity, - )? - .into_array()) + Ok(ListViewArray::try_new(elements, offsets, sizes, validity)?.into_array()) } _ => ArrayRef::from_arrow(array.as_ref(), field.is_nullable()), }