feat(buffer): add allocator-backed storage

Cargo.toml

@@ -97,6 +97,7 @@ rust-version = "1.95"
 version = "0.1.0"
 
 [workspace.dependencies]
+allocator-api2 = "0.2.21"
 alp = "0.0.2"
 anyhow = "1.0.100"
 arbitrary = "1.3.2"

encodings/zstd/src/array.rs

@@ -1144,7 +1144,7 @@ impl ZstdData {
 
         let value_bytes = values.buffer_handle().try_to_host_sync()?;
         // Align frames to buffer alignment. This is necessary for overaligned buffers.
-        let alignment = *value_bytes.alignment();
+        let alignment = value_bytes.alignment().as_usize();
         let step_width = (values_per_frame * byte_width).div_ceil(alignment) * alignment;
 
         let frame_byte_starts = (0..n_values * byte_width)

encodings/zstd/src/zstd_buffers.rs

@@ -357,7 +357,7 @@ fn compute_output_layout(
     let mut total_size = 0usize;
 
     for (&size, &alignment) in output_sizes.iter().zip(output_alignments.iter()) {
-        total_size = total_size.next_multiple_of(*alignment);
+        total_size = total_size.next_multiple_of(alignment.as_usize());
         offsets.push(total_size);
         total_size += size;
     }

vortex-array/src/serde.rs

@@ -84,7 +84,7 @@ impl ArrayRef {
             .unwrap_or_else(FlatBuffer::alignment);
 
         // Create a shared buffer of zeros we can use for padding
-        let zeros = ByteBuffer::zeroed(*max_alignment);
+        let zeros = ByteBuffer::zeroed(max_alignment.as_usize());
 
         // We push an empty buffer with the maximum alignment, so then subsequent buffers
         // will be aligned. For subsequent buffers, we always push a 1-byte alignment.
@@ -96,7 +96,7 @@ impl ArrayRef {
         // Push all the array buffers with padding as necessary.
         for buffer in array_buffers {
             let padding = if options.include_padding {
-                let padding = pos.next_multiple_of(*buffer.alignment()) - pos;
+                let padding = pos.next_multiple_of(buffer.alignment().as_usize()) - pos;
                 if padding > 0 {
                     pos += padding;
                     buffers.push(zeros.slice(0..padding));
@@ -139,7 +139,7 @@ impl ArrayRef {
         let fb_length = fb_buffer.len();
 
         if options.include_padding {
-            let padding = pos.next_multiple_of(*FlatBuffer::alignment()) - pos;
+            let padding = pos.next_multiple_of(FlatBuffer::alignment().as_usize()) - pos;
             if padding > 0 {
                 buffers.push(zeros.slice(0..padding));
             }

vortex-buffer/Cargo.toml

@@ -23,6 +23,7 @@ serde = ["dep:serde", "serde/serde_derive"]
 warn-copy = ["dep:tracing"]
 
 [dependencies]
+allocator-api2 = { workspace = true }
 arrow-buffer = { workspace = true }
 bitvec = { workspace = true }
 bytes = { workspace = true }

vortex-buffer/src/alignment.rs

@@ -2,7 +2,6 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::fmt::Display;
-use std::ops::Deref;
 
 use vortex_error::VortexError;
 use vortex_error::VortexExpect;
@@ -12,9 +11,9 @@ use vortex_error::vortex_err;
 
 /// The alignment of a buffer.
 ///
-/// This type is a wrapper around `usize` that ensures the alignment is a non-zero power of 2.
+/// This type stores the base-2 exponent of a non-zero power-of-two alignment.
 #[derive(Clone, Debug, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct Alignment(usize);
+pub struct Alignment(u8);
 
 impl Alignment {
     /// Largest alignment accepted from untrusted serialized input.
@@ -41,10 +40,11 @@ impl Alignment {
     ///
     /// Panics if `align` is zero or is not a power of 2.
     #[inline]
+    #[expect(clippy::cast_possible_truncation, reason = "usize has at most 64 bits")]
     pub const fn new(align: usize) -> Self {
         assert!(align > 0, "Alignment must be greater than 0");
         assert!(align.is_power_of_two(), "Alignment must be a power of 2");
-        Self(align)
+        Self(align.trailing_zeros() as u8)
     }
 
     /// Create a new 1-byte alignment.
@@ -104,7 +104,7 @@ impl Alignment {
     #[inline]
     pub const fn is_offset_aligned(&self, offset: usize) -> bool {
         // Alignment is always a power of 2, so a mask test is equivalent to `offset % self == 0`.
-        offset & (self.0 - 1) == 0
+        offset & (self.as_usize() - 1) == 0
     }
 
     /// Check if the given pointer is aligned to this alignment.
@@ -115,8 +115,7 @@ impl Alignment {
 
     /// Returns the log2 of the alignment.
     pub fn exponent(&self) -> u8 {
-        u8::try_from(self.0.trailing_zeros())
-            .vortex_expect("alignment is a power of 2 within usize, so its exponent fits in u8")
+        self.0
     }
 
     /// Create from the log2 exponent of the alignment.
@@ -131,7 +130,7 @@ impl Alignment {
             (exponent as u32) < usize::BITS,
             "Alignment exponent must fit in usize"
         );
-        Self::new(1 << exponent)
+        Self(exponent)
     }
 
     /// Create from the log2 exponent of the alignment, returning an error rather than panicking if
@@ -166,20 +165,17 @@ impl Alignment {
         }
         Ok(alignment)
     }
-}
 
-impl Display for Alignment {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.0)
+    /// Return the alignment in bytes.
+    #[inline]
+    pub const fn as_usize(self) -> usize {
+        1 << self.0
     }
 }
 
-impl Deref for Alignment {
-    type Target = usize;
-
-    #[inline]
-    fn deref(&self) -> &Self::Target {
-        &self.0
+impl Display for Alignment {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.as_usize())
     }
 }
 
@@ -200,14 +196,14 @@ impl From<u16> for Alignment {
 impl From<Alignment> for usize {
     #[inline]
     fn from(value: Alignment) -> Self {
-        value.0
+        value.as_usize()
     }
 }
 
 impl From<Alignment> for u32 {
     #[inline]
     fn from(value: Alignment) -> Self {
-        u32::try_from(value.0).vortex_expect("Alignment must fit into u32")
+        u32::try_from(value.as_usize()).vortex_expect("Alignment must fit into u32")
     }
 }
 
@@ -225,7 +221,7 @@ impl TryFrom<u32> for Alignment {
             return Err(vortex_err!("Alignment must be a power of 2, got {value}"));
         }
 
-        Ok(Self(value))
+        Ok(Self::new(value))
     }
 }
 
@@ -243,7 +239,7 @@ mod test {
     fn alignment_above_u16() {
         // 64KiB alignment (one past `u16::MAX`) is valid — common on ARM with 64K pages.
         let alignment = Alignment::new(u16::MAX as usize + 1);
-        assert_eq!(*alignment, 1 << 16);
+        assert_eq!(alignment.as_usize(), 1 << 16);
         assert_eq!(alignment, Alignment::from_exponent(16));
     }