diff --git a/Cargo.lock b/Cargo.lock index c8ea96df402..e227ed34f88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9769,7 +9769,6 @@ dependencies = [ "arrow-data 58.3.0", "arrow-schema 58.3.0", "arrow-select 58.3.0", - "arrow-string 58.3.0", "async-lock", "bytes", "cfg-if", @@ -9785,6 +9784,7 @@ dependencies = [ "inventory", "itertools 0.14.0", "jiff", + "memchr", "num-traits", "num_enum", "parking_lot", @@ -9794,6 +9794,8 @@ dependencies = [ "prost 0.14.4", "rand 0.10.2", "rand_distr 0.6.0", + "regex", + "regex-syntax", "rstest", "rstest_reuse", "rustc-hash", diff --git a/Cargo.toml b/Cargo.toml index b3c1932809a..109f0a66f0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -181,6 +181,7 @@ libfuzzer-sys = "0.4" libloading = "0.8" liblzma = "0.4" log = { version = "0.4.21" } +memchr = "2.8.0" memmap2 = "0.9.5" mimalloc = "0.1.42" moka = { version = "0.12.10", default-features = false } @@ -215,7 +216,9 @@ quote = "1.0.44" rand = "0.10.1" rand_distr = "0.6" ratatui = { version = "0.30", default-features = false } -regex = "1.11.0" +regex = "1.12" +regex-automata = "0.4" +regex-syntax = "0.8.9" reqwest = { version = "0.13.0", features = [ "blocking", "charset", diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index 310c936a019..ecbcd8e456e 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -29,7 +29,6 @@ arrow-cast = { workspace = true } arrow-data = { workspace = true } arrow-schema = { workspace = true, features = ["canonical_extension_types"] } arrow-select = { workspace = true } -arrow-string = { workspace = true } async-lock = { workspace = true } bytes = { workspace = true } cfg-if = { workspace = true } @@ -43,6 +42,7 @@ humansize = { workspace = true } inventory = { workspace = true } itertools = { workspace = true } jiff = { workspace = true } +memchr = { workspace = true } num-traits = { workspace = true } num_enum = { workspace = true } parking_lot = { workspace = true } @@ -53,6 +53,8 @@ primitive-types = { workspace = true, optional = true, features = [ ] } prost = { workspace = true } rand = { workspace = true } +regex = { workspace = true } +regex-syntax = { workspace = true } rstest = { workspace = true, optional = true } rstest_reuse = { workspace = true, optional = true } rustc-hash = { workspace = true } @@ -136,6 +138,10 @@ harness = false name = "kleene_bool" harness = false +[[bench]] +name = "like" +harness = false + [[bench]] name = "interleave" harness = false diff --git a/vortex-array/benches/like.rs b/vortex-array/benches/like.rs new file mode 100644 index 00000000000..68219724717 --- /dev/null +++ b/vortex-array/benches/like.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#![expect(clippy::unwrap_used)] + +use divan::Bencher; +use rand::RngExt; +use rand::SeedableRng; +use rand::distr::Uniform; +use rand::prelude::StdRng; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::VarBinViewArray; +use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; +use vortex_array::scalar_fn::fns::like::Like; +use vortex_array::scalar_fn::fns::like::LikeOptions; + +fn main() { + divan::main(); +} + +const ARRAY_SIZE: usize = 2_048; + +/// Random lowercase strings of 4..=24 bytes, some with a `hello` infix. +fn strings() -> ArrayRef { + let mut rng = StdRng::seed_from_u64(0); + let len_dist = Uniform::new_inclusive(4usize, 24).unwrap(); + VarBinViewArray::from_iter_str((0..ARRAY_SIZE).map(|i| { + let len = rng.sample(len_dist); + let mut s: String = (0..len) + .map(|_| char::from(rng.random_range(b'a'..=b'z'))) + .collect(); + if i % 7 == 0 { + s.insert_str(len / 2, "hello"); + } + s + })) + .into_array() +} + +fn bench_like(bencher: Bencher, pattern: &str, options: LikeOptions) { + let session = vortex_array::array_session(); + let array = strings(); + bencher + .with_inputs(|| { + ( + Like.try_new_array( + ARRAY_SIZE, + options, + [ + array.clone(), + ConstantArray::new(pattern, ARRAY_SIZE).into_array(), + ], + ) + .unwrap(), + session.create_execution_ctx(), + ) + }) + .bench_values(|(array, mut ctx)| array.execute::(&mut ctx).unwrap()); +} + +#[divan::bench] +fn like_exact(bencher: Bencher) { + bench_like(bencher, "hello", LikeOptions::default()); +} + +#[divan::bench] +fn like_prefix(bencher: Bencher) { + bench_like(bencher, "hello%", LikeOptions::default()); +} + +#[divan::bench] +fn like_suffix(bencher: Bencher) { + bench_like(bencher, "%hello", LikeOptions::default()); +} + +#[divan::bench] +fn like_contains(bencher: Bencher) { + bench_like(bencher, "%hello%", LikeOptions::default()); +} + +#[divan::bench] +fn like_regex(bencher: Bencher) { + bench_like(bencher, "h_llo%w%d", LikeOptions::default()); +} + +#[divan::bench] +fn like_per_row_patterns(bencher: Bencher) { + let session = vortex_array::array_session(); + let array = strings(); + // A non-constant pattern child takes the per-row path; repeated patterns hit the + // compile cache. + let patterns = VarBinViewArray::from_iter_str((0..ARRAY_SIZE).map(|_| "hello%")).into_array(); + bencher + .with_inputs(|| { + ( + Like.try_new_array( + ARRAY_SIZE, + LikeOptions::default(), + [array.clone(), patterns.clone()], + ) + .unwrap(), + session.create_execution_ctx(), + ) + }) + .bench_values(|(array, mut ctx)| array.execute::(&mut ctx).unwrap()); +} + +#[divan::bench] +fn ilike_contains(bencher: Bencher) { + bench_like( + bencher, + "%HELLO%", + LikeOptions { + negated: false, + case_insensitive: true, + }, + ); +} diff --git a/vortex-array/src/scalar_fn/fns/like/mod.rs b/vortex-array/src/scalar_fn/fns/like/mod.rs index 63ced07c8e7..bdedce7112b 100644 --- a/vortex-array/src/scalar_fn/fns/like/mod.rs +++ b/vortex-array/src/scalar_fn/fns/like/mod.rs @@ -2,26 +2,36 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod kernel; +mod pattern; use std::borrow::Cow; use std::fmt::Display; use std::fmt::Formatter; pub use kernel::*; +use pattern::LikePattern; use prost::Message; +use vortex_buffer::BitBuffer; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_error::vortex_err; use vortex_proto::expr as pb; use vortex_session::VortexSession; use vortex_session::registry::CachedId; use crate::ArrayRef; +use crate::Canonical; use crate::ExecutionCtx; -use crate::arrow::Datum; -use crate::arrow::from_arrow_columnar; +use crate::IntoArray; +use crate::arrays::BoolArray; +use crate::arrays::ConstantArray; +use crate::arrays::VarBinViewArray; +use crate::arrays::varbinview::BinaryView; use crate::dtype::DType; +use crate::dtype::Nullability; use crate::expr::Expression; use crate::expr::and; +use crate::scalar::Scalar; use crate::scalar_fn::Arity; use crate::scalar_fn::ChildName; use crate::scalar_fn::ExecutionArgs; @@ -140,7 +150,7 @@ impl ScalarFnVTable for Like { let child = args.get(0)?; let pattern = args.get(1)?; - arrow_like(&child, &pattern, *options, ctx) + execute_like(&child, &pattern, *options, ctx) } fn validity( @@ -163,34 +173,261 @@ impl ScalarFnVTable for Like { } } -/// Implementation of LIKE using the Arrow crate. -pub(crate) fn arrow_like( +/// Native implementation of LIKE over canonical [`VarBinViewArray`]s. +/// +/// Patterns are compiled once per distinct pattern (see [`LikePattern`]) and evaluated per +/// value directly on the view bytes. +pub(crate) fn execute_like( array: &ArrayRef, pattern: &ArrayRef, options: LikeOptions, ctx: &mut ExecutionCtx, ) -> VortexResult { - let nullable = array.dtype().is_nullable() | pattern.dtype().is_nullable(); - let len = array.len(); assert_eq!( array.len(), pattern.len(), - "Arrow Like: length mismatch for {}", + "LIKE: length mismatch for {}", array.encoding_id() ); + let len = array.len(); + let nullability = + Nullability::from(array.dtype().is_nullable() || pattern.dtype().is_nullable()); - // convert the pattern to the preferred array datatype - let lhs = Datum::try_new(array, ctx)?; - let rhs = Datum::try_new_with_target_datatype(pattern, lhs.data_type(), ctx)?; + if len == 0 { + return Ok(Canonical::empty(&DType::Bool(nullability)).into_array()); + } - let result = match (options.negated, options.case_insensitive) { - (false, false) => arrow_string::like::like(&lhs, &rhs)?, - (true, false) => arrow_string::like::nlike(&lhs, &rhs)?, - (false, true) => arrow_string::like::ilike(&lhs, &rhs)?, - (true, true) => arrow_string::like::nilike(&lhs, &rhs)?, - }; + if let Some(pattern_const) = pattern.as_constant() { + let Some(pattern_str) = pattern_const.as_utf8().value() else { + // A null pattern makes every row null. + return Ok( + ConstantArray::new(Scalar::null(DType::Bool(Nullability::Nullable)), len) + .into_array(), + ); + }; + let values = array.clone().execute::(ctx)?; + let haystack = ResolvedViews::new(&values); + // The ASCII case-insensitive fast paths are only sound when the haystack is pure + // ASCII; see `LikePattern::compile`. + let ascii_haystack = + options.case_insensitive && pattern_str.as_str().is_ascii() && haystack.is_ascii(); + let compiled = LikePattern::compile( + pattern_str.as_str(), + options.case_insensitive, + ascii_haystack, + )?; + let bits = eval_pattern(&haystack, &compiled, options.negated); + let validity = values.validity()?.union_nullability(nullability); + return Ok(BoolArray::new(bits, validity).into_array()); + } - from_arrow_columnar(&result, len, nullable, ctx) + // Per-row patterns: compile each pattern individually. + let values = array.clone().execute::(ctx)?; + let patterns = pattern.clone().execute::(ctx)?; + let haystack = ResolvedViews::new(&values); + let pattern_views = ResolvedViews::new(&patterns); + let ascii_haystack = options.case_insensitive && haystack.is_ascii(); + + let mut bits = Vec::with_capacity(len); + // Reuse the previous row's compiled pattern while the pattern bytes repeat, so runs of + // identical patterns (the common case for non-constant pattern children) compile once. + let mut cached: Option<(&[u8], LikePattern)> = None; + for i in 0..len { + let pattern_bytes = pattern_views.bytes(i); + let compiled = match &cached { + Some((bytes, compiled)) if *bytes == pattern_bytes => compiled, + _ => { + let pattern_str = std::str::from_utf8(pattern_bytes) + .map_err(|e| vortex_err!("LIKE pattern is not valid UTF-8: {e}"))?; + let compiled = LikePattern::compile( + pattern_str, + options.case_insensitive, + ascii_haystack && pattern_str.is_ascii(), + )?; + &cached.insert((pattern_bytes, compiled)).1 + } + }; + bits.push(compiled.matches(haystack.bytes(i)) != options.negated); + } + let validity = values + .validity()? + .and(patterns.validity()?)? + .union_nullability(nullability); + Ok(BoolArray::new(BitBuffer::from_iter(bits), validity).into_array()) +} + +/// Resolved views over a canonical [`VarBinViewArray`]: the view structs plus borrowed slices +/// of every data buffer, supporting cheap per-element byte access. +struct ResolvedViews<'a> { + views: &'a [BinaryView], + buffers: Vec<&'a [u8]>, +} + +impl<'a> ResolvedViews<'a> { + fn new(array: &'a VarBinViewArray) -> Self { + Self { + views: array.views(), + buffers: (0..array.data_buffers().len()) + .map(|idx| array.buffer(idx).as_slice()) + .collect(), + } + } + + #[inline] + fn bytes(&self, index: usize) -> &'a [u8] { + let view = &self.views[index]; + if view.is_inlined() { + view.as_inlined().value() + } else { + let view = view.as_view(); + &self.buffers[view.buffer_index as usize][view.as_range()] + } + } + + /// Whether every value (including values under a null) is pure ASCII. + fn is_ascii(&self) -> bool { + (0..self.views.len()).all(|i| self.bytes(i).is_ascii()) + } + + /// The last `suffix_len` bytes of `view`, which must belong to this array. + /// + /// # Safety + /// + /// `suffix_len` must be at most `view.len()`. Buffer bounds are guaranteed by + /// [`VarBinViewArray::validate`], which checks every view against its data buffer at + /// construction. + #[inline] + unsafe fn suffix_bytes_unchecked(&self, view: &'a BinaryView, suffix_len: usize) -> &'a [u8] { + let len = view.len() as usize; + if view.is_inlined() { + // SAFETY: inlined values hold `len <= 12` value bytes, and the caller + // guarantees `suffix_len <= len`. + unsafe { view.as_inlined().value().get_unchecked(len - suffix_len..) } + } else { + let view = view.as_view(); + let end = view.offset as usize + len; + // SAFETY: validated views reference `buffer_index < buffers.len()` and bytes + // `offset..offset + len` within that buffer. + unsafe { + self.buffers + .get_unchecked(view.buffer_index as usize) + .get_unchecked(end - suffix_len..end) + } + } + } +} + +/// Evaluate `pattern` against every element of `haystack`. +/// +/// The equality, prefix, and suffix patterns exploit the view layout: a view stores the value +/// length and its first four bytes inline, which settles most elements without touching the +/// data buffers (values of up to 12 bytes are stored entirely inline). +fn eval_pattern(haystack: &ResolvedViews<'_>, pattern: &LikePattern, negated: bool) -> BitBuffer { + let len = haystack.views.len(); + match pattern { + LikePattern::Eq(needle) if needle.len() <= BinaryView::MAX_INLINED_SIZE => { + // The needle fits in a view, so equality is a single 16-byte comparison: a view + // of a different length or prefix can never share the same bit pattern. + let needle_view = BinaryView::new_inlined(needle).as_u128(); + BitBuffer::collect_bool(len, |i| { + (haystack.views[i].as_u128() == needle_view) != negated + }) + } + LikePattern::Eq(needle) => { + // Compare the view head (length plus 4-byte prefix) first; only views that agree + // on both dereference their data buffer for the remaining bytes. + let needle_head = needle_head(needle); + BitBuffer::collect_bool(len, |i| { + let view = &haystack.views[i]; + let matched = + view_head(view) == needle_head && haystack.bytes(i)[4..] == needle[4..]; + matched != negated + }) + } + LikePattern::StartsWith(needle) => { + // A branch-free masked comparison of the view's inline 4-byte prefix rejects + // almost every element without touching the data buffers; only elements whose + // prefix matches compare the remaining needle bytes. + let needle_len = needle.len(); + let prefix_len = needle_len.min(4); + let needle_prefix = u32::from_le_bytes({ + let mut padded = [0u8; 4]; + padded[..prefix_len].copy_from_slice(&needle[..prefix_len]); + padded + }); + let prefix_mask = if prefix_len == 4 { + u32::MAX + } else { + (1u32 << (8 * prefix_len)) - 1 + }; + BitBuffer::collect_bool(len, |i| { + let view = &haystack.views[i]; + let matched = view.len() as usize >= needle_len + && (view_prefix(view) & prefix_mask) == needle_prefix + && (needle_len <= 4 || haystack.bytes(i)[4..needle_len] == needle[4..]); + matched != negated + }) + } + LikePattern::EndsWith(needle) => { + // Inlined values compare their suffix inside the view struct without touching the + // data buffers; reference views slice exactly the suffix out of their buffer. + let needle_len = needle.len(); + BitBuffer::collect_bool(len, |i| { + // SAFETY: `i` is below the array length, and the suffix length is only read + // once the view is known to be at least `needle_len` long. + let matched = unsafe { + let view = haystack.views.get_unchecked(i); + view.len() as usize >= needle_len + && bytes_eq(haystack.suffix_bytes_unchecked(view, needle_len), needle) + }; + matched != negated + }) + } + LikePattern::IEqAscii(needle) => BitBuffer::collect_bool(len, |i| { + let view = &haystack.views[i]; + let matched = view.len() as usize == needle.len() + && haystack.bytes(i).eq_ignore_ascii_case(needle); + matched != negated + }), + LikePattern::Contains(finder, needle_len) => BitBuffer::collect_bool(len, |i| { + let view = &haystack.views[i]; + let matched = + view.len() as usize >= *needle_len && finder.find(haystack.bytes(i)).is_some(); + matched != negated + }), + _ => BitBuffer::collect_bool(len, |i| pattern.matches(haystack.bytes(i)) != negated), + } +} + +/// Byte equality as an inlined loop with early exit. +/// +/// Faster than slice `==` for the short, unpredictable-length comparisons in this module, +/// which otherwise lower to a `memcmp` libcall per element. +#[inline] +fn bytes_eq(lhs: &[u8], rhs: &[u8]) -> bool { + lhs.len() == rhs.len() && std::iter::zip(lhs, rhs).all(|(l, r)| l == r) +} + +/// The leading 8 bytes of a view: the `u32` length plus the first 4 bytes of the value +/// (zero-padded for values shorter than 4 bytes). +#[inline] +#[expect(clippy::cast_possible_truncation, reason = "intentional bit slicing")] +fn view_head(view: &BinaryView) -> u64 { + view.as_u128() as u64 +} + +/// The view head a needle of more than 4 bytes would have: its length plus first 4 bytes. +fn needle_head(needle: &[u8]) -> u64 { + let prefix: [u8; 4] = [needle[0], needle[1], needle[2], needle[3]]; + (needle.len() as u64) | (u64::from(u32::from_le_bytes(prefix)) << 32) +} + +/// The first 4 value bytes stored inline in any view (zero-padded for values shorter than +/// 4 bytes), as a raw little-endian `u32` in memory order. +#[inline] +#[expect(clippy::cast_possible_truncation, reason = "intentional bit slicing")] +fn view_prefix(view: &BinaryView) -> u32 { + (view.as_u128() >> 32) as u32 } /// Variants of the LIKE filter that we know how to turn into a stats pruning predicate. @@ -243,10 +480,16 @@ impl<'a> LikeVariant<'a> { mod tests { use std::borrow::Cow; + use rstest::rstest; + use crate::IntoArray; use crate::VortexSessionExecute; use crate::array_session; use crate::arrays::BoolArray; + use crate::arrays::ConstantArray; + use crate::arrays::VarBinArray; + use crate::arrays::VarBinViewArray; + use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::assert_arrays_eq; use crate::dtype::DType; use crate::dtype::Nullability; @@ -256,8 +499,205 @@ mod tests { use crate::expr::not; use crate::expr::not_ilike; use crate::expr::root; + use crate::scalar::Scalar; + use crate::scalar_fn::fns::like::Like; + use crate::scalar_fn::fns::like::LikeOptions; use crate::scalar_fn::fns::like::LikeVariant; + fn run_like( + array: crate::ArrayRef, + pattern: crate::ArrayRef, + options: LikeOptions, + ) -> crate::ArrayRef { + let len = array.len(); + Like.try_new_array(len, options, [array, pattern]).unwrap() + } + + #[rstest] + // Exact, prefix, suffix, contains fast paths. + #[case("hello", [true, false, false, false])] + #[case("he%", [true, false, true, false])] + #[case("%llo", [true, false, false, true])] + #[case("%ell%", [true, false, false, true])] + // Wildcards that require the regex path. + #[case("h_llo", [true, false, false, false])] + #[case("h%o", [true, false, false, false])] + #[case("%", [true, true, true, true])] + #[case("_____", [true, true, false, true])] + fn test_like_patterns(#[case] pattern: &str, #[case] expected: [bool; 4]) { + let mut ctx = array_session().create_execution_ctx(); + let array = + VarBinViewArray::from_iter_str(["hello", "world", "help", "jello"]).into_array(); + let result = run_like( + array, + ConstantArray::new(pattern, 4).into_array(), + LikeOptions::default(), + ); + assert_arrays_eq!(result, BoolArray::from_iter(expected), &mut ctx); + } + + #[test] + fn test_like_escapes() { + let mut ctx = array_session().create_execution_ctx(); + let array = VarBinViewArray::from_iter_str(["100%", "100x", "a_b", "axb"]).into_array(); + + let result = run_like( + array.clone(), + ConstantArray::new(r"100\%", 4).into_array(), + LikeOptions::default(), + ); + assert_arrays_eq!( + result, + BoolArray::from_iter([true, false, false, false]), + &mut ctx + ); + + let result = run_like( + array, + ConstantArray::new(r"a\_b", 4).into_array(), + LikeOptions::default(), + ); + assert_arrays_eq!( + result, + BoolArray::from_iter([false, false, true, false]), + &mut ctx + ); + } + + #[test] + fn test_like_regex_meta_characters_are_literal() { + let mut ctx = array_session().create_execution_ctx(); + let array = VarBinViewArray::from_iter_str(["a.c", "abc", "a$c"]).into_array(); + let result = run_like( + array, + ConstantArray::new("a.%", 3).into_array(), + LikeOptions::default(), + ); + assert_arrays_eq!(result, BoolArray::from_iter([true, false, false]), &mut ctx); + } + + #[test] + fn test_like_unicode() { + let mut ctx = array_session().create_execution_ctx(); + let array = + VarBinViewArray::from_iter_str(["h\u{00a3}llo", "hxllo", "h\u{00a3}xllo"]).into_array(); + // `_` matches exactly one character of any byte width. + let result = run_like( + array, + ConstantArray::new("h_llo", 3).into_array(), + LikeOptions::default(), + ); + assert_arrays_eq!(result, BoolArray::from_iter([true, true, false]), &mut ctx); + } + + #[test] + fn test_nlike() { + let mut ctx = array_session().create_execution_ctx(); + let array = VarBinViewArray::from_iter_str(["hello", "world"]).into_array(); + let result = run_like( + array, + ConstantArray::new("he%", 2).into_array(), + LikeOptions { + negated: true, + case_insensitive: false, + }, + ); + assert_arrays_eq!(result, BoolArray::from_iter([false, true]), &mut ctx); + } + + #[test] + fn test_ilike() { + let mut ctx = array_session().create_execution_ctx(); + let ilike = LikeOptions { + negated: false, + case_insensitive: true, + }; + + // Pure-ASCII data takes the ASCII fast paths. + let array = VarBinViewArray::from_iter_str(["HELLO", "world", "Help"]).into_array(); + let result = run_like(array, ConstantArray::new("he%", 3).into_array(), ilike); + assert_arrays_eq!(result, BoolArray::from_iter([true, false, true]), &mut ctx); + + // Non-ASCII data requires full case folding: `k` matches U+212A KELVIN SIGN. + let array = VarBinViewArray::from_iter_str(["\u{212a}", "k", "x"]).into_array(); + let result = run_like(array, ConstantArray::new("k", 3).into_array(), ilike); + assert_arrays_eq!(result, BoolArray::from_iter([true, true, false]), &mut ctx); + } + + #[test] + fn test_nilike() { + let mut ctx = array_session().create_execution_ctx(); + let array = VarBinViewArray::from_iter_str(["HELLO", "world"]).into_array(); + let result = run_like( + array, + ConstantArray::new("he%", 2).into_array(), + LikeOptions { + negated: true, + case_insensitive: true, + }, + ); + assert_arrays_eq!(result, BoolArray::from_iter([false, true]), &mut ctx); + } + + #[test] + fn test_like_nullable_input() { + let mut ctx = array_session().create_execution_ctx(); + let array = VarBinViewArray::from_iter_nullable_str([Some("hello"), None, Some("help")]) + .into_array(); + let result = run_like( + array, + ConstantArray::new("he%", 3).into_array(), + LikeOptions::default(), + ); + assert_arrays_eq!( + result, + BoolArray::from_iter([Some(true), None, Some(true)]), + &mut ctx + ); + } + + #[test] + fn test_like_null_pattern() { + let mut ctx = array_session().create_execution_ctx(); + let array = VarBinViewArray::from_iter_str(["hello", "world"]).into_array(); + let result = run_like( + array, + ConstantArray::new(Scalar::null(DType::Utf8(Nullability::Nullable)), 2).into_array(), + LikeOptions::default(), + ); + assert_arrays_eq!(result, BoolArray::from_iter([None, None]), &mut ctx); + } + + #[test] + fn test_like_per_row_patterns() { + let mut ctx = array_session().create_execution_ctx(); + let array = VarBinViewArray::from_iter_str(["hello", "hello", "hello"]).into_array(); + let patterns = VarBinViewArray::from_iter_str(["he%", "%world", "h_llo"]).into_array(); + let result = run_like(array, patterns, LikeOptions::default()); + assert_arrays_eq!(result, BoolArray::from_iter([true, false, true]), &mut ctx); + } + + #[test] + fn test_like_non_canonical_input() { + let mut ctx = array_session().create_execution_ctx(); + // VarBin (not the canonical VarBinView) is canonicalized before evaluation. + let array = VarBinArray::from_iter( + [Some("hello"), Some("world")], + DType::Utf8(Nullability::Nullable), + ) + .into_array(); + let result = run_like( + array, + ConstantArray::new("he%", 2).into_array(), + LikeOptions::default(), + ); + assert_arrays_eq!( + result, + BoolArray::from_iter([Some(true), Some(false)]), + &mut ctx + ); + } + #[test] fn invert_booleans() { let not_expr = not(root()); diff --git a/vortex-array/src/scalar_fn/fns/like/pattern.rs b/vortex-array/src/scalar_fn/fns/like/pattern.rs new file mode 100644 index 00000000000..8a667159ecb --- /dev/null +++ b/vortex-array/src/scalar_fn/fns/like/pattern.rs @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Compiled SQL LIKE patterns evaluated over UTF-8 byte slices. +//! +//! A [`LikePattern`] is compiled once per pattern and then evaluated per value. Patterns that +//! reduce to plain equality, prefix, suffix, or substring searches use direct byte comparisons +//! (with `memchr`'s SIMD substring search for the contains case); everything else falls back to +//! a regex translated from the LIKE pattern. +//! +//! The pattern grammar and its regex translation mirror the `arrow-string` LIKE kernels so that +//! results are identical to the previous Arrow-backed implementation: `%` matches any sequence +//! of characters (including across newlines), `_` matches exactly one character, and `\` escapes +//! the next character. + +use memchr::memchr3; +use memchr::memmem::Finder; +use regex::bytes::Regex; +use regex::bytes::RegexBuilder; +use vortex_error::VortexResult; +use vortex_error::vortex_err; + +/// A LIKE pattern compiled for repeated evaluation against UTF-8 haystacks. +pub(crate) enum LikePattern { + /// The pattern has no wildcards: plain byte equality. + Eq(Vec), + /// `prefix%`: byte prefix comparison. + StartsWith(Vec), + /// `%suffix`: byte suffix comparison. + EndsWith(Vec), + /// `%needle%`: SIMD substring search, along with the needle length. + /// + /// The searcher is boxed to keep this variant close in size to the others. + Contains(Box>, usize), + /// Equality ignoring ASCII case (ILIKE over ASCII-only data). + IEqAscii(Vec), + /// Prefix comparison ignoring ASCII case (ILIKE over ASCII-only data). + IStartsWithAscii(Vec), + /// Suffix comparison ignoring ASCII case (ILIKE over ASCII-only data). + IEndsWithAscii(Vec), + /// Everything else: the LIKE pattern translated to an anchored regex. + Regex(Regex), +} + +impl LikePattern { + /// Compile `pattern`, with `case_insensitive` selecting ILIKE semantics. + /// + /// `ascii_haystack` enables the ASCII-only case-insensitive fast paths and must only be + /// `true` when every haystack this pattern will be evaluated against is pure ASCII: + /// full case folding can match ASCII pattern characters against non-ASCII haystack + /// characters (e.g. `k` folds to U+212A KELVIN SIGN), which the ASCII fast paths would + /// miss. + pub(crate) fn compile( + pattern: &str, + case_insensitive: bool, + ascii_haystack: bool, + ) -> VortexResult { + if case_insensitive { + if ascii_haystack && pattern.is_ascii() { + if !contains_like_pattern(pattern) { + return Ok(Self::IEqAscii(pattern.as_bytes().to_vec())); + } else if pattern.ends_with('%') + && !contains_like_pattern(&pattern[..pattern.len() - 1]) + { + return Ok(Self::IStartsWithAscii( + pattern.as_bytes()[..pattern.len() - 1].to_vec(), + )); + } else if pattern.starts_with('%') && !contains_like_pattern(&pattern[1..]) { + return Ok(Self::IEndsWithAscii(pattern.as_bytes()[1..].to_vec())); + } + } + return Ok(Self::Regex(regex_like(pattern, true)?)); + } + + Ok(if !contains_like_pattern(pattern) { + Self::Eq(pattern.as_bytes().to_vec()) + } else if pattern.ends_with('%') && !contains_like_pattern(&pattern[..pattern.len() - 1]) { + Self::StartsWith(pattern.as_bytes()[..pattern.len() - 1].to_vec()) + } else if pattern.starts_with('%') && !contains_like_pattern(&pattern[1..]) { + Self::EndsWith(pattern.as_bytes()[1..].to_vec()) + } else if pattern.starts_with('%') + && pattern.ends_with('%') + && !contains_like_pattern(&pattern[1..pattern.len() - 1]) + { + Self::Contains( + Box::new(Finder::new(&pattern.as_bytes()[1..pattern.len() - 1]).into_owned()), + pattern.len() - 2, + ) + } else { + Self::Regex(regex_like(pattern, false)?) + }) + } + + /// Evaluate this pattern against `haystack`, which must be valid UTF-8. + #[inline] + pub(crate) fn matches(&self, haystack: &[u8]) -> bool { + match self { + Self::Eq(needle) => needle == haystack, + Self::StartsWith(needle) => { + needle.len() <= haystack.len() && &haystack[..needle.len()] == needle.as_slice() + } + Self::EndsWith(needle) => { + needle.len() <= haystack.len() + && &haystack[haystack.len() - needle.len()..] == needle.as_slice() + } + Self::Contains(finder, needle_len) => { + haystack.len() >= *needle_len && finder.find(haystack).is_some() + } + Self::IEqAscii(needle) => haystack.eq_ignore_ascii_case(needle), + Self::IStartsWithAscii(needle) => { + needle.len() <= haystack.len() + && haystack[..needle.len()].eq_ignore_ascii_case(needle) + } + Self::IEndsWithAscii(needle) => { + needle.len() <= haystack.len() + && haystack[haystack.len() - needle.len()..].eq_ignore_ascii_case(needle) + } + Self::Regex(regex) => regex.is_match(haystack), + } + } +} + +/// Returns whether `pattern` contains a wildcard (`%`, `_`) or an escape (`\`). +fn contains_like_pattern(pattern: &str) -> bool { + memchr3(b'%', b'_', b'\\', pattern.as_bytes()).is_some() +} + +/// Transforms a LIKE `pattern` into an equivalent anchored regex. +/// +/// This is a port of the `arrow-string` translation so that pattern semantics are unchanged: +/// +/// 1. `%` => `.*` (a leading/trailing `%` truncates the anchor instead, e.g. `%foo%` => `foo` +/// rather than `^.*foo.*$`); +/// 2. `_` => `.`; +/// 3. regex meta characters are escaped so they match literally; +/// 4. `\x` matches `x` literally (a trailing `\` matches a literal backslash). +/// +/// The regex runs over the haystack bytes directly (`regex::bytes`) in Unicode mode, which +/// matches identically to a `&str` regex on valid UTF-8 input. +fn regex_like(pattern: &str, case_insensitive: bool) -> VortexResult { + let mut result = String::with_capacity(pattern.len() * 2); + let mut chars_iter = pattern.chars().peekable(); + match chars_iter.peek() { + // If the pattern starts with `%`, avoid starting the regex with a slow but + // meaningless `^.*`. + Some('%') => { + chars_iter.next(); + } + _ => result.push('^'), + }; + + while let Some(c) = chars_iter.next() { + match c { + '\\' => { + match chars_iter.peek() { + Some(&next) => { + if regex_syntax::is_meta_character(next) { + result.push('\\'); + } + result.push(next); + // Skip the next char as it is already appended. + chars_iter.next(); + } + None => { + // A trailing backslash matches a literal backslash. + result.push('\\'); + result.push('\\'); + } + } + } + '%' => result.push_str(".*"), + '_' => result.push('.'), + c => { + if regex_syntax::is_meta_character(c) { + result.push('\\'); + } + result.push(c); + } + } + } + // Instead of ending the regex with `.*$` and making it needlessly slow, just end it. + if result.ends_with(".*") { + result.pop(); + result.pop(); + } else { + result.push('$'); + } + RegexBuilder::new(&result) + .case_insensitive(case_insensitive) + .dot_matches_new_line(true) + .build() + .map_err(|e| vortex_err!("Unable to build regex from LIKE pattern: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn like(pattern: &str) -> LikePattern { + LikePattern::compile(pattern, false, false).unwrap() + } + + #[test] + fn compile_fast_paths() { + assert!(matches!(like("foo"), LikePattern::Eq(_))); + assert!(matches!(like("foo%"), LikePattern::StartsWith(_))); + assert!(matches!(like("%foo"), LikePattern::EndsWith(_))); + assert!(matches!(like("%foo%"), LikePattern::Contains(..))); + assert!(matches!(like("%"), LikePattern::StartsWith(_))); + assert!(matches!(like("%%"), LikePattern::Contains(..))); + assert!(matches!(like("f%o"), LikePattern::Regex(_))); + assert!(matches!(like("f_o"), LikePattern::Regex(_))); + assert!(matches!(like(r"foo\%"), LikePattern::Regex(_))); + } + + #[test] + fn regex_translation() { + // (pattern, expected regex) pairs from the arrow-string translation. + let cases = [ + (r"%foobar%", r"foobar"), + (r"foo%bar", r"^foo.*bar$"), + (r"foo_bar", r"^foo.bar$"), + (r"\%\_", r"^%_$"), + (r"\a", r"^a$"), + (r"\\%", r"^\\"), + (r"\\a", r"^\\a$"), + (r".", r"^\.$"), + (r"$", r"^\$$"), + (r"\\", r"^\\$"), + ]; + for (pattern, expected) in cases { + assert_eq!(regex_like(pattern, false).unwrap().to_string(), expected); + } + } + + #[test] + fn matches_wildcards() { + assert!(like("h_llo w%d").matches(b"hello world")); + assert!(like("h_llo w%d").matches("h\u{00a3}llo wd".as_bytes())); + assert!(!like("h_llo w%d").matches(b"hxxllo world")); + // `%` crosses newlines. + assert!(like("hello%world").matches(b"hello\nworld")); + // `_` matches exactly one character, of any width. + assert!(like("_").matches("\u{00a3}".as_bytes())); + assert!(!like("_").matches("\u{00a3}x".as_bytes())); + } + + #[test] + fn matches_escapes() { + assert!(like(r"100\%").matches(b"100%")); + assert!(!like(r"100\%").matches(b"100x")); + assert!(like(r"a\_b").matches(b"a_b")); + assert!(!like(r"a\_b").matches(b"axb")); + // A trailing backslash matches a literal backslash. + assert!(like("trailing\\").matches(b"trailing\\")); + } + + #[test] + fn matches_case_insensitive() { + let ilike = |pattern: &str| LikePattern::compile(pattern, true, false).unwrap(); + assert!(ilike("hello%").matches(b"HELLO WORLD")); + assert!(ilike("%WORLD").matches(b"hello world")); + // Full case folding: the ASCII pattern `k` matches U+212A KELVIN SIGN. + assert!(ilike("k").matches("\u{212a}".as_bytes())); + // Greek sigma case folds across all three forms. + assert!(ilike("\u{03c3}").matches("\u{03a3}".as_bytes())); + assert!(ilike("\u{03c3}").matches("\u{03c2}".as_bytes())); + + // The ASCII fast paths agree with the regex on ASCII haystacks. + let ascii = |pattern: &str| LikePattern::compile(pattern, true, true).unwrap(); + assert!(matches!(ascii("abc"), LikePattern::IEqAscii(_))); + assert!(matches!(ascii("abc%"), LikePattern::IStartsWithAscii(_))); + assert!(matches!(ascii("%abc"), LikePattern::IEndsWithAscii(_))); + assert!(ascii("abc").matches(b"AbC")); + assert!(ascii("abc%").matches(b"ABCDEF")); + assert!(ascii("%def").matches(b"ABCDEF")); + assert!(!ascii("abc").matches(b"AbCd")); + } +}