diff --git a/include/xsimd/arch/xsimd_avx.hpp b/include/xsimd/arch/xsimd_avx.hpp index d03f5cf79..7d745f140 100644 --- a/include/xsimd/arch/xsimd_avx.hpp +++ b/include/xsimd/arch/xsimd_avx.hpp @@ -985,6 +985,19 @@ namespace xsimd { return _mm256_insertf128_pd(_mm256_setzero_pd(), hi, 1); } + + // 128-bit batch into the lower half, upper half zero (no instruction) + template + XSIMD_INLINE batch zero_extend_lo(batch const& lo) noexcept + { + return _mm256_zextps128_ps256(lo); + } + + template + XSIMD_INLINE batch zero_extend_lo(batch const& lo) noexcept + { + return _mm256_zextpd128_pd256(lo); + } } // Runtime-mask load (float/double). @@ -1021,16 +1034,27 @@ namespace xsimd template ::value>> XSIMD_INLINE batch load_masked(T const* mem, batch_bool_constant mask, convert, Mode, requires_arch) noexcept { - using int_t = as_integer_t; constexpr size_t half_size = batch::size / 2; - using half_arch = typename ::xsimd::make_sized_batch_t::arch_type; + using half_batch = make_sized_batch_t; + using half_arch = typename half_batch::arch_type; - // lower 128-bit half - XSIMD_IF_CONSTEXPR(mask.countl_zero() >= half_size) + // exactly the lower 128-bit half: one plain load, upper lanes zero + XSIMD_IF_CONSTEXPR(mask.prefix() == half_size) { - constexpr auto mlo = ::xsimd::detail::lower_half(batch_bool_constant {}); - const auto lo = load_masked(reinterpret_cast(mem), mlo, convert {}, Mode {}, half_arch {}); - return bitwise_cast(batch(_mm256_zextsi128_si256(lo))); + return detail::zero_extend_lo(half_batch::load(mem, Mode {})); + } + // lower 128-bit half: stay in the value domain so the half kernel can + // lower pure-prefix shapes to plain narrow moves (movss/movlps/movsd) + else XSIMD_IF_CONSTEXPR(mask.countl_zero() >= half_size) + { + constexpr auto mlo = ::xsimd::detail::lower_half(mask); + const auto lo = load_masked(mem, mlo, convert {}, Mode {}, half_arch {}); + return detail::zero_extend_lo(lo); + } + // exactly the upper 128-bit half: one plain load into the upper lanes + else XSIMD_IF_CONSTEXPR(mask.suffix() == half_size) + { + return detail::zero_extend(half_batch::load(mem + half_size, Mode {})); } // upper 128-bit half else XSIMD_IF_CONSTEXPR(mask.countr_zero() >= half_size) @@ -1076,8 +1100,30 @@ namespace xsimd using half_batch = ::xsimd::make_sized_batch_t; using half_arch = typename half_batch::arch_type; + // exactly the lower 128-bit half: one plain store + XSIMD_IF_CONSTEXPR(mask.prefix() == half_size) + { + const half_batch lo = detail::lower_half(src); + lo.store(mem, Mode {}); + } + // prefix crossing the 128-bit boundary: plain lower half + prefix-masked + // upper half. Never emits vmaskmov, which does not store-forward. + else XSIMD_IF_CONSTEXPR(mask.prefix() > half_size && mask.prefix() < batch::size) + { + const half_batch lo = detail::lower_half(src); + lo.store(mem, Mode {}); + constexpr auto mhi = ::xsimd::detail::upper_half(mask); + const half_batch hi = detail::upper_half(src); + store_masked(mem + half_size, hi, mhi, Mode {}, half_arch {}); + } + // exactly the upper 128-bit half: one plain store + else XSIMD_IF_CONSTEXPR(mask.suffix() == half_size) + { + const half_batch hi = detail::upper_half(src); + hi.store(mem + half_size, Mode {}); + } // lower 128-bit half - XSIMD_IF_CONSTEXPR(mask.countl_zero() >= half_size) + else XSIMD_IF_CONSTEXPR(mask.countl_zero() >= half_size) { constexpr auto mlo = ::xsimd::detail::lower_half(mask); const half_batch lo = detail::lower_half(src); diff --git a/include/xsimd/arch/xsimd_avx2_128.hpp b/include/xsimd/arch/xsimd_avx2_128.hpp index 76861f6dd..aad0bc3c1 100644 --- a/include/xsimd/arch/xsimd_avx2_128.hpp +++ b/include/xsimd/arch/xsimd_avx2_128.hpp @@ -141,19 +141,36 @@ namespace xsimd } } - // constant masks gain nothing on a single register; forward to runtime + // Masks that lower to plain moves go to the sse2 float/double kernels + // (int bitcast to same-width float); the rest keep the native vpmaskmov. template ::value && (sizeof(T) == 4 || sizeof(T) == 8)>> XSIMD_INLINE batch load_masked(T const* mem, batch_bool_constant mask, convert, Mode, requires_arch) noexcept { - return load_masked(mem, mask.as_batch_bool(), convert {}, Mode {}, avx2_128 {}); + XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask)) + { + using F = std::conditional_t; + return bitwise_cast(batch(load_masked(reinterpret_cast(mem), batch_bool_constant {}, convert {}, Mode {}, sse2 {}))); + } + else + { + return load_masked(mem, mask.as_batch_bool(), convert {}, Mode {}, avx2_128 {}); + } } template ::value && (sizeof(T) == 4 || sizeof(T) == 8)>> XSIMD_INLINE void store_masked(T* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept { - store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx2_128 {}); + XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask)) + { + using F = std::conditional_t; + store_masked(reinterpret_cast(mem), bitwise_cast(src), batch_bool_constant {}, Mode {}, sse2 {}); + } + else + { + store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx2_128 {}); + } } template