Skip to content

Avx256 constant mask plain moves #1378

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DiamonDinoia merged 1 commit into from
Jul 10, 2026
+74 −11
Merged

Avx256 constant mask plain moves #1378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 54 additions & 8 deletions include/xsimd/arch/xsimd_avx.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -985,6 +985,19 @@ namespace xsimd
{
return _mm256_insertf128_pd(_mm256_setzero_pd(), hi, 1);
}

// 128-bit batch into the lower half, upper half zero (no instruction)
template <class A, class SrcA>
XSIMD_INLINE batch<float, A> zero_extend_lo(batch<float, SrcA> const& lo) noexcept
{
return _mm256_zextps128_ps256(lo);
}

template <class A, class SrcA>
XSIMD_INLINE batch<double, A> zero_extend_lo(batch<double, SrcA> const& lo) noexcept
{
return _mm256_zextpd128_pd256(lo);
}
}

// Runtime-mask load (float/double).
Expand Down Expand Up @@ -1021,16 +1034,27 @@ namespace xsimd
template <class A, class T, bool... Values, class Mode, class = std::enable_if_t<std::is_floating_point<T>::value>>
XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Values...> mask, convert<T>, Mode, requires_arch<avx>) noexcept
{
using int_t = as_integer_t<T>;
constexpr size_t half_size = batch<T, A>::size / 2;
using half_arch = typename ::xsimd::make_sized_batch_t<T, half_size>::arch_type;
using half_batch = make_sized_batch_t<T, half_size>;
using half_arch = typename half_batch::arch_type;

// lower 128-bit half
XSIMD_IF_CONSTEXPR(mask.countl_zero() >= half_size)
// exactly the lower 128-bit half: one plain load, upper lanes zero
XSIMD_IF_CONSTEXPR(mask.prefix() == half_size)
{
constexpr auto mlo = ::xsimd::detail::lower_half<half_arch>(batch_bool_constant<int_t, A, Values...> {});
const auto lo = load_masked(reinterpret_cast<int_t const*>(mem), mlo, convert<int_t> {}, Mode {}, half_arch {});
return bitwise_cast<T>(batch<int_t, A>(_mm256_zextsi128_si256(lo)));
return detail::zero_extend_lo<A>(half_batch::load(mem, Mode {}));
}
// lower 128-bit half: stay in the value domain so the half kernel can
// lower pure-prefix shapes to plain narrow moves (movss/movlps/movsd)
else XSIMD_IF_CONSTEXPR(mask.countl_zero() >= half_size)

@serge-sans-paille serge-sans-paille Jul 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: we could assert that mask.countl_zero() > half_size, right?

@DiamonDinoia DiamonDinoia Jul 10, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will improve in a follow up PR that I am almost done drafting.

{
constexpr auto mlo = ::xsimd::detail::lower_half<half_arch>(mask);
const auto lo = load_masked(mem, mlo, convert<T> {}, Mode {}, half_arch {});
return detail::zero_extend_lo<A>(lo);
}
// exactly the upper 128-bit half: one plain load into the upper lanes
else XSIMD_IF_CONSTEXPR(mask.suffix() == half_size)
{
return detail::zero_extend<A>(half_batch::load(mem + half_size, Mode {}));
}
// upper 128-bit half
else XSIMD_IF_CONSTEXPR(mask.countr_zero() >= half_size)
Expand Down Expand Up @@ -1076,8 +1100,30 @@ namespace xsimd
using half_batch = ::xsimd::make_sized_batch_t<T, half_size>;
using half_arch = typename half_batch::arch_type;

// exactly the lower 128-bit half: one plain store
XSIMD_IF_CONSTEXPR(mask.prefix() == half_size)
{
const half_batch lo = detail::lower_half(src);
lo.store(mem, Mode {});
}
// prefix crossing the 128-bit boundary: plain lower half + prefix-masked
// upper half. Never emits vmaskmov, which does not store-forward.
else XSIMD_IF_CONSTEXPR(mask.prefix() > half_size && mask.prefix() < batch<T, A>::size)
{
const half_batch lo = detail::lower_half(src);
lo.store(mem, Mode {});
constexpr auto mhi = ::xsimd::detail::upper_half<half_arch>(mask);
const half_batch hi = detail::upper_half(src);
store_masked<half_arch>(mem + half_size, hi, mhi, Mode {}, half_arch {});
}
// exactly the upper 128-bit half: one plain store
else XSIMD_IF_CONSTEXPR(mask.suffix() == half_size)
{
const half_batch hi = detail::upper_half(src);
hi.store(mem + half_size, Mode {});
}
// lower 128-bit half
XSIMD_IF_CONSTEXPR(mask.countl_zero() >= half_size)
else XSIMD_IF_CONSTEXPR(mask.countl_zero() >= half_size)
{
constexpr auto mlo = ::xsimd::detail::lower_half<half_arch>(mask);
const half_batch lo = detail::lower_half(src);
Expand Down
23 changes: 20 additions & 3 deletions include/xsimd/arch/xsimd_avx2_128.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,19 +141,36 @@ namespace xsimd
}
}

// constant masks gain nothing on a single register; forward to runtime
// Masks that lower to plain moves go to the sse2 float/double kernels
// (int bitcast to same-width float); the rest keep the native vpmaskmov.
template <class A, class T, bool... Values, class Mode,
typename = std::enable_if_t<std::is_integral<T>::value && (sizeof(T) == 4 || sizeof(T) == 8)>>
XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Values...> mask, convert<T>, Mode, requires_arch<avx2_128>) noexcept
{
return load_masked(mem, mask.as_batch_bool(), convert<T> {}, Mode {}, avx2_128 {});
XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
{
using F = std::conditional_t<sizeof(T) == 4, float, double>;
return bitwise_cast<T>(batch<F, A>(load_masked(reinterpret_cast<F const*>(mem), batch_bool_constant<F, A, Values...> {}, convert<F> {}, Mode {}, sse2 {})));
}
else
{
return load_masked(mem, mask.as_batch_bool(), convert<T> {}, Mode {}, avx2_128 {});
}
}

template <class A, class T, bool... Values, class Mode,
typename = std::enable_if_t<std::is_integral<T>::value && (sizeof(T) == 4 || sizeof(T) == 8)>>
XSIMD_INLINE void store_masked(T* mem, batch<T, A> const& src, batch_bool_constant<T, A, Values...> mask, Mode, requires_arch<avx2_128>) noexcept
{
store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx2_128 {});
XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
{
using F = std::conditional_t<sizeof(T) == 4, float, double>;
store_masked(reinterpret_cast<F*>(mem), bitwise_cast<F>(src), batch_bool_constant<F, A, Values...> {}, Mode {}, sse2 {});
}
else
{
store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx2_128 {});
}
}

template <class A, class T, class Mode>
Expand Down
Loading