From fd7bcc6270bb6b4061fc89006fa40aae979ac2c4 Mon Sep 17 00:00:00 2001 From: Marco Barbone Date: Wed, 2 Sep 2026 10:13:40 -0400 Subject: [PATCH 1/3] fix: make sve and rvv fast_cast visible to the dispatcher batch_cast dispatches through kernel::detail::batch_cast, which calls fast_cast unqualified. sve and rvv declared their overloads in kernel::detail_sve and kernel::detail_rvv, which neither ordinary lookup nor ADL reaches, so conversion_type resolved to with_slow_conversion and every float <-> int conversion on those two architectures went through a scalar loop over a stack buffer instead of svcvt / vfcvt. Move the overloads into kernel::detail. Each one is keyed on requires_arch or requires_arch, which carry XSIMD_SVE_BITS and XSIMD_RVV_BITS, so the ODR concern that motivated the private namespaces does not apply to them. double -> int64 on sve256, gcc-10 -O3, whole function: before stack frame, 4 vector spills, an out-of-line call after ptrue / ld1d / fcvtzs / st1d / ret The uses_fast_cast_v asserts in test_batch_cast were gated on XSIMD_WITH_SSE2; they now also run on sve and rvv, where they fail to compile if the overloads move back out of kernel::detail. vxe declares no fast_cast at all and vsx has no int64 -> double, so those two stay out. Assisted-by: Claude Opus 5 --- include/xsimd/arch/xsimd_rvv.hpp | 14 +++++-- include/xsimd/arch/xsimd_sve.hpp | 71 ++++++++++++++++---------------- test/test_batch_cast.cpp | 5 ++- 3 files changed, 50 insertions(+), 40 deletions(-) diff --git a/include/xsimd/arch/xsimd_rvv.hpp b/include/xsimd/arch/xsimd_rvv.hpp index 270102cc7..cf1d2092a 100644 --- a/include/xsimd/arch/xsimd_rvv.hpp +++ b/include/xsimd/arch/xsimd_rvv.hpp @@ -1254,16 +1254,22 @@ namespace xsimd using rvv_enable_ftoi_t = std::enable_if_t<(sizeof(T) == sizeof(U) && std::is_floating_point_v && !std::is_floating_point_v), int>; template using rvv_enable_itof_t = std::enable_if_t<(sizeof(T) == sizeof(U) && !std::is_floating_point_v && std::is_floating_point_v), int>; + } - template = 0> + // The dispatcher looks fast_cast up from kernel::detail, so both overloads have + // to live there: in detail_rvv neither ADL nor unqualified lookup reaches them + // and every conversion falls back to the scalar loop. + namespace detail + { + template = 0> XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept { - return rvvfcvt_rtz(U {}, arg); + return detail_rvv::rvvfcvt_rtz(U {}, arg); } - template = 0> + template = 0> XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept { - return rvvfcvt_f(arg); + return detail_rvv::rvvfcvt_f(arg); } } diff --git a/include/xsimd/arch/xsimd_sve.hpp b/include/xsimd/arch/xsimd_sve.hpp index dc36bcf00..b5a9c9612 100644 --- a/include/xsimd/arch/xsimd_sve.hpp +++ b/include/xsimd/arch/xsimd_sve.hpp @@ -905,47 +905,48 @@ namespace xsimd ******************************/ // fast_cast - namespace detail_sve - { - inline namespace XSIMD_SVE_NAMESPACE + // The dispatcher looks these up from kernel::detail, so they have to live there: + // in detail_sve neither ADL nor unqualified lookup reaches them and every + // conversion falls back to the scalar loop. Each overload is keyed on + // requires_arch, which carries XSIMD_SVE_BITS, so no ODR issue arises. + namespace detail + { + template = 0> + XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept { - template = 0> - XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept - { - return svcvt_f32_x(detail_sve::ptrue(), arg); - } + return svcvt_f32_x(detail_sve::ptrue(), arg); + } - template = 0> - XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept - { - return svcvt_f64_x(detail_sve::ptrue(), arg); - } + template = 0> + XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept + { + return svcvt_f64_x(detail_sve::ptrue(), arg); + } - template - XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept - { - return svcvt_s32_x(detail_sve::ptrue(), arg); - } + template + XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept + { + return svcvt_s32_x(detail_sve::ptrue(), arg); + } - template - XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept - { - return svcvt_u32_x(detail_sve::ptrue(), arg); - } + template + XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept + { + return svcvt_u32_x(detail_sve::ptrue(), arg); + } - template - XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept - { - return svcvt_s64_x(detail_sve::ptrue(), arg); - } + template + XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept + { + return svcvt_s64_x(detail_sve::ptrue(), arg); + } - template - XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept - { - return svcvt_u64_x(detail_sve::ptrue(), arg); - } - } // namespace XSIMD_SVE_NAMESPACE - } // namespace detail_sve + template + XSIMD_INLINE batch fast_cast(batch const& arg, batch const&, requires_arch) noexcept + { + return svcvt_u64_x(detail_sve::ptrue(), arg); + } + } // namespace detail /********* * Miscs * diff --git a/test/test_batch_cast.cpp b/test/test_batch_cast.cpp index 954ae0e66..5ac89496b 100644 --- a/test/test_batch_cast.cpp +++ b/test/test_batch_cast.cpp @@ -396,7 +396,10 @@ TYPED_TEST(batch_cast_test, cast_sizeshift2) } #endif -#if XSIMD_WITH_SSE2 +// sve and rvv used to declare fast_cast in detail_sve / detail_rvv, where the +// dispatcher in kernel::detail cannot see it, so every conversion silently fell back +// to the scalar loop. Assert on those two arches as well as on sse2. +#if XSIMD_WITH_SSE2 || XSIMD_WITH_SVE || XSIMD_WITH_RVV TEST_CASE_TEMPLATE("[xsimd cast tests]", B, CONVERSION_TYPES) { SUBCASE("use fastcast") From 7e7e7642f9ac50c1fc734fccdd5a316765a9b8bb Mon Sep 17 00:00:00 2001 From: Marco Barbone Date: Wed, 2 Sep 2026 06:45:31 -0400 Subject: [PATCH 2/3] perf: native sse2 trunc for batch The common trunc is select(abs(x) < maxflint, to_float(to_int(x)), x), so on sse2 it goes through batch_cast. Adding 2^52 rounds |x| to an integer in any rounding mode and one conditional decrement turns that into floor, which needs no conversion at all. 3.743 -> 0.551 ns/element on a 128 KB L2-resident loop (Xeon w5-3435X, gcc 13.3 -O3, min of 300 reps). Assisted-by: Claude Opus 5 --- include/xsimd/arch/xsimd_sse2.hpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/include/xsimd/arch/xsimd_sse2.hpp b/include/xsimd/arch/xsimd_sse2.hpp index eccba3b36..58e35254d 100644 --- a/include/xsimd/arch/xsimd_sse2.hpp +++ b/include/xsimd/arch/xsimd_sse2.hpp @@ -2259,6 +2259,26 @@ namespace xsimd transpose(reinterpret_cast*>(matrix_begin), reinterpret_cast*>(matrix_end), A {}); } + // trunc + template + XSIMD_INLINE batch trunc(batch const& self, requires_arch) noexcept + { + // The common implementation goes through batch_cast, which on sse2 + // is itself built on trunc. Adding 2^52 rounds |self| to an integer whatever + // the rounding mode is; one conditional decrement turns that into floor. + __m128d signmask = _mm_set1_pd(-0.); + __m128d t2n = _mm_set1_pd(4503599627370496.); // 2^52 + __m128d s = _mm_and_pd(self, signmask); + __m128d v = _mm_andnot_pd(signmask, self); // |self| + __m128d d0 = _mm_add_pd(v, t2n); + detail::reassociation_barrier(d0, "prevent collapsing (v + 2^52) - 2^52 back to v"); + __m128d d = _mm_sub_pd(d0, t2n); + d = _mm_sub_pd(d, _mm_and_pd(_mm_cmpgt_pd(d, v), _mm_set1_pd(1.))); // floor(|self|) + d = _mm_andnot_pd(signmask, d); // roundTowardNegative makes 2^52 - 2^52 negative + __m128d small = _mm_cmplt_pd(v, t2n); + return _mm_or_pd(s, _mm_or_pd(_mm_and_pd(small, d), _mm_andnot_pd(small, v))); + } + // zip_hi template XSIMD_INLINE batch zip_hi(batch const& self, batch const& other, requires_arch) noexcept From dea9546caab4a6e87dda5b4a4fa918eb89411f1d Mon Sep 17 00:00:00 2001 From: Marco Barbone Date: Wed, 2 Sep 2026 06:45:31 -0400 Subject: [PATCH 3/3] feat: common double -> int64 fast_cast batch_cast(batch) fell back to a scalar loop through a stack buffer on every architecture without a native instruction. Split trunc(x) into hi * 2^32 + lo and read each half out of the mantissa with the 1.5 * 2^52 magic constant: both halves are small enough that the add is exact, so the result is independent of the rounding mode. Selected on sse2, sse4.1, avx, avx2, avx512f-without-DQ, vsx, vxe and wasm. avx512dq keeps vcvttpd2qq, neon64, sve, rvv and emulated keep their own. ns/element on a 128 KB L2-resident loop (Xeon w5-3435X, gcc 13.3 -O3, min of 300 reps, interleaved): sse2 2.387 -> 1.896 1.26x sse4.1 2.332 -> 0.636 3.67x avx 1.221 -> 0.501 2.44x avx2 1.221 -> 0.289 4.22x avx512f 0.697 -> 0.206 3.38x The unchanged int64 -> double control arm moves by at most 13%. test_cast_all_lanes covers float -> int32 and double -> int64 with a different value in every lane; the other cast tests are splats that only inspect lane 0. Assisted-by: Claude Opus 5 --- .../arch/common/xsimd_common_details.hpp | 20 +++++ test/test_batch_cast.cpp | 75 ++++++++++++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/include/xsimd/arch/common/xsimd_common_details.hpp b/include/xsimd/arch/common/xsimd_common_details.hpp index 47a3bf84e..13a71d5fc 100644 --- a/include/xsimd/arch/common/xsimd_common_details.hpp +++ b/include/xsimd/arch/common/xsimd_common_details.hpp @@ -251,6 +251,26 @@ namespace xsimd return bitwise_cast(self); } + // Common double -> int64_t cast. Adding 1.5 * 2^52 puts an integer + // |v| < 2^51 exactly in the mantissa, so subtracting the bit patterns + // reads it back out. trunc first, then split into halves that small. + // An architecture reaching this overload needs a non-common trunc: + // the common one goes through to_int and would recurse into here. + template + XSIMD_INLINE batch fast_cast(batch const& x, batch const&, requires_arch) noexcept + { + using batch_type = batch; + batch_type magic(0x1.8p52); + batch magic_i = bitwise_cast(magic); + batch_type t = trunc(x); + batch_type hi = trunc(t * batch_type(0x1p-32)); // |hi| <= 2^31 + batch_type lo = t - hi * batch_type(0x1p32); // exact, |lo| < 2^32 + detail::reassociation_barrier(lo, "keep lo below 2^32 before the magic add"); + batch hi_i = bitwise_cast(hi + magic) - magic_i; + batch lo_i = bitwise_cast(lo + magic) - magic_i; + return (hi_i << 32) + lo_i; + } + // Provide a common uint32_t -> float cast only if we have a // non-common int32_t -> float fast_cast template const&>(), std::declval const&>(), A {}))> diff --git a/test/test_batch_cast.cpp b/test/test_batch_cast.cpp index 5ac89496b..c78d0cb78 100644 --- a/test/test_batch_cast.cpp +++ b/test/test_batch_cast.cpp @@ -14,6 +14,8 @@ #include "test_utils.hpp" +#include + #if !XSIMD_WITH_NEON || XSIMD_WITH_NEON64 namespace detail { @@ -152,6 +154,12 @@ struct batch_cast_test }; } + void test_cast_all_lanes() const + { + test_cast_all_lanes_impl("batch cast float -> int32"); + test_cast_all_lanes_impl("batch cast double -> int64"); + } + void test_bool_cast() const { test_bool_cast_impl("batch bool cast float -> int32"); @@ -345,6 +353,56 @@ struct batch_cast_test } } + // A float -> same-width-int cast can recombine per-lane data, so every lane has to + // carry a different value; the other cast tests are splats and only look at lane 0. + template + void test_cast_all_lanes_impl(const std::string& name) const + { + using T_in = typename B_in::value_type; + using T_out = typename B_out::value_type; + constexpr int digits = std::numeric_limits::digits; // 31 or 63 + const T_in beyond = std::ldexp(T_in(1), digits); // one past the top of T_out + const T_in top = std::nextafter(beyond, T_in(0)); // the largest T_in that fits + // The ends of the range, then every power of two the cast can reach probed on + // both sides: those are where a carry between halves and the end of the + // mantissa live, and a random draw never lands on one of them. + std::vector values = { T_in(0), -T_in(0), -beyond, top, -top }; + for (int e = 0; e < digits; ++e) + { + const T_in p = std::ldexp(T_in(1), e); + for (const T_in v : { p - T_in(1.5), p - T_in(1), p - T_in(.5), std::nextafter(p, T_in(0)), + p, std::nextafter(p, beyond), p + T_in(.5), p + T_in(1) }) + { + values.push_back(v); + values.push_back(-v); + } + } + // Eight random values in every binade the cast can reach, both signs. The + // engine is default seeded, so a failure reproduces. + std::default_random_engine generator; + std::uniform_real_distribution mantissa(T_in(1), T_in(2)); + for (int e = -1; e < digits; ++e) + for (int k = 0; k < 8; ++k) + { + const T_in v = std::ldexp(mantissa(generator), e); + values.push_back(v); + values.push_back(-v); + } + constexpr size_t n = B_in::size; + T_in buffer[n]; + for (size_t i = 0; i < values.size(); i += n) + { + for (size_t l = 0; l < n; ++l) + buffer[l] = values[(i + l) % values.size()]; + B_out res = xsimd::batch_cast(B_in::load_unaligned(buffer)); + for (size_t l = 0; l < n; ++l) + { + INFO(name, ", lane ", l, " holding ", buffer[l]); + CHECK_SCALAR_EQ(static_cast(buffer[l]), res.get(l)); + } + } + } + template void test_bool_cast_impl(const std::string& name) const { @@ -380,6 +438,11 @@ TEST_CASE_TEMPLATE("[xsimd cast tests]", B, CONVERSION_TYPES) { Test.test_cast(); } + + SUBCASE("cast all lanes") + { + Test.test_cast_all_lanes(); + } } #endif #if 0 && XSIMD_X86_INSTR_SET > D_X86_AVX_VERSION @@ -396,19 +459,27 @@ TYPED_TEST(batch_cast_test, cast_sizeshift2) } #endif +// neon32 has no batch, and detail::uses_fast_cast_v lives inside the guard +// that excludes it, so the whole block needs the same guard. +#if !XSIMD_WITH_NEON || XSIMD_WITH_NEON64 // sve and rvv used to declare fast_cast in detail_sve / detail_rvv, where the // dispatcher in kernel::detail cannot see it, so every conversion silently fell back -// to the scalar loop. Assert on those two arches as well as on sse2. -#if XSIMD_WITH_SSE2 || XSIMD_WITH_SVE || XSIMD_WITH_RVV +// to the scalar loop. int32 <-> float has no common overload, so both asserts below +// fail if an architecture declares its fast_cast outside kernel::detail again. TEST_CASE_TEMPLATE("[xsimd cast tests]", B, CONVERSION_TYPES) { SUBCASE("use fastcast") { using A = xsimd::default_arch; +#if XSIMD_WITH_SSE2 || XSIMD_WITH_SVE || XSIMD_WITH_RVV static_assert(detail::uses_fast_cast_v, "expected int32 to float conversion to use fast_cast"); static_assert(detail::uses_fast_cast_v, "expected float to int32 conversion to use fast_cast"); +#endif + // the common overload answers double -> int64_t on every architecture + static_assert(detail::uses_fast_cast_v, + "expected double to int64 conversion to use fast_cast"); } } #endif