diff --git a/Cargo.toml b/Cargo.toml index 2f51bf5d5ff5f8..d25116d49589a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,3 +8,7 @@ rust-version = "1.49.0" crate-type = ["staticlib"] [dependencies] +sha1dc = { version = "0.1.5", optional = true } + +[features] +sha1dc-rs = ["sha1dc"] diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 4d40c16ede9cfe..4adffc30fa9c43 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -382,6 +382,14 @@ core.repositoryFormatVersion:: Internal variable identifying the repository format and layout version. See linkgit:gitrepository-layout[5]. +core.sha1dcBackend:: + Select the collision-detecting SHA-1 implementation: `c` keeps the + original C code and is the default, `accel` selects accelerated C, + and `rust` selects the Rust implementation. Rust is available only + in builds with `DC_SHA1_RS`, and accelerated C can be omitted at build + time. Selecting an unavailable implementation is an error. + This setting has no effect with other SHA-1 backends. + core.sharedRepository:: When 'group' (or 'true'), the repository is made shareable between several users in a group (making sure all the files and objects are diff --git a/Makefile b/Makefile index c98be9155c54ae..367b7c78d971fe 100644 --- a/Makefile +++ b/Makefile @@ -572,6 +572,13 @@ include shared.mak # by the git project to migrate to using sha1collisiondetection as a # submodule. # +# Define DC_SHA1_RS to make the sha1dc Rust crate available via +# core.sha1dcBackend=rust. This requires Rust 1.87 or newer. +# +# The original C implementation remains the default. Select the faster +# sha1dc-accel code with core.sha1dcBackend=accel. Define DC_SHA1_NO_ACCEL +# to omit that implementation. +# # === SHA-256 backend === # # ==== Security ==== @@ -1564,6 +1571,7 @@ CLAR_TEST_SUITES += u-reftable-readwrite CLAR_TEST_SUITES += u-reftable-stack CLAR_TEST_SUITES += u-reftable-table CLAR_TEST_SUITES += u-reftable-tree +CLAR_TEST_SUITES += u-sha1dc CLAR_TEST_SUITES += u-strbuf CLAR_TEST_SUITES += u-strcmp-offset CLAR_TEST_SUITES += u-string-list @@ -2159,6 +2167,23 @@ ifdef PPC_SHA1 $(error the PPC_SHA1 flag has been removed along with the PowerPC-specific SHA-1 implementation.) endif +ifdef DC_SHA1_RS +ifdef NO_RUST +$(error DC_SHA1_RS requires Rust support) +endif +ifneq ($(strip $(OPENSSL_SHA1)$(BLK_SHA1)$(APPLE_COMMON_CRYPTO_SHA1)),) +$(error DC_SHA1_RS cannot be combined with another SHA-1 backend) +endif +ifdef DC_SHA1_EXTERNAL +$(error Only set DC_SHA1_RS or DC_SHA1_EXTERNAL, not both) +endif +ifdef DC_SHA1_SUBMODULE +ifneq ($(DC_SHA1_SUBMODULE),auto) +$(error Only set DC_SHA1_RS or DC_SHA1_SUBMODULE, not both) +endif +endif +endif + ifdef OPENSSL_SHA1 EXTLIBS += $(LIB_4_CRYPTO) BASIC_CFLAGS += -DSHA1_OPENSSL @@ -2173,6 +2198,14 @@ ifdef APPLE_COMMON_CRYPTO_SHA1 else BASIC_CFLAGS += -DSHA1_DC LIB_OBJS += sha1dc_git.o +ifdef DC_SHA1_RS + BASIC_CFLAGS += -DDC_SHA1_RS + CARGO_ARGS += --features sha1dc-rs + RUST_SOURCES += src/sha1dc_rs.rs +ifeq ($(uname_S),MINGW) + EXTLIBS += -luserenv +endif +endif ifdef DC_SHA1_EXTERNAL ifdef DC_SHA1_SUBMODULE ifneq ($(DC_SHA1_SUBMODULE),auto) @@ -2189,6 +2222,14 @@ ifdef DC_SHA1_SUBMODULE else LIB_OBJS += sha1dc/sha1.o LIB_OBJS += sha1dc/ubc_check.o +endif +ifdef DC_SHA1_NO_ACCEL + BASIC_CFLAGS += -DDC_SHA1_NO_ACCEL +else + LIB_OBJS += sha1dc-accel/arm.o + LIB_OBJS += sha1dc-accel/sha1.o + LIB_OBJS += sha1dc-accel/ubc_check.o + LIB_OBJS += sha1dc-accel/x86.o endif BASIC_CFLAGS += \ -DSHA1DC_NO_STANDARD_INCLUDES \ @@ -3466,6 +3507,15 @@ check-sha1:: t/helper/test-tool$X SP_SRC = $(filter-out $(THIRD_PARTY_SOURCES),$(patsubst %.o,%.c,$(OBJECTS))) SP_OBJ = $(patsubst %.c,%.sp,$(SP_SRC)) +# Sparse cannot suppress this warning only in the intrinsic headers. +sha1dc-accel/arm.sp sha1dc-accel/sha1.sp \ +sha1dc-accel/ubc_check.sp sha1dc-accel/x86.sp \ +$(UNIT_TEST_DIR)/u-sha1dc.sp: SP_EXTRA_FLAGS += -Wno-declaration-after-statement + +# Sparse 0.6.4 cannot evaluate GCC's SIMD builtins. +sha1dc-accel/ubc_check.sp sha1dc-accel/x86.sp: \ + SP_EXTRA_FLAGS += -DSHA1DC_ACCEL_PORTABLE_ONLY + $(SP_OBJ): %.sp: %.c %.o $(GENERATED_H) $(QUIET_SP)cgcc -no-compile $(ALL_CFLAGS) $(EXTRA_CPPFLAGS) \ -Wsparse-error \ diff --git a/compat/win32/pthread.c b/compat/win32/pthread.c index 398caa96029718..5af95edd3bc145 100644 --- a/compat/win32/pthread.c +++ b/compat/win32/pthread.c @@ -60,6 +60,22 @@ pthread_t pthread_self(void) return t; } +static BOOL CALLBACK win32_pthread_once(PINIT_ONCE once UNUSED, + PVOID parameter, + PVOID *context UNUSED) +{ + (*(void (**)(void))parameter)(); + return TRUE; +} + +int pthread_once(pthread_once_t *once_control, void (*init_routine)(void)) +{ + if (!InitOnceExecuteOnce(once_control, win32_pthread_once, + &init_routine, NULL)) + return err_win_to_posix(GetLastError()); + return 0; +} + int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex) { if (SleepConditionVariableCS(cond, mutex, INFINITE) == 0) diff --git a/compat/win32/pthread.h b/compat/win32/pthread.h index d80df8d12af2dc..79a6bd9680b9bd 100644 --- a/compat/win32/pthread.h +++ b/compat/win32/pthread.h @@ -26,6 +26,11 @@ static inline int return_0(int i UNUSED) { #define pthread_mutex_lock EnterCriticalSection #define pthread_mutex_unlock LeaveCriticalSection +typedef INIT_ONCE pthread_once_t; +#define PTHREAD_ONCE_INIT INIT_ONCE_STATIC_INIT + +int pthread_once(pthread_once_t *once_control, void (*init_routine)(void)); + typedef int pthread_mutexattr_t; #define pthread_mutexattr_init(a) (*(a) = 0) #define pthread_mutexattr_destroy(a) do {} while (0) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 524b3aad9247d3..3ddd6ede5a5c6d 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -244,7 +244,7 @@ add_compile_definitions(NO_OPENSSL SHA1_DC SHA1DC_NO_STANDARD_INCLUDES SHA1DC_INIT_SAFE_HASH_DEFAULT=0 SHA1DC_CUSTOM_INCLUDE_SHA1_C="git-compat-util.h" SHA1DC_CUSTOM_INCLUDE_UBC_CHECK_C="git-compat-util.h" ) -list(APPEND compat_SOURCES sha1dc_git.c sha1dc/sha1.c sha1dc/ubc_check.c block-sha1/sha1.c sha256/block/sha256.c compat/qsort_s.c) +list(APPEND compat_SOURCES sha1dc_git.c sha1dc/sha1.c sha1dc/ubc_check.c sha1dc-accel/arm.c sha1dc-accel/sha1.c sha1dc-accel/ubc_check.c sha1dc-accel/x86.c block-sha1/sha1.c sha256/block/sha256.c compat/qsort_s.c) add_compile_definitions(PAGER_ENV="LESS=FRX LV=-c" diff --git a/hash.h b/hash.h index cf94ad57002788..dd2e66e1c9ce87 100644 --- a/hash.h +++ b/hash.h @@ -12,8 +12,13 @@ # include "sha1/openssl.h" # endif #elif defined(SHA1_DC) +#ifdef DC_SHA1_RS +#define SHA1_BACKEND "SHA1_DC-rs" +#include "sha1dc_rs.h" +#else #define SHA1_BACKEND "SHA1_DC" #include "sha1dc_git.h" +#endif #else /* SHA1_BLK */ #define SHA1_BACKEND "SHA1_BLK (No collision detection)" #include "block-sha1/sha1.h" diff --git a/meson.build b/meson.build index 79b6af6eaf9641..120cabed62c6f4 100644 --- a/meson.build +++ b/meson.build @@ -1634,6 +1634,10 @@ if sha1_backend == 'sha1dc' 'sha1dc_git.c', 'sha1dc/sha1.c', 'sha1dc/ubc_check.c', + 'sha1dc-accel/arm.c', + 'sha1dc-accel/sha1.c', + 'sha1dc-accel/ubc_check.c', + 'sha1dc-accel/x86.c', ] endif if sha1_backend == 'CommonCrypto' or sha1_unsafe_backend == 'CommonCrypto' diff --git a/sha1dc-accel/arm.c b/sha1dc-accel/arm.c new file mode 100644 index 00000000000000..fa5b1e6e398ede --- /dev/null +++ b/sha1dc-accel/arm.c @@ -0,0 +1,276 @@ +/* + * The SHA-1 compression and recompression on the ARMv8 SHA-1 instructions, + * for sha1.c. `vsha1{c,p,m}q_u32` do four steps at a time on `abcd`, with + * the fifth working word carried apart and derived by `vsha1h_u32`. + */ + +#include "../git-compat-util.h" +#include "internal.h" + +#ifdef SHA1DC_HAVE_ARMV8 + +#if !defined(__ARM_FEATURE_SHA2) && !defined(__ARM_FEATURE_CRYPTO) && \ + defined(__linux__) +#include +#ifndef HWCAP_SHA1 +#define HWCAP_SHA1 (1 << 5) +#endif +#endif + +int sha1dc_armv8_available(void) +{ +#if defined(__ARM_FEATURE_SHA2) || defined(__ARM_FEATURE_CRYPTO) || \ + defined(__APPLE__) + return 1; +#elif defined(__linux__) + return !!(getauxval(AT_HWCAP) & HWCAP_SHA1); +#elif defined(GIT_WINDOWS_NATIVE) + return IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); +#else + return 0; +#endif +} + +#define K0 0x5A827999 +#define K1 0x6ED9EBA1 +#define K2 0x8F1BBCDC +#define K3 0xCA62C1D6 + +/* Big-endian message words t..t+3. */ +#define MSG(t) vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(block + 4 * (t)))) + +/* + * Four steps of a compression that keeps the schedule four groups ahead: + * `f` the round instruction, `e` the fifth word in, `next` where the one + * for the next group goes, `wk` the schedule words plus the constant. + */ +#define QUAD(f, e, next, wk) \ + do { \ + next = vsha1h_u32(vgetq_lane_u32(abcd, 0)); \ + abcd = f(abcd, e, wk); \ + } while (0) + +SHA1DC_TARGET_ARMV8 +void sha1dc_compress_armv8(uint32_t ihv[5], const unsigned char *block, + uint32_t w[80], uint32_t at_60[5], uint32_t at_64[5]) +{ + uint32x4_t abcd = vld1q_u32(ihv); + const uint32x4_t abcd_in = abcd; + uint32_t e0 = ihv[4], e1; + const uint32_t e_in = e0; + const uint32x4_t k0 = vdupq_n_u32(K0), k1 = vdupq_n_u32(K1); + const uint32x4_t k2 = vdupq_n_u32(K2), k3 = vdupq_n_u32(K3); + uint32x4_t msg0 = MSG(0), msg1 = MSG(4), msg2 = MSG(8), msg3 = MSG(12); + uint32x4_t tmp0, tmp1; + + vst1q_u32(w + 0, msg0); + vst1q_u32(w + 4, msg1); + vst1q_u32(w + 8, msg2); + vst1q_u32(w + 12, msg3); + + tmp0 = vaddq_u32(msg0, k0); + tmp1 = vaddq_u32(msg1, k0); + + /* Steps 0-3 */ + QUAD(vsha1cq_u32, e0, e1, tmp0); + tmp0 = vaddq_u32(msg2, k0); + msg0 = vsha1su0q_u32(msg0, msg1, msg2); + + /* Steps 4-7 */ + QUAD(vsha1cq_u32, e1, e0, tmp1); + tmp1 = vaddq_u32(msg3, k0); + msg0 = vsha1su1q_u32(msg0, msg3); + vst1q_u32(w + 16, msg0); + msg1 = vsha1su0q_u32(msg1, msg2, msg3); + + /* Steps 8-11 */ + QUAD(vsha1cq_u32, e0, e1, tmp0); + tmp0 = vaddq_u32(msg0, k0); + msg1 = vsha1su1q_u32(msg1, msg0); + vst1q_u32(w + 20, msg1); + msg2 = vsha1su0q_u32(msg2, msg3, msg0); + + /* Steps 12-15 */ + QUAD(vsha1cq_u32, e1, e0, tmp1); + tmp1 = vaddq_u32(msg1, k1); + msg2 = vsha1su1q_u32(msg2, msg1); + vst1q_u32(w + 24, msg2); + msg3 = vsha1su0q_u32(msg3, msg0, msg1); + + /* Steps 16-19 */ + QUAD(vsha1cq_u32, e0, e1, tmp0); + tmp0 = vaddq_u32(msg2, k1); + msg3 = vsha1su1q_u32(msg3, msg2); + vst1q_u32(w + 28, msg3); + msg0 = vsha1su0q_u32(msg0, msg1, msg2); + + /* Steps 20-23 */ + QUAD(vsha1pq_u32, e1, e0, tmp1); + tmp1 = vaddq_u32(msg3, k1); + msg0 = vsha1su1q_u32(msg0, msg3); + vst1q_u32(w + 32, msg0); + msg1 = vsha1su0q_u32(msg1, msg2, msg3); + + /* Steps 24-27 */ + QUAD(vsha1pq_u32, e0, e1, tmp0); + tmp0 = vaddq_u32(msg0, k1); + msg1 = vsha1su1q_u32(msg1, msg0); + vst1q_u32(w + 36, msg1); + msg2 = vsha1su0q_u32(msg2, msg3, msg0); + + /* Steps 28-31 */ + QUAD(vsha1pq_u32, e1, e0, tmp1); + tmp1 = vaddq_u32(msg1, k1); + msg2 = vsha1su1q_u32(msg2, msg1); + vst1q_u32(w + 40, msg2); + msg3 = vsha1su0q_u32(msg3, msg0, msg1); + + /* Steps 32-35 */ + QUAD(vsha1pq_u32, e0, e1, tmp0); + tmp0 = vaddq_u32(msg2, k2); + msg3 = vsha1su1q_u32(msg3, msg2); + vst1q_u32(w + 44, msg3); + msg0 = vsha1su0q_u32(msg0, msg1, msg2); + + /* Steps 36-39 */ + QUAD(vsha1pq_u32, e1, e0, tmp1); + tmp1 = vaddq_u32(msg3, k2); + msg0 = vsha1su1q_u32(msg0, msg3); + vst1q_u32(w + 48, msg0); + msg1 = vsha1su0q_u32(msg1, msg2, msg3); + + /* Steps 40-43 */ + QUAD(vsha1mq_u32, e0, e1, tmp0); + tmp0 = vaddq_u32(msg0, k2); + msg1 = vsha1su1q_u32(msg1, msg0); + vst1q_u32(w + 52, msg1); + msg2 = vsha1su0q_u32(msg2, msg3, msg0); + + /* Steps 44-47 */ + QUAD(vsha1mq_u32, e1, e0, tmp1); + tmp1 = vaddq_u32(msg1, k2); + msg2 = vsha1su1q_u32(msg2, msg1); + vst1q_u32(w + 56, msg2); + msg3 = vsha1su0q_u32(msg3, msg0, msg1); + + /* Steps 48-51 */ + QUAD(vsha1mq_u32, e0, e1, tmp0); + tmp0 = vaddq_u32(msg2, k2); + msg3 = vsha1su1q_u32(msg3, msg2); + vst1q_u32(w + 60, msg3); + msg0 = vsha1su0q_u32(msg0, msg1, msg2); + + /* Steps 52-55 */ + QUAD(vsha1mq_u32, e1, e0, tmp1); + tmp1 = vaddq_u32(msg3, k3); + msg0 = vsha1su1q_u32(msg0, msg3); + vst1q_u32(w + 64, msg0); + msg1 = vsha1su0q_u32(msg1, msg2, msg3); + + /* Steps 56-59 */ + QUAD(vsha1mq_u32, e0, e1, tmp0); + tmp0 = vaddq_u32(msg0, k3); + msg1 = vsha1su1q_u32(msg1, msg0); + vst1q_u32(w + 68, msg1); + msg2 = vsha1su0q_u32(msg2, msg3, msg0); + + /* The state at step 60, for recompression to start from. */ + vst1q_u32(at_60, abcd); + at_60[4] = e1; + + /* Steps 60-63 */ + QUAD(vsha1pq_u32, e1, e0, tmp1); + tmp1 = vaddq_u32(msg1, k3); + msg2 = vsha1su1q_u32(msg2, msg1); + vst1q_u32(w + 72, msg2); + msg3 = vsha1su0q_u32(msg3, msg0, msg1); + + /* And at step 64. */ + vst1q_u32(at_64, abcd); + at_64[4] = e0; + + /* Steps 64-67 */ + QUAD(vsha1pq_u32, e0, e1, tmp0); + tmp0 = vaddq_u32(msg2, k3); + msg3 = vsha1su1q_u32(msg3, msg2); + vst1q_u32(w + 76, msg3); + + /* Steps 68-71 */ + QUAD(vsha1pq_u32, e1, e0, tmp1); + tmp1 = vaddq_u32(msg3, k3); + + /* Steps 72-75 */ + QUAD(vsha1pq_u32, e0, e1, tmp0); + + /* Steps 76-79 */ + QUAD(vsha1pq_u32, e1, e0, tmp1); + + vst1q_u32(ihv, vaddq_u32(abcd_in, abcd)); + ihv[4] = e0 + e_in; +} + +/* Four steps of the partner block, on schedule group `g`. */ +#define GROUP(f, k, g) \ + do { \ + uint32x4_t wk_ = vaddq_u32(veorq_u32(vld1q_u32(m1 + 4 * (g)), \ + vld1q_u32(dm + 4 * (g))), \ + vdupq_n_u32(k)); \ + uint32_t next_ = vsha1h_u32(vgetq_lane_u32(abcd, 0)); \ + abcd = f(abcd, e, wk_); \ + e = next_; \ + } while (0) + +/* + * Whether the partner block, whose state at `from` is `state`, ends on + * `ihv_out`: out from its state at step 60 or 64 to step 80, then in from + * the chaining value that implies, which must arrive back at that state. + */ +SHA1DC_TARGET_ARMV8 +int sha1dc_recompress_armv8(enum sha1dc_from from, const uint32_t m1[80], + const uint32_t dm[80], const uint32_t state[5], + const uint32_t ihv_out[5]) +{ + uint32_t at[5], reached[5], e; + uint32x4_t abcd; + + sha1dc_partner_boundary(from, m1, dm, state, at); + abcd = vld1q_u32(at); + e = at[4]; + + /* Out to step 80, from 60 or from 64. */ + if (from == SHA1DC_FROM_58) + GROUP(vsha1pq_u32, K3, 15); + GROUP(vsha1pq_u32, K3, 16); + GROUP(vsha1pq_u32, K3, 17); + GROUP(vsha1pq_u32, K3, 18); + GROUP(vsha1pq_u32, K3, 19); + + /* The only input that gives this output is the output less the state. */ + abcd = vsubq_u32(vld1q_u32(ihv_out), abcd); + e = ihv_out[4] - e; + + /* In from there, as far as the state the way out started from. */ + GROUP(vsha1cq_u32, K0, 0); + GROUP(vsha1cq_u32, K0, 1); + GROUP(vsha1cq_u32, K0, 2); + GROUP(vsha1cq_u32, K0, 3); + GROUP(vsha1cq_u32, K0, 4); + GROUP(vsha1pq_u32, K1, 5); + GROUP(vsha1pq_u32, K1, 6); + GROUP(vsha1pq_u32, K1, 7); + GROUP(vsha1pq_u32, K1, 8); + GROUP(vsha1pq_u32, K1, 9); + GROUP(vsha1mq_u32, K2, 10); + GROUP(vsha1mq_u32, K2, 11); + GROUP(vsha1mq_u32, K2, 12); + GROUP(vsha1mq_u32, K2, 13); + GROUP(vsha1mq_u32, K2, 14); + if (from == SHA1DC_FROM_65) + GROUP(vsha1pq_u32, K3, 15); + + vst1q_u32(reached, abcd); + reached[4] = e; + return !memcmp(reached, at, sizeof(at)); +} + +#endif /* SHA1DC_HAVE_ARMV8 */ diff --git a/sha1dc-accel/internal.h b/sha1dc-accel/internal.h new file mode 100644 index 00000000000000..bcb60811a8fd33 --- /dev/null +++ b/sha1dc-accel/internal.h @@ -0,0 +1,129 @@ +#ifndef SHA1DC_ACCEL_INTERNAL_H +#define SHA1DC_ACCEL_INTERNAL_H + +/* + * Shared between the files of sha1dc-accel/. See sha1.c for an overview. + * + * The schedule `w` has 80 expanded message words. Normally w[t] is at + * index t, as in sha1dc/. SHA-NI spills w[t] at index 79 - t; a candidate + * is put back in step order before recompression. + * + * A "state" is the five working words [a, b, c, d, e] before a step. + */ + +/* + * Which forms this build can have. The vector and hardware forms need + * GCC-compatible target attributes and intrinsics; anything else gets the + * portable form, which every build has. + */ +#if defined(__GNUC__) && !defined(SHA1DC_ACCEL_PORTABLE_ONLY) +# if defined(__x86_64__) && (defined(__clang__) || __GNUC__ >= 5) +# define SHA1DC_HAVE_SSE2 1 +# define SHA1DC_HAVE_AVX2 1 +# define SHA1DC_HAVE_SHANI 1 +# include +# define SHA1DC_TARGET_SSE2 +# define SHA1DC_TARGET_AVX2 __attribute__((target("avx2"))) +# define SHA1DC_TARGET_SHANI __attribute__((target("sha,sse4.1,ssse3"))) +# elif defined(__aarch64__) && defined(__ARM_NEON) +# define SHA1DC_HAVE_NEON 1 +# include +/* + * The SHA-1 instructions, where the compiler targets them already or can + * enable them for one function: older compilers declare their intrinsics + * only when the whole file targets them. + */ +# if defined(__ARM_FEATURE_SHA2) || defined(__ARM_FEATURE_CRYPTO) +# define SHA1DC_HAVE_ARMV8 1 +# define SHA1DC_TARGET_ARMV8 +# elif defined(__clang__) && __clang_major__ >= 17 +# define SHA1DC_HAVE_ARMV8 1 +# define SHA1DC_TARGET_ARMV8 __attribute__((target("sha2"))) +# elif !defined(__clang__) && __GNUC__ >= 9 +# define SHA1DC_HAVE_ARMV8 1 +# define SHA1DC_TARGET_ARMV8 __attribute__((target("+crypto"))) +# endif +# endif +#endif + +#if defined(__GNUC__) +# define SHA1DC_NOINLINE __attribute__((noinline)) +# define sha1dc_ctz(x) ((unsigned)__builtin_ctz(x)) +#else +# define SHA1DC_NOINLINE +static inline unsigned sha1dc_ctz(uint32_t x) +{ + unsigned n = 0; + while (!(x & 1)) { + x >>= 1; + n++; + } + return n; +} +#endif + +/* The step a DV's recompression starts from. */ +enum sha1dc_from { + SHA1DC_FROM_58 = 58, + SHA1DC_FROM_65 = 65 +}; + +/* + * The UBC check, one form per instruction set. Each returns the same mask + * as ubc_check() in sha1dc/: a set bit names a DV that is still possible. + * See ubc_check.c. + */ +uint32_t sha1dc_ubc_check_scalar(const uint32_t w[80]); +#ifdef SHA1DC_HAVE_SSE2 +uint32_t sha1dc_ubc_check_sse2(const uint32_t w[80]); +uint32_t sha1dc_ubc_check_sse2_mirrored(const uint32_t w[80]); +#endif +#ifdef SHA1DC_HAVE_AVX2 +uint32_t sha1dc_ubc_check_avx2(const uint32_t w[80]); +uint32_t sha1dc_ubc_check_avx2_mirrored(const uint32_t w[80]); +#endif +#ifdef SHA1DC_HAVE_NEON +uint32_t sha1dc_ubc_check_neon(const uint32_t w[80]); +#endif + +/* Whether the CPU (and OS) can run the AVX2 form. In x86.c. */ +#ifdef SHA1DC_HAVE_AVX2 +int sha1dc_avx2_available(void); +#endif + +/* + * The state the partner block of a DV reaches at the group boundary next to + * where its recompression starts: step 60 from step 58, step 64 from step + * 65. `state` is this block's state at `from`. In sha1.c. + */ +void sha1dc_partner_boundary(enum sha1dc_from from, const uint32_t m1[80], + const uint32_t dm[80], const uint32_t state[5], + uint32_t out[5]); + +/* + * The hardware compressions. Each compresses one 64-byte block into `ihv`, + * writes the expanded schedule to `w`, and this block's own states at + * steps 60 and 64 to `at_60` and `at_64`. + * + * Their recompressions answer whether a DV candidate is really an attack, + * running the partner block forwards from `state` (this block's state at + * `from`) with the SHA-1 instructions. + */ +#ifdef SHA1DC_HAVE_SHANI +int sha1dc_shani_available(void); +void sha1dc_compress_shani(uint32_t ihv[5], const unsigned char *block, + uint32_t w[80], uint32_t at_60[5], uint32_t at_64[5]); +int sha1dc_recompress_shani(enum sha1dc_from from, const uint32_t m1[80], + const uint32_t dm[80], const uint32_t state[5], + const uint32_t ihv_out[5]); +#endif +#ifdef SHA1DC_HAVE_ARMV8 +int sha1dc_armv8_available(void); +void sha1dc_compress_armv8(uint32_t ihv[5], const unsigned char *block, + uint32_t w[80], uint32_t at_60[5], uint32_t at_64[5]); +int sha1dc_recompress_armv8(enum sha1dc_from from, const uint32_t m1[80], + const uint32_t dm[80], const uint32_t state[5], + const uint32_t ihv_out[5]); +#endif + +#endif /* SHA1DC_ACCEL_INTERNAL_H */ diff --git a/sha1dc-accel/sha1.c b/sha1dc-accel/sha1.c new file mode 100644 index 00000000000000..296f10666c8b30 --- /dev/null +++ b/sha1dc-accel/sha1.c @@ -0,0 +1,510 @@ +/* + * SHA-1 with collision detection, faster. + * + * This computes exactly what sha1dc/ computes: the SHA-1 digest of the + * input, and whether any block of it looks like one half of a collision + * made by one of the 32 known disturbance vectors (DVs) of Stevens and + * Shumow. It is a port to C of the approach of the "sha1dc" Rust crate by + * Sam Reis (https://github.com/srijs/sha1dc), which gitoxide uses: + * + * - The compression runs on the CPU's SHA-1 instructions where it has them + * (SHA-NI on x86-64, the ARMv8 cryptography extension on arm64), and + * spills the expanded message schedule as it goes, which is all that + * detection needs from an ordinary block. The hardware keeps no + * intermediate states around, so the two states that recompression + * starts from (at steps 58 and 65) are recovered from the ones at steps + * 60 and 64, and only for the rare block that needs them. + * + * - The unavoidable-bitconditions (UBC) filter, which rules out about 95% + * of blocks and is most of what detection costs, has one form per + * instruction set (SSE2, AVX2, NEON and portable C). For each, the + * crate's solver picked conditions equivalent to the published ones + * that fill vector lanes well, for a prefix run on every block, and left + * the rest to a scalar tail that few blocks reach. Those choices are + * kept as tables, run by a short loop per form. See ubc_check.c. + * + * - The recompression of a flagged block also runs on the SHA-1 + * instructions, forwards from the partner block's state at step 60 or + * 64, since that is the only direction they go. + * + * Without SHA-1 instructions the compression is portable C, and the + * best-suited form of the filter still applies. + */ + +#include "../git-compat-util.h" +#include "../sha1dc_git.h" +#if defined(DC_SHA1_SUBMODULE) +#include "../sha1collisiondetection/lib/ubc_check.h" +#else +#include "../sha1dc/ubc_check.h" +#endif +#include "sha1.h" +#include "internal.h" + +#define ROL(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) + +#define F_CH(b, c, d) ((d) ^ ((b) & ((c) ^ (d)))) +#define F_PARITY(b, c, d) ((b) ^ (c) ^ (d)) +#define F_MAJ(b, c, d) (((b) & (c)) | ((d) & ((b) | (c)))) + +#define K0 0x5A827999 +#define K1 0x6ED9EBA1 +#define K2 0x8F1BBCDC +#define K3 0xCA62C1D6 + +/* + * One step, on names that rotate: after it, (e, a, b, c, d) are the new + * (a, b, c, d, e). + */ +#define STEP(f, k, a, b, c, d, e, x) \ + do { \ + e += ROL(a, 5) + f(b, c, d) + (k) + (x); \ + b = ROL(b, 30); \ + } while (0) + +#define LOAD(t) (w[t] = get_be32(block + 4 * (t))) +#define EXPAND(t) (w[t] = ROL(w[(t) - 3] ^ w[(t) - 8] ^ w[(t) - 14] ^ w[(t) - 16], 1)) + +#define FIVE_LOAD(f, k, a, b, c, d, e, t) \ + do { \ + STEP(f, k, a, b, c, d, e, LOAD(t)); \ + STEP(f, k, e, a, b, c, d, LOAD((t) + 1)); \ + STEP(f, k, d, e, a, b, c, LOAD((t) + 2)); \ + STEP(f, k, c, d, e, a, b, LOAD((t) + 3)); \ + STEP(f, k, b, c, d, e, a, LOAD((t) + 4)); \ + } while (0) + +#define FIVE_EXPAND(f, k, a, b, c, d, e, t) \ + do { \ + STEP(f, k, a, b, c, d, e, EXPAND(t)); \ + STEP(f, k, e, a, b, c, d, EXPAND((t) + 1)); \ + STEP(f, k, d, e, a, b, c, EXPAND((t) + 2)); \ + STEP(f, k, c, d, e, a, b, EXPAND((t) + 3)); \ + STEP(f, k, b, c, d, e, a, EXPAND((t) + 4)); \ + } while (0) + +/* + * The portable compression. Like the hardware ones it spills the schedule, + * but it writes the states at steps 58 and 65 directly, on the way past. + */ +static void compress_portable(uint32_t ihv[5], const unsigned char *block, + uint32_t w[80], uint32_t state_58[5], + uint32_t state_65[5]) +{ + uint32_t a = ihv[0], b = ihv[1], c = ihv[2], d = ihv[3], e = ihv[4]; + + FIVE_LOAD(F_CH, K0, a, b, c, d, e, 0); + FIVE_LOAD(F_CH, K0, a, b, c, d, e, 5); + FIVE_LOAD(F_CH, K0, a, b, c, d, e, 10); + STEP(F_CH, K0, a, b, c, d, e, LOAD(15)); + STEP(F_CH, K0, e, a, b, c, d, EXPAND(16)); + STEP(F_CH, K0, d, e, a, b, c, EXPAND(17)); + STEP(F_CH, K0, c, d, e, a, b, EXPAND(18)); + STEP(F_CH, K0, b, c, d, e, a, EXPAND(19)); + + FIVE_EXPAND(F_PARITY, K1, a, b, c, d, e, 20); + FIVE_EXPAND(F_PARITY, K1, a, b, c, d, e, 25); + FIVE_EXPAND(F_PARITY, K1, a, b, c, d, e, 30); + FIVE_EXPAND(F_PARITY, K1, a, b, c, d, e, 35); + + FIVE_EXPAND(F_MAJ, K2, a, b, c, d, e, 40); + FIVE_EXPAND(F_MAJ, K2, a, b, c, d, e, 45); + FIVE_EXPAND(F_MAJ, K2, a, b, c, d, e, 50); + STEP(F_MAJ, K2, a, b, c, d, e, EXPAND(55)); + STEP(F_MAJ, K2, e, a, b, c, d, EXPAND(56)); + STEP(F_MAJ, K2, d, e, a, b, c, EXPAND(57)); + state_58[0] = c; + state_58[1] = d; + state_58[2] = e; + state_58[3] = a; + state_58[4] = b; + STEP(F_MAJ, K2, c, d, e, a, b, EXPAND(58)); + STEP(F_MAJ, K2, b, c, d, e, a, EXPAND(59)); + + FIVE_EXPAND(F_PARITY, K3, a, b, c, d, e, 60); + state_65[0] = a; + state_65[1] = b; + state_65[2] = c; + state_65[3] = d; + state_65[4] = e; + FIVE_EXPAND(F_PARITY, K3, a, b, c, d, e, 65); + FIVE_EXPAND(F_PARITY, K3, a, b, c, d, e, 70); + FIVE_EXPAND(F_PARITY, K3, a, b, c, d, e, 75); + + ihv[0] += a; + ihv[1] += b; + ihv[2] += c; + ihv[3] += d; + ihv[4] += e; +} + +/* + * The rest runs only for blocks the filter flags, about one in twenty, so + * it favours being short over being fast. + */ + +/* + * Moves `s` from the state before step `from` to the one before step `to`, + * either way, on the schedule `m1 ^ dm` (`dm` may be NULL). One loop per + * round function keeps the steps free of branches. + */ +#define WORD(t) (dm ? m1[t] ^ dm[t] : m1[t]) +#define FORWARD(f, k, end) \ + for (; t < to && t < (end); t++) { \ + x = ROL(a, 5) + f(b, c, d) + (k) + e + WORD(t); \ + e = d; \ + d = c; \ + c = ROL(b, 30); \ + b = a; \ + a = x; \ + } +#define BACKWARD(f, k, start) \ + for (; t > to && t > (start); t--) { \ + x = a; \ + a = b; \ + b = ROL(c, 2); \ + c = d; \ + d = e; \ + e = x - (ROL(a, 5) + f(b, c, d) + (k) + WORD(t - 1)); \ + } + +static void walk(uint32_t s[5], const uint32_t *m1, const uint32_t *dm, + unsigned from, unsigned to) +{ + uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], x; + unsigned t = from; + + FORWARD(F_CH, K0, 20); + FORWARD(F_PARITY, K1, 40); + FORWARD(F_MAJ, K2, 60); + FORWARD(F_PARITY, K3, 80); + + BACKWARD(F_PARITY, K3, 60); + BACKWARD(F_MAJ, K2, 40); + BACKWARD(F_PARITY, K1, 20); + BACKWARD(F_CH, K0, 0); + + s[0] = a; + s[1] = b; + s[2] = c; + s[3] = d; + s[4] = e; +} + +void sha1dc_partner_boundary(enum sha1dc_from from, const uint32_t m1[80], + const uint32_t dm[80], const uint32_t state[5], + uint32_t out[5]) +{ + memcpy(out, state, 5 * sizeof(*out)); + walk(out, m1, dm, from, from == SHA1DC_FROM_58 ? 60 : 64); +} + +/* + * The recompression, the way sha1dc/ does it: from this block's state at + * `from`, the partner block's chaining value on the way in (`ihv_in`) and + * on the way out (`ihv_out`). + */ +static void recompress_portable(enum sha1dc_from from, const uint32_t m1[80], + const uint32_t dm[80], const uint32_t state[5], + uint32_t ihv_in[5], uint32_t ihv_out[5]) +{ + uint32_t fwd[5]; + int i; + + memcpy(ihv_in, state, 5 * sizeof(*ihv_in)); + walk(ihv_in, m1, dm, from, 0); + memcpy(fwd, state, sizeof(fwd)); + walk(fwd, m1, dm, from, 80); + for (i = 0; i < 5; i++) + ihv_out[i] = ihv_in[i] + fwd[i]; +} + +/* For a mitigated ("safe") hash: one more compression of the schedule. */ +static void compress_schedule(uint32_t ihv[5], const uint32_t w[80]) +{ + uint32_t s[5]; + int i; + + memcpy(s, ihv, sizeof(s)); + walk(s, w, NULL, 0, 80); + for (i = 0; i < 5; i++) + ihv[i] += s[i]; +} + +struct backend { + const char *name; + /* + * Compresses a block and spills its schedule. If `at_60_64` is set, + * it leaves the states at steps 60 and 64 where the others leave the + * ones at 58 and 65. + */ + void (*compress)(uint32_t ihv[5], const unsigned char *block, + uint32_t w[80], uint32_t s1[5], uint32_t s2[5]); + int at_60_64; + uint32_t (*ubc_check)(const uint32_t w[80]); + /* Whether a candidate is an attack; NULL for recompress_portable(). */ + int (*recompress)(enum sha1dc_from from, const uint32_t m1[80], + const uint32_t dm[80], const uint32_t state[5], + const uint32_t ihv_out[5]); + int (*available)(void); +}; + +#ifdef SHA1DC_HAVE_SHANI +static int shani_avx2_available(void) +{ + return sha1dc_shani_available() && sha1dc_avx2_available(); +} +#endif + +/* In order of preference. */ +static const struct backend backends[] = { +#ifdef SHA1DC_HAVE_SHANI + { "shani+avx2", sha1dc_compress_shani, 1, + sha1dc_ubc_check_avx2_mirrored, + sha1dc_recompress_shani, shani_avx2_available }, + { "shani+sse2", sha1dc_compress_shani, 1, + sha1dc_ubc_check_sse2_mirrored, + sha1dc_recompress_shani, sha1dc_shani_available }, +#endif +#ifdef SHA1DC_HAVE_ARMV8 + { "armv8+neon", sha1dc_compress_armv8, 1, sha1dc_ubc_check_neon, + sha1dc_recompress_armv8, sha1dc_armv8_available }, +#endif +#ifdef SHA1DC_HAVE_AVX2 + { "portable+avx2", compress_portable, 0, sha1dc_ubc_check_avx2, NULL, + sha1dc_avx2_available }, +#endif +#ifdef SHA1DC_HAVE_SSE2 + { "portable+sse2", compress_portable, 0, sha1dc_ubc_check_sse2, NULL, + NULL }, +#endif +#ifdef SHA1DC_HAVE_NEON + { "portable+neon", compress_portable, 0, sha1dc_ubc_check_neon, NULL, + NULL }, +#endif + { "portable", compress_portable, 0, sha1dc_ubc_check_scalar, NULL, NULL }, +}; + +static int usable(const struct backend *be) +{ + return !be->available || be->available(); +} + +#if defined(__GNUC__) +/* + * The backend in use, chosen on first use. Threads that race to choose it + * all store the same value, with relaxed atomics so that they may. + */ +static const struct backend *selected; + +static const struct backend *backend(void) +{ + const struct backend *be = __atomic_load_n(&selected, __ATOMIC_RELAXED); + size_t i; + + if (be) + return be; + /* The last one, "portable", is always usable. */ + for (i = 0; !usable(&backends[i]); i++) + ; + be = &backends[i]; + __atomic_store_n(&selected, be, __ATOMIC_RELAXED); + return be; +} + +static void select_backend(const struct backend *be) +{ + __atomic_store_n(&selected, be, __ATOMIC_RELAXED); +} +#else +/* + * Other compilers only get the portable backend (see internal.h), so + * there is nothing to choose, and no shared state to guard. + */ +static const struct backend *backend(void) +{ + return &backends[0]; +} + +static void select_backend(const struct backend *be UNUSED) +{ +} +#endif + +const char *sha1dc_accel_backend(void) +{ + return backend()->name; +} + +int sha1dc_accel_select(const char *name) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(backends); i++) { + if (!strcmp(backends[i].name, name) && usable(&backends[i])) { + select_backend(&backends[i]); + return 0; + } + } + return -1; +} + +const char *const *sha1dc_accel_backends(void) +{ + static const char *names[ARRAY_SIZE(backends) + 1]; + size_t i, n = 0; + + for (i = 0; i < ARRAY_SIZE(backends); i++) + if (usable(&backends[i])) + names[n++] = backends[i].name; + names[n] = NULL; + return names; +} + +/* + * Whether any DV in `candidates` makes this block half of a collision. + * `s1` and `s2` are what the compression left, `ihv_in` and `ihv_out` the + * chaining values before and after it. + */ +static SHA1DC_NOINLINE int attacked(const struct backend *be, SHA1_CTX *ctx, + uint32_t candidates, const uint32_t w[80], + uint32_t s1[5], uint32_t s2[5], + const uint32_t ihv_in[5], + const uint32_t ihv_out[5]) +{ + const uint32_t *state_58 = s1, *state_65 = s2; + int i; + + if (be->at_60_64) { + walk(s1, w, NULL, 60, 58); + walk(s2, w, NULL, 64, 65); + } + + for (i = 0; sha1_dvs[i].dvType != 0; i++) { + const dv_info_t *dv = &sha1_dvs[i]; + enum sha1dc_from from; + const uint32_t *state; + + if (!(candidates & ((uint32_t)1 << dv->maskb))) + continue; + switch (dv->testt) { + case 58: + from = SHA1DC_FROM_58; + state = state_58; + break; + case 65: + from = SHA1DC_FROM_65; + state = state_65; + break; + default: + BUG("sha1dc DV %d(%d,%d) recompresses from step %d, " + "which sha1dc-accel does not save", + dv->dvType, dv->dvK, dv->dvB, dv->testt); + } + + /* + * Reduced-round collisions are recognized by the partner + * block's chaining value on the way in, which only the + * portable recompression computes. + */ + if (be->recompress && !ctx->reduced_round_coll) { + if (be->recompress(from, w, dv->dm, state, ihv_out)) + return 1; + } else { + uint32_t ihv2_in[5], ihv2_out[5]; + + recompress_portable(from, w, dv->dm, state, ihv2_in, ihv2_out); + if (!memcmp(ihv2_out, ihv_out, sizeof(ihv2_out)) || + (ctx->reduced_round_coll && + !memcmp(ihv2_in, ihv_in, sizeof(ihv2_in)))) + return 1; + } + } + return 0; +} + +static inline void process(const struct backend *be, SHA1_CTX *ctx, + const unsigned char *block) +{ + uint32_t w[80], s1[5], s2[5], ihv_in[5], candidates; + + memcpy(ihv_in, ctx->ihv, sizeof(ihv_in)); + be->compress(ctx->ihv, block, w, s1, s2); + if (!ctx->detect_coll) + return; + + candidates = ctx->ubc_check ? be->ubc_check(w) : 0xFFFFFFFF; + if (!candidates) + return; +#ifdef SHA1DC_HAVE_SHANI + if (be->compress == sha1dc_compress_shani) { + int i; + + for (i = 0; i < 40; i++) { + SWAP(w[i], w[79 - i]); + } + } +#endif + if (attacked(be, ctx, candidates, w, s1, s2, ihv_in, ctx->ihv)) { + ctx->found_collision = 1; + /* + * Two more compressions of this block give a digest that the + * other block of the pair does not share. + */ + if (ctx->safe_hash) { + compress_schedule(ctx->ihv, w); + compress_schedule(ctx->ihv, w); + } + } +} + +void sha1dc_accel_update(SHA1_CTX *ctx, const void *data, size_t len) +{ + const struct backend *be = backend(); + const unsigned char *buf = data; + unsigned left, fill; + + if (!len) + return; + + left = ctx->total & 63; + fill = 64 - left; + + if (left && len >= fill) { + ctx->total += fill; + memcpy(ctx->buffer + left, buf, fill); + process(be, ctx, ctx->buffer); + buf += fill; + len -= fill; + left = 0; + } + while (len >= 64) { + ctx->total += 64; + process(be, ctx, buf); + buf += 64; + len -= 64; + } + if (len) { + ctx->total += len; + memcpy(ctx->buffer + left, buf, len); + } +} + +int sha1dc_accel_final(unsigned char hash[20], SHA1_CTX *ctx) +{ + static const unsigned char padding[64] = { 0x80 }; + uint32_t last = ctx->total & 63; + uint32_t padn = last < 56 ? 56 - last : 120 - last; + uint64_t bits; + int i; + + sha1dc_accel_update(ctx, padding, padn); + bits = (ctx->total - padn) << 3; + put_be32(ctx->buffer + 56, (uint32_t)(bits >> 32)); + put_be32(ctx->buffer + 60, (uint32_t)bits); + process(backend(), ctx, ctx->buffer); + + for (i = 0; i < 5; i++) + put_be32(hash + 4 * i, ctx->ihv[i]); + return ctx->found_collision; +} diff --git a/sha1dc-accel/sha1.h b/sha1dc-accel/sha1.h new file mode 100644 index 00000000000000..53dda518f0c83d --- /dev/null +++ b/sha1dc-accel/sha1.h @@ -0,0 +1,31 @@ +#ifndef SHA1DC_ACCEL_SHA1_H +#define SHA1DC_ACCEL_SHA1_H + +#if defined(DC_SHA1_SUBMODULE) +#include "../sha1collisiondetection/lib/sha1.h" +#else +#include "../sha1dc/sha1.h" +#endif + +/* + * A faster implementation of SHA-1 with collision detection, working on + * the SHA1_CTX of sha1dc/ (or of the sha1collisiondetection submodule). + * SHA1DCInit() and the SHA1DCSet*() functions set it up as usual; these + * then take the place of SHA1DCUpdate() and SHA1DCFinal(), with the same + * results. See sha1.c. + */ + +void sha1dc_accel_update(SHA1_CTX *ctx, const void *data, size_t len); +int sha1dc_accel_final(unsigned char hash[20], SHA1_CTX *ctx); + +/* + * The implementation in use, such as "shani+avx2" or "portable". For tests, + * sha1dc_accel_select() switches to another one by that name; it returns -1 + * if this build or CPU does not have it. Neither is thread-safe. + */ +const char *sha1dc_accel_backend(void); +int sha1dc_accel_select(const char *name); +/* The names this build and CPU can use, NULL-terminated. */ +const char *const *sha1dc_accel_backends(void); + +#endif /* SHA1DC_ACCEL_SHA1_H */ diff --git a/sha1dc-accel/ubc_check.c b/sha1dc-accel/ubc_check.c new file mode 100644 index 00000000000000..cbd21d33e683bd --- /dev/null +++ b/sha1dc-accel/ubc_check.c @@ -0,0 +1,1829 @@ +/* + * The unavoidable-bitconditions (UBC) check of SHA-1 collision detection, + * in one form per instruction set. + * + * Every form returns the same mask as ubc_check() in sha1dc/ubc_check.c: + * one bit per disturbance vector (DV) that the expanded message w[] has not + * ruled out. A DV is ruled out as soon as one of its bitconditions fails; + * each condition says that bit a of w[i] XOR bit b of w[j] equals c. + * + * Each form runs in two parts. The prefix tests a fixed set of conditions, + * chosen and packed into vector lanes for that instruction set, on every + * block; it rules out every DV for almost all blocks. The few blocks that + * survive it run the tail, which tests the remaining conditions of each DV + * still alive. + * + * The tables in this file were derived from the output of the solver in + * the "sha1dc" Rust crate by Sam Reis (https://github.com/srijs/sha1dc, + * commit 426b4afd), which picks the conditions and their packing. They are + * used under the MIT license: + * + * Copyright (c) 2017 Marc Stevens (Cryptology Group, Centrum Wiskunde & + * Informatica) + * Copyright (c) 2017 Dan Shumow (Microsoft Research) + * Copyright (c) 2026 Sam Reis + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "../git-compat-util.h" +#include "internal.h" + +#define DV_I_43_0_BIT ((uint32_t)1 << 0) +#define DV_I_44_0_BIT ((uint32_t)1 << 1) +#define DV_I_45_0_BIT ((uint32_t)1 << 2) +#define DV_I_46_0_BIT ((uint32_t)1 << 3) +#define DV_I_46_2_BIT ((uint32_t)1 << 4) +#define DV_I_47_0_BIT ((uint32_t)1 << 5) +#define DV_I_47_2_BIT ((uint32_t)1 << 6) +#define DV_I_48_0_BIT ((uint32_t)1 << 7) +#define DV_I_48_2_BIT ((uint32_t)1 << 8) +#define DV_I_49_0_BIT ((uint32_t)1 << 9) +#define DV_I_49_2_BIT ((uint32_t)1 << 10) +#define DV_I_50_0_BIT ((uint32_t)1 << 11) +#define DV_I_50_2_BIT ((uint32_t)1 << 12) +#define DV_I_51_0_BIT ((uint32_t)1 << 13) +#define DV_I_51_2_BIT ((uint32_t)1 << 14) +#define DV_I_52_0_BIT ((uint32_t)1 << 15) +#define DV_II_45_0_BIT ((uint32_t)1 << 16) +#define DV_II_46_0_BIT ((uint32_t)1 << 17) +#define DV_II_46_2_BIT ((uint32_t)1 << 18) +#define DV_II_47_0_BIT ((uint32_t)1 << 19) +#define DV_II_48_0_BIT ((uint32_t)1 << 20) +#define DV_II_49_0_BIT ((uint32_t)1 << 21) +#define DV_II_49_2_BIT ((uint32_t)1 << 22) +#define DV_II_50_0_BIT ((uint32_t)1 << 23) +#define DV_II_50_2_BIT ((uint32_t)1 << 24) +#define DV_II_51_0_BIT ((uint32_t)1 << 25) +#define DV_II_51_2_BIT ((uint32_t)1 << 26) +#define DV_II_52_0_BIT ((uint32_t)1 << 27) +#define DV_II_53_0_BIT ((uint32_t)1 << 28) +#define DV_II_54_0_BIT ((uint32_t)1 << 29) +#define DV_II_55_0_BIT ((uint32_t)1 << 30) +#define DV_II_56_0_BIT ((uint32_t)1 << 31) + +/* + * The prefixes loop over constant tables. Unrolling those loops fully lets + * the compiler fold each entry into the code as immediates, which is what + * makes them fast; without it they are two to three times slower. + */ +#if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 8) +#define UNROLL_TABLE _Pragma("GCC unroll 128") +#else +#define UNROLL_TABLE +#endif + +/* + * The tail loops run over a few conditions at a time, too few to gain from + * vectorizing; clang would otherwise vectorize them when targeting AVX2. + */ +#ifdef __clang__ +#define NO_VECTORIZE _Pragma("clang loop vectorize(disable)") +#else +#define NO_VECTORIZE +#endif + +/* A bitcondition: bit a of w[i] XOR bit b of w[j] must equal c. */ +struct ubc_cond { + uint8_t i, a, j, b, c; +}; + +static inline uint32_t cond_fails(const uint32_t *w, const struct ubc_cond *c, + int mirrored) +{ + unsigned i = mirrored ? 79 - c->i : c->i; + unsigned j = mirrored ? 79 - c->j : c->j; + + return (((w[i] >> c->a) ^ (w[j] >> c->b) ^ c->c) & 1); +} + +/* A condition of the scalar prefix, and the DVs it rules out if it fails. */ +struct ubc_prefix_cond { + struct ubc_cond cond; + uint32_t dvs; +}; + +/* + * A group of conditions for a vector prefix, one per lane. In lane k, the + * bit selected by test[k] of (w[lo + k] >> lo_shift) ^ (w[hi + k] >> hi_shift) + * must equal want, or the DVs in dvs[k] are ruled out. The lanes share + * everything but test[] and dvs[]; an unused lane has both 0. + * + * No group reads past w[66]. + */ +struct ubc_group4 { + uint8_t lo, lo_shift, hi, hi_shift, want; + uint32_t test[4]; + uint32_t dvs[4]; +}; + +struct ubc_group8 { + uint8_t lo, lo_shift, hi, hi_shift, want; + uint32_t test[8]; + uint32_t dvs[8]; +}; + +/* The tail conditions of DV d are checks[spans[d].start] onwards. */ +struct tail_span { + uint16_t start; + uint8_t len; +}; + +/* + * Runs the tail conditions of each DV still alive in `mask`, and clears + * the DVs with a failing one. Each condition fails about half the time, so + * it tests all of a DV's conditions rather than branch on each one. + */ +static inline uint32_t run_tail(const uint32_t *w, uint32_t mask, + const struct ubc_cond *checks, + const struct tail_span *spans, int mirrored) +{ + uint32_t out = mask; + while (mask) { + unsigned d = sha1dc_ctz(mask); + const struct ubc_cond *c = checks + spans[d].start; + const struct ubc_cond *end = c + spans[d].len; + uint32_t fail = 0; + + NO_VECTORIZE + for (; c < end; c++) + fail |= cond_fails(w, c, mirrored); + out &= ~(fail << d); + mask &= mask - 1; + } + return out; +} + +/* scalar form */ + +static const struct ubc_prefix_cond scalar_prefix_conds[] = { + { { 44, 29, 45, 29, 0 }, + DV_I_48_0_BIT | DV_I_51_0_BIT | DV_I_52_0_BIT | DV_II_45_0_BIT | + DV_II_46_0_BIT | DV_II_50_0_BIT | DV_II_51_0_BIT }, + { { 46, 29, 47, 29, 0 }, + DV_I_43_0_BIT | DV_I_50_0_BIT | DV_II_47_0_BIT | DV_II_48_0_BIT | + DV_II_52_0_BIT | DV_II_53_0_BIT }, + { { 45, 4, 48, 29, 0 }, + DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_49_0_BIT | DV_I_51_0_BIT | + DV_II_49_0_BIT | DV_II_54_0_BIT }, + { { 49, 29, 50, 29, 0 }, + DV_I_46_0_BIT | DV_II_45_0_BIT | DV_II_50_0_BIT | DV_II_51_0_BIT | + DV_II_55_0_BIT | DV_II_56_0_BIT }, + { { 40, 29, 41, 29, 0 }, + DV_I_44_0_BIT | DV_I_47_0_BIT | DV_I_48_0_BIT | DV_II_46_0_BIT | + DV_II_47_0_BIT | DV_II_56_0_BIT }, + { { 36, 1, 37, 6, 1 }, + DV_I_47_2_BIT | DV_I_50_2_BIT | DV_II_46_2_BIT }, + { { 47, 29, 48, 29, 0 }, + DV_I_44_0_BIT | DV_I_51_0_BIT | DV_II_48_0_BIT | DV_II_49_0_BIT | + DV_II_53_0_BIT | DV_II_54_0_BIT }, + { { 39, 1, 40, 6, 1 }, + DV_I_46_2_BIT | DV_I_50_2_BIT | DV_II_49_2_BIT }, + { { 40, 1, 41, 6, 1 }, + DV_I_47_2_BIT | DV_I_51_2_BIT | DV_II_50_2_BIT }, + { { 41, 1, 42, 6, 1 }, + DV_I_48_2_BIT | DV_II_46_2_BIT | DV_II_51_2_BIT }, + { { 43, 4, 46, 29, 0 }, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_49_0_BIT | + DV_II_47_0_BIT | DV_II_52_0_BIT }, + { { 46, 4, 49, 29, 0 }, + DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_50_0_BIT | DV_I_52_0_BIT | + DV_II_50_0_BIT | DV_II_55_0_BIT }, + { { 45, 6, 47, 6, 0 }, + DV_I_47_2_BIT | DV_I_49_2_BIT | DV_I_51_2_BIT }, + { { 45, 29, 46, 29, 0 }, + DV_I_49_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT | DV_II_47_0_BIT | + DV_II_51_0_BIT | DV_II_52_0_BIT }, + { { 44, 4, 47, 29, 0 }, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_50_0_BIT | + DV_II_48_0_BIT | DV_II_53_0_BIT }, + { { 48, 29, 49, 29, 0 }, + DV_I_45_0_BIT | DV_I_52_0_BIT | DV_II_49_0_BIT | DV_II_50_0_BIT | + DV_II_54_0_BIT | DV_II_55_0_BIT }, + { { 44, 6, 46, 6, 0 }, + DV_I_46_2_BIT | DV_I_48_2_BIT | DV_I_50_2_BIT }, + { { 47, 4, 50, 29, 0 }, + DV_I_47_0_BIT | DV_I_49_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT | + DV_II_51_0_BIT | DV_II_56_0_BIT }, + { { 35, 1, 36, 6, 1 }, + DV_I_46_2_BIT | DV_I_49_2_BIT }, + { { 44, 1, 45, 6, 1 }, + DV_I_51_2_BIT | DV_II_49_2_BIT }, + { { 42, 6, 43, 1, 0 }, + DV_II_46_2_BIT | DV_II_51_2_BIT }, + { { 37, 4, 40, 29, 0 }, + DV_I_43_0_BIT | DV_I_47_0_BIT | DV_II_46_0_BIT | DV_II_53_0_BIT | + DV_II_55_0_BIT }, + { { 41, 6, 42, 1, 0 }, + DV_I_51_2_BIT | DV_II_50_2_BIT }, + { { 40, 4, 43, 29, 0 }, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_50_0_BIT | DV_II_49_0_BIT | + DV_II_56_0_BIT }, + { { 41, 4, 44, 29, 0 }, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_51_0_BIT | + DV_II_45_0_BIT | DV_II_50_0_BIT }, + { { 52, 29, 53, 29, 0 }, + DV_I_49_0_BIT | DV_II_45_0_BIT | DV_II_48_0_BIT | DV_II_53_0_BIT | + DV_II_54_0_BIT }, + { { 40, 6, 41, 1, 0 }, + DV_I_50_2_BIT | DV_II_49_2_BIT }, + { { 46, 6, 47, 1, 0 }, + DV_I_46_2_BIT | DV_II_50_2_BIT }, + { { 47, 6, 48, 1, 0 }, + DV_I_47_2_BIT | DV_II_51_2_BIT }, + { { 42, 4, 45, 29, 0 }, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_52_0_BIT | + DV_II_46_0_BIT | DV_II_51_0_BIT }, + { { 37, 1, 38, 6, 1 }, + DV_I_48_2_BIT | DV_I_51_2_BIT }, + { { 43, 6, 45, 6, 0 }, + DV_I_47_2_BIT | DV_I_49_2_BIT }, + { { 53, 29, 54, 29, 0 }, + DV_I_50_0_BIT | DV_II_46_0_BIT | DV_II_49_0_BIT | DV_II_54_0_BIT | + DV_II_55_0_BIT }, + { { 41, 29, 42, 29, 0 }, + DV_I_45_0_BIT | DV_I_48_0_BIT | DV_I_49_0_BIT | DV_II_47_0_BIT | + DV_II_48_0_BIT }, + { { 50, 29, 51, 29, 0 }, + DV_I_47_0_BIT | DV_II_46_0_BIT | DV_II_51_0_BIT | DV_II_52_0_BIT | + DV_II_56_0_BIT }, + { { 61, 2, 62, 7, 1 }, + DV_I_46_2_BIT | DV_II_46_2_BIT }, + { { 46, 6, 48, 6, 0 }, + DV_I_48_2_BIT | DV_I_50_2_BIT }, + { { 39, 4, 42, 29, 0 }, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_49_0_BIT | DV_II_48_0_BIT | + DV_II_55_0_BIT }, + { { 43, 29, 44, 29, 0 }, + DV_I_47_0_BIT | DV_I_50_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT | + DV_II_49_0_BIT | DV_II_50_0_BIT }, + { { 47, 6, 49, 6, 0 }, + DV_I_49_2_BIT | DV_I_51_2_BIT }, + { { 51, 29, 52, 29, 0 }, + DV_I_48_0_BIT | DV_II_47_0_BIT | DV_II_52_0_BIT | DV_II_53_0_BIT }, + { { 36, 0, 37, 5, 1 }, + DV_II_49_2_BIT }, + { { 37, 0, 38, 5, 1 }, + DV_II_50_2_BIT }, + { { 38, 0, 39, 5, 1 }, + DV_II_51_2_BIT }, + { { 38, 4, 41, 29, 0 }, + DV_I_44_0_BIT | DV_I_48_0_BIT | DV_II_47_0_BIT | DV_II_54_0_BIT | + DV_II_56_0_BIT }, + { { 50, 6, 51, 1, 0 }, + DV_I_50_2_BIT | DV_II_46_2_BIT }, + { { 42, 6, 44, 6, 0 }, + DV_I_46_2_BIT | DV_I_48_2_BIT }, + { { 48, 4, 51, 29, 0 }, + DV_I_48_0_BIT | DV_I_50_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT | + DV_II_52_0_BIT }, + { { 42, 29, 43, 29, 0 }, + DV_I_46_0_BIT | DV_I_49_0_BIT | DV_I_50_0_BIT | DV_II_48_0_BIT | + DV_II_49_0_BIT }, + { { 54, 29, 55, 29, 0 }, + DV_I_51_0_BIT | DV_II_47_0_BIT | DV_II_50_0_BIT | DV_II_55_0_BIT | + DV_II_56_0_BIT }, + { { 38, 1, 39, 6, 1 }, + DV_I_49_2_BIT }, + { { 45, 1, 46, 6, 1 }, + DV_II_50_2_BIT }, + { { 46, 1, 47, 6, 1 }, + DV_II_51_2_BIT }, + { { 50, 1, 51, 6, 1 }, + DV_II_49_2_BIT }, + { { 39, 4, 40, 29, 1 }, + DV_I_43_0_BIT | DV_II_53_0_BIT | DV_II_55_0_BIT }, + { { 55, 29, 56, 29, 0 }, + DV_I_52_0_BIT | DV_II_48_0_BIT | DV_II_51_0_BIT | DV_II_56_0_BIT }, + { { 48, 6, 50, 6, 0 }, + DV_I_50_2_BIT | DV_II_46_2_BIT }, + { { 49, 4, 52, 29, 0 }, + DV_I_49_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT | DV_II_47_0_BIT | + DV_II_53_0_BIT }, + { { 40, 4, 41, 29, 1 }, + DV_I_44_0_BIT | DV_II_54_0_BIT | DV_II_56_0_BIT }, + { { 41, 4, 42, 29, 1 }, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_II_55_0_BIT }, + { { 42, 1, 43, 6, 1 }, + DV_I_49_2_BIT }, + { { 51, 1, 52, 6, 1 }, + DV_II_50_2_BIT }, + { { 52, 1, 53, 6, 1 }, + DV_II_51_2_BIT }, + { { 62, 2, 63, 7, 1 }, + DV_I_47_2_BIT }, + { { 63, 2, 64, 7, 1 }, + DV_I_48_2_BIT }, + { { 45, 6, 46, 1, 0 }, + DV_II_49_2_BIT }, + { { 36, 4, 40, 29, 0 }, + DV_I_46_0_BIT | DV_I_49_0_BIT | DV_II_45_0_BIT | DV_II_48_0_BIT }, + { { 50, 4, 53, 29, 0 }, + DV_I_50_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT | DV_II_48_0_BIT | + DV_II_54_0_BIT }, + { { 56, 29, 57, 29, 0 }, + DV_II_49_0_BIT | DV_II_52_0_BIT }, + { { 43, 4, 44, 29, 1 }, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_47_0_BIT } +}; + +static const struct ubc_cond scalar_tail_checks[] = { + /* DV_I_43_0 */ + { 58, 0, 59, 5, 1 }, + { 58, 0, 63, 30, 1 }, + { 61, 1, 62, 6, 1 }, + { 43, 4, 47, 29, 0 }, + /* DV_I_44_0 */ + { 40, 4, 42, 4, 1 }, + { 42, 4, 44, 4, 1 }, + { 59, 0, 60, 5, 1 }, + { 59, 0, 64, 30, 1 }, + { 62, 1, 63, 6, 1 }, + { 44, 4, 48, 29, 0 }, + /* DV_I_45_0 */ + { 43, 4, 45, 4, 1 }, + { 60, 0, 61, 5, 1 }, + { 63, 1, 64, 6, 1 }, + { 35, 4, 39, 29, 0 }, + /* DV_I_46_0 */ + { 40, 4, 42, 4, 1 }, + { 42, 4, 44, 4, 1 }, + { 44, 4, 46, 4, 1 }, + { 61, 0, 62, 5, 1 }, + /* DV_I_46_2 */ + { 39, 1, 42, 6, 1 }, + /* DV_I_47_0 */ + { 43, 4, 45, 4, 1 }, + { 45, 4, 47, 4, 1 }, + { 62, 0, 63, 5, 1 }, + { 37, 4, 41, 29, 0 }, + /* DV_I_47_2 */ + { 40, 1, 43, 6, 1 }, + /* DV_I_48_0 */ + { 42, 4, 44, 4, 1 }, + { 44, 4, 46, 4, 1 }, + { 46, 4, 48, 4, 1 }, + { 63, 0, 64, 5, 1 }, + { 35, 4, 39, 29, 0 }, + { 38, 4, 42, 29, 0 }, + /* DV_I_48_2 */ + { 41, 1, 49, 1, 1 }, + /* DV_I_49_0 */ + { 43, 4, 45, 4, 1 }, + { 45, 4, 47, 4, 1 }, + { 47, 4, 49, 4, 1 }, + { 39, 4, 43, 29, 0 }, + /* DV_I_49_2 */ + { 38, 1, 40, 1, 1 }, + { 42, 1, 50, 1, 1 }, + /* DV_I_50_0 */ + { 36, 4, 37, 4, 1 }, + { 44, 4, 46, 4, 1 }, + { 46, 4, 48, 4, 1 }, + { 48, 4, 50, 4, 1 }, + { 37, 4, 41, 29, 0 }, + { 40, 4, 44, 29, 0 }, + /* DV_I_50_2 */ + { 43, 1, 44, 6, 1 }, + /* DV_I_51_0 */ + { 37, 4, 38, 4, 1 }, + { 45, 4, 47, 4, 1 }, + { 47, 4, 49, 4, 1 }, + { 52, 29, 55, 29, 1 }, + { 35, 3, 39, 28, 0 }, + { 38, 4, 42, 29, 0 }, + { 41, 4, 45, 29, 0 }, + { 51, 4, 54, 29, 0 }, + /* DV_I_51_2 */ + { 44, 1, 51, 6, 1 }, + { 44, 1, 52, 1, 1 }, + { 35, 5, 39, 30, 0 }, + { 37, 1, 37, 6, 0 }, + /* DV_I_52_0 */ + { 38, 4, 39, 4, 1 }, + { 46, 4, 48, 4, 1 }, + { 48, 4, 50, 4, 1 }, + { 53, 29, 56, 29, 1 }, + { 39, 4, 43, 29, 0 }, + { 42, 4, 46, 29, 0 }, + { 52, 4, 55, 29, 0 }, + /* DV_II_45_0 */ + { 47, 4, 49, 4, 1 }, + { 60, 0, 61, 5, 1 }, + { 63, 1, 64, 6, 1 }, + { 41, 4, 45, 29, 0 }, + /* DV_II_46_0 */ + { 48, 4, 50, 4, 1 }, + { 61, 0, 62, 5, 1 }, + { 37, 4, 41, 29, 0 }, + { 42, 4, 46, 29, 0 }, + /* DV_II_46_2 */ + { 47, 1, 48, 6, 1 }, + /* DV_II_47_0 */ + { 52, 29, 55, 29, 1 }, + { 62, 0, 63, 5, 1 }, + { 35, 3, 39, 28, 0 }, + { 35, 4, 39, 29, 0 }, + { 38, 4, 42, 29, 0 }, + { 43, 4, 47, 29, 0 }, + { 51, 4, 54, 29, 0 }, + /* DV_II_48_0 */ + { 35, 30, 36, 3, 1 }, + { 35, 30, 40, 28, 1 }, + { 52, 29, 55, 29, 1 }, + { 53, 29, 56, 29, 1 }, + { 63, 0, 64, 5, 1 }, + { 39, 4, 43, 29, 0 }, + { 44, 4, 48, 29, 0 }, + { 52, 4, 55, 29, 0 }, + /* DV_II_49_0 */ + { 36, 30, 37, 3, 1 }, + { 36, 30, 41, 28, 1 }, + { 53, 29, 56, 29, 1 }, + { 37, 4, 41, 29, 0 }, + { 40, 4, 44, 29, 0 }, + { 51, 4, 54, 29, 0 }, + { 53, 4, 56, 29, 0 }, + /* DV_II_49_2 */ + { 36, 0, 41, 30, 1 }, + { 50, 1, 53, 6, 1 }, + { 50, 1, 54, 1, 1 }, + /* DV_II_50_0 */ + { 37, 30, 38, 3, 1 }, + { 37, 30, 42, 28, 1 }, + { 55, 29, 58, 29, 1 }, + { 38, 4, 42, 29, 0 }, + { 41, 4, 45, 29, 0 }, + { 52, 4, 55, 29, 0 }, + { 54, 4, 57, 29, 0 }, + { 57, 29, 58, 29, 0 }, + /* DV_II_50_2 */ + { 37, 0, 42, 30, 1 }, + { 51, 1, 54, 6, 1 }, + { 51, 1, 55, 1, 1 }, + /* DV_II_51_0 */ + { 38, 30, 39, 3, 1 }, + { 38, 30, 43, 28, 1 }, + { 55, 29, 58, 29, 1 }, + { 56, 29, 59, 29, 1 }, + { 39, 4, 43, 29, 0 }, + { 42, 4, 46, 29, 0 }, + { 53, 4, 56, 29, 0 }, + { 55, 4, 58, 29, 0 }, + /* DV_II_51_2 */ + { 38, 0, 43, 30, 1 }, + { 52, 1, 55, 6, 1 }, + { 52, 1, 56, 1, 1 }, + /* DV_II_52_0 */ + { 36, 4, 38, 4, 1 }, + { 39, 30, 40, 3, 1 }, + { 39, 30, 44, 28, 1 }, + { 54, 4, 60, 29, 1 }, + { 56, 29, 59, 29, 1 }, + { 40, 4, 44, 29, 0 }, + { 43, 4, 47, 29, 0 }, + { 54, 4, 57, 29, 0 }, + { 56, 4, 59, 29, 0 }, + /* DV_II_53_0 */ + { 55, 4, 57, 4, 1 }, + { 55, 4, 61, 29, 1 }, + { 41, 3, 45, 28, 0 }, + { 41, 4, 45, 29, 0 }, + { 44, 4, 48, 29, 0 }, + { 55, 4, 58, 29, 0 }, + { 57, 29, 58, 29, 0 }, + /* DV_II_54_0 */ + { 36, 4, 38, 4, 1 }, + { 42, 3, 46, 28, 0 }, + { 42, 4, 46, 29, 0 }, + { 56, 4, 59, 29, 0 }, + { 56, 4, 58, 29, 0 }, + { 58, 4, 62, 29, 0 }, + /* DV_II_55_0 */ + { 43, 3, 47, 28, 0 }, + { 43, 4, 47, 29, 0 }, + { 51, 4, 54, 29, 0 }, + { 57, 4, 59, 29, 0 }, + { 59, 4, 63, 29, 0 }, + /* DV_II_56_0 */ + { 40, 4, 42, 4, 1 }, + { 44, 3, 48, 28, 0 }, + { 44, 4, 48, 29, 0 }, + { 52, 4, 55, 29, 0 }, + { 60, 4, 64, 29, 0 } +}; + +static const struct tail_span scalar_tail_spans[32] = { + { 0, 4 }, /* DV_I_43_0 */ + { 4, 6 }, /* DV_I_44_0 */ + { 10, 4 }, /* DV_I_45_0 */ + { 14, 4 }, /* DV_I_46_0 */ + { 18, 1 }, /* DV_I_46_2 */ + { 19, 4 }, /* DV_I_47_0 */ + { 23, 1 }, /* DV_I_47_2 */ + { 24, 6 }, /* DV_I_48_0 */ + { 30, 1 }, /* DV_I_48_2 */ + { 31, 4 }, /* DV_I_49_0 */ + { 35, 2 }, /* DV_I_49_2 */ + { 37, 6 }, /* DV_I_50_0 */ + { 43, 1 }, /* DV_I_50_2 */ + { 44, 8 }, /* DV_I_51_0 */ + { 52, 4 }, /* DV_I_51_2 */ + { 56, 7 }, /* DV_I_52_0 */ + { 63, 4 }, /* DV_II_45_0 */ + { 67, 4 }, /* DV_II_46_0 */ + { 71, 1 }, /* DV_II_46_2 */ + { 72, 7 }, /* DV_II_47_0 */ + { 79, 8 }, /* DV_II_48_0 */ + { 87, 7 }, /* DV_II_49_0 */ + { 94, 3 }, /* DV_II_49_2 */ + { 97, 8 }, /* DV_II_50_0 */ + { 105, 3 }, /* DV_II_50_2 */ + { 108, 8 }, /* DV_II_51_0 */ + { 116, 3 }, /* DV_II_51_2 */ + { 119, 9 }, /* DV_II_52_0 */ + { 128, 7 }, /* DV_II_53_0 */ + { 135, 6 }, /* DV_II_54_0 */ + { 141, 5 }, /* DV_II_55_0 */ + { 146, 5 } /* DV_II_56_0 */ +}; + +static uint32_t scalar_prefix(const uint32_t *w) +{ + uint32_t mask = 0xFFFFFFFF; + size_t i; + + UNROLL_TABLE + for (i = 0; i < ARRAY_SIZE(scalar_prefix_conds); i++) { + const struct ubc_prefix_cond *p = &scalar_prefix_conds[i]; + mask &= ~(p->dvs & (0 - cond_fails(w, &p->cond, 0))); + } + return mask; +} + +uint32_t sha1dc_ubc_check_scalar(const uint32_t w[80]) +{ + uint32_t mask = scalar_prefix(w); + /* Every check only clears bits, so an empty mask settles it. */ + if (!mask) + return 0; + return run_tail(w, mask, scalar_tail_checks, scalar_tail_spans, 0); +} + +/* neon form */ + +#ifdef SHA1DC_HAVE_NEON + +static const struct ubc_group4 neon_groups[] = { + { 35, 0, 36, 5, 1, + { 1u << 1, 1u << 1, 1u << 1, 1u << 0 }, + { DV_I_46_2_BIT | DV_I_49_2_BIT, + DV_I_47_2_BIT | DV_I_50_2_BIT | DV_II_46_2_BIT, + DV_I_48_2_BIT | DV_I_51_2_BIT, + DV_II_51_2_BIT } }, + { 36, 0, 37, 5, 1, + { 1u << 0, 1u << 0, 1u << 1, 1u << 1 }, + { DV_II_49_2_BIT, + DV_II_50_2_BIT, + DV_I_49_2_BIT, + DV_I_46_2_BIT | DV_I_50_2_BIT | DV_II_49_2_BIT } }, + { 36, 0, 38, 0, 1, + { 1u << 4, 1u << 4, 1u << 1, 1u << 4 }, + { DV_II_52_0_BIT | DV_II_54_0_BIT, + DV_I_43_0_BIT | DV_II_53_0_BIT | DV_II_55_0_BIT, + DV_I_49_2_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_II_55_0_BIT } }, + { 36, 0, 40, 25, 0, + { 1u << 4, 1u << 5, 1u << 5, 1u << 5 }, + { DV_I_46_0_BIT | DV_I_49_0_BIT | DV_II_45_0_BIT | + DV_II_48_0_BIT, + DV_II_49_2_BIT, + DV_II_50_2_BIT, + DV_II_51_2_BIT } }, + { 38, 0, 40, 0, 1, + { 1u << 4, 1u << 1, 1u << 1, 1u << 1 }, + { DV_I_44_0_BIT | DV_II_54_0_BIT | DV_II_56_0_BIT, + DV_I_50_2_BIT | DV_II_49_2_BIT, + DV_I_51_2_BIT | DV_II_50_2_BIT, + DV_II_46_2_BIT | DV_II_51_2_BIT } }, + { 37, 0, 40, 25, 0, + { 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_I_43_0_BIT | DV_I_47_0_BIT | DV_II_46_0_BIT | + DV_II_53_0_BIT | DV_II_55_0_BIT, + DV_I_44_0_BIT | DV_I_48_0_BIT | DV_II_47_0_BIT | + DV_II_54_0_BIT | DV_II_56_0_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_49_0_BIT | DV_II_48_0_BIT | + DV_II_55_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_50_0_BIT | DV_II_49_0_BIT | + DV_II_56_0_BIT } }, + { 40, 0, 41, 5, 1, + { 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_I_47_2_BIT | DV_I_51_2_BIT | DV_II_50_2_BIT, + DV_I_48_2_BIT | DV_II_46_2_BIT | DV_II_51_2_BIT, + DV_I_49_2_BIT, + DV_I_50_2_BIT } }, + { 40, 0, 41, 0, 0, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_44_0_BIT | DV_I_47_0_BIT | DV_I_48_0_BIT | DV_II_46_0_BIT | + DV_II_47_0_BIT | DV_II_56_0_BIT, + DV_I_45_0_BIT | DV_I_48_0_BIT | DV_I_49_0_BIT | DV_II_47_0_BIT | + DV_II_48_0_BIT, + DV_I_46_0_BIT | DV_I_49_0_BIT | DV_I_50_0_BIT | DV_II_48_0_BIT | + DV_II_49_0_BIT, + DV_I_47_0_BIT | DV_I_50_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT | + DV_II_49_0_BIT | DV_II_50_0_BIT } }, + { 41, 0, 43, 0, 0, + { 1u << 6, 1u << 6, 1u << 6, 1u << 6 }, + { DV_I_47_2_BIT, + DV_I_46_2_BIT | DV_I_48_2_BIT, + DV_I_47_2_BIT | DV_I_49_2_BIT, + DV_I_46_2_BIT | DV_I_48_2_BIT | DV_I_50_2_BIT } }, + { 41, 0, 44, 25, 0, + { 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_51_0_BIT | + DV_II_45_0_BIT | DV_II_50_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_52_0_BIT | + DV_II_46_0_BIT | DV_II_51_0_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_49_0_BIT | + DV_II_47_0_BIT | DV_II_52_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_50_0_BIT | + DV_II_48_0_BIT | DV_II_53_0_BIT } }, + { 44, 0, 45, 5, 1, + { 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_I_51_2_BIT | DV_II_49_2_BIT, + DV_II_50_2_BIT, + DV_II_51_2_BIT, + DV_II_46_2_BIT } }, + { 44, 0, 45, 0, 0, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_48_0_BIT | DV_I_51_0_BIT | DV_I_52_0_BIT | DV_II_45_0_BIT | + DV_II_46_0_BIT | DV_II_50_0_BIT | DV_II_51_0_BIT, + DV_I_49_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT | + DV_II_47_0_BIT | DV_II_51_0_BIT | DV_II_52_0_BIT, + DV_I_43_0_BIT | DV_I_50_0_BIT | DV_II_47_0_BIT | + DV_II_48_0_BIT | DV_II_52_0_BIT | DV_II_53_0_BIT, + DV_I_44_0_BIT | DV_I_51_0_BIT | DV_II_48_0_BIT | + DV_II_49_0_BIT | DV_II_53_0_BIT | DV_II_54_0_BIT } }, + { 45, 5, 46, 0, 0, + { 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_II_49_2_BIT, + DV_I_46_2_BIT | DV_II_50_2_BIT, + DV_I_47_2_BIT | DV_II_51_2_BIT, + DV_I_48_2_BIT } }, + { 45, 0, 47, 0, 0, + { 1u << 6, 1u << 6, 1u << 6, 1u << 6 }, + { DV_I_47_2_BIT | DV_I_49_2_BIT | DV_I_51_2_BIT, + DV_I_48_2_BIT | DV_I_50_2_BIT, + DV_I_49_2_BIT | DV_I_51_2_BIT, + DV_I_50_2_BIT | DV_II_46_2_BIT } }, + { 45, 0, 47, 0, 1, + { 1u << 29, 1u << 29, 1u << 4, 1u << 4 }, + { DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_48_0_BIT, + DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_49_0_BIT, + DV_I_49_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT, + DV_I_50_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT } }, + { 45, 0, 48, 25, 0, + { 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_49_0_BIT | DV_I_51_0_BIT | + DV_II_49_0_BIT | DV_II_54_0_BIT, + DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_50_0_BIT | DV_I_52_0_BIT | + DV_II_50_0_BIT | DV_II_55_0_BIT, + DV_I_47_0_BIT | DV_I_49_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT | + DV_II_51_0_BIT | DV_II_56_0_BIT, + DV_I_48_0_BIT | DV_I_50_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT | + DV_II_52_0_BIT } }, + { 48, 0, 49, 0, 0, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_45_0_BIT | DV_I_52_0_BIT | DV_II_49_0_BIT | + DV_II_50_0_BIT | DV_II_54_0_BIT | DV_II_55_0_BIT, + DV_I_46_0_BIT | DV_II_45_0_BIT | DV_II_50_0_BIT | + DV_II_51_0_BIT | DV_II_55_0_BIT | DV_II_56_0_BIT, + DV_I_47_0_BIT | DV_II_46_0_BIT | DV_II_51_0_BIT | + DV_II_52_0_BIT | DV_II_56_0_BIT, + DV_I_48_0_BIT | DV_II_47_0_BIT | DV_II_52_0_BIT | + DV_II_53_0_BIT } }, + { 52, 0, 53, 0, 0, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_49_0_BIT | DV_II_45_0_BIT | DV_II_48_0_BIT | + DV_II_53_0_BIT | DV_II_54_0_BIT, + DV_I_50_0_BIT | DV_II_46_0_BIT | DV_II_49_0_BIT | + DV_II_54_0_BIT | DV_II_55_0_BIT, + DV_I_51_0_BIT | DV_II_47_0_BIT | DV_II_50_0_BIT | + DV_II_55_0_BIT | DV_II_56_0_BIT, + DV_I_52_0_BIT | DV_II_48_0_BIT | DV_II_51_0_BIT | + DV_II_56_0_BIT } }, + { 52, 0, 55, 0, 1, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_51_0_BIT | DV_II_47_0_BIT | DV_II_48_0_BIT, + DV_I_52_0_BIT | DV_II_48_0_BIT | DV_II_49_0_BIT, + DV_II_49_0_BIT | DV_II_50_0_BIT, + DV_II_50_0_BIT | DV_II_51_0_BIT } }, + { 60, 0, 61, 5, 1, + { 1u << 0, 1u << 2, 1u << 2, 1u << 2 }, + { DV_I_45_0_BIT | DV_II_45_0_BIT, + DV_I_46_2_BIT | DV_II_46_2_BIT, + DV_I_47_2_BIT, + DV_I_48_2_BIT } } +}; + +static const struct ubc_cond neon_tail_checks[] = { + /* DV_I_43_0 */ + { 41, 4, 43, 4, 1 }, + { 58, 0, 59, 5, 1 }, + { 58, 0, 63, 30, 1 }, + { 61, 1, 62, 6, 1 }, + { 43, 4, 47, 29, 0 }, + /* DV_I_44_0 */ + { 40, 4, 42, 4, 1 }, + { 59, 0, 60, 5, 1 }, + { 59, 0, 64, 30, 1 }, + { 62, 1, 63, 6, 1 }, + { 44, 4, 48, 29, 0 }, + /* DV_I_45_0 */ + { 41, 4, 43, 4, 1 }, + { 63, 1, 64, 6, 1 }, + { 35, 4, 39, 29, 0 }, + /* DV_I_46_0 */ + { 40, 4, 42, 4, 1 }, + { 44, 4, 46, 4, 1 }, + { 61, 0, 62, 5, 1 }, + /* DV_I_46_2 */ + { 39, 1, 42, 6, 1 }, + /* DV_I_47_0 */ + { 41, 4, 43, 4, 1 }, + { 45, 4, 47, 4, 1 }, + { 62, 0, 63, 5, 1 }, + { 37, 4, 41, 29, 0 }, + /* DV_I_48_0 */ + { 44, 4, 46, 4, 1 }, + { 46, 4, 48, 4, 1 }, + { 63, 0, 64, 5, 1 }, + { 35, 4, 39, 29, 0 }, + { 38, 4, 42, 29, 0 }, + /* DV_I_49_0 */ + { 45, 4, 47, 4, 1 }, + { 39, 4, 43, 29, 0 }, + { 49, 4, 52, 29, 0 }, + /* DV_I_49_2 */ + { 42, 1, 50, 1, 1 }, + /* DV_I_50_0 */ + { 36, 4, 37, 4, 1 }, + { 44, 4, 46, 4, 1 }, + { 46, 4, 48, 4, 1 }, + { 37, 4, 41, 29, 0 }, + { 40, 4, 44, 29, 0 }, + { 50, 4, 53, 29, 0 }, + /* DV_I_50_2 */ + { 48, 6, 51, 1, 0 }, + /* DV_I_51_0 */ + { 37, 4, 38, 4, 1 }, + { 45, 4, 47, 4, 1 }, + { 35, 3, 39, 28, 0 }, + { 38, 4, 42, 29, 0 }, + { 41, 4, 45, 29, 0 }, + { 49, 4, 52, 29, 0 }, + { 51, 4, 54, 29, 0 }, + /* DV_I_51_2 */ + { 44, 1, 51, 6, 1 }, + { 44, 1, 52, 1, 1 }, + { 35, 5, 39, 30, 0 }, + { 37, 1, 37, 6, 0 }, + /* DV_I_52_0 */ + { 38, 4, 39, 4, 1 }, + { 46, 4, 48, 4, 1 }, + { 39, 4, 43, 29, 0 }, + { 42, 4, 46, 29, 0 }, + { 50, 4, 53, 29, 0 }, + { 52, 4, 55, 29, 0 }, + /* DV_II_45_0 */ + { 63, 1, 64, 6, 1 }, + { 41, 4, 45, 29, 0 }, + { 49, 4, 52, 29, 0 }, + /* DV_II_46_0 */ + { 61, 0, 62, 5, 1 }, + { 37, 4, 41, 29, 0 }, + { 42, 4, 46, 29, 0 }, + { 50, 4, 53, 29, 0 }, + /* DV_II_46_2 */ + { 48, 6, 51, 1, 0 }, + /* DV_II_47_0 */ + { 62, 0, 63, 5, 1 }, + { 35, 3, 39, 28, 0 }, + { 35, 4, 39, 29, 0 }, + { 38, 4, 42, 29, 0 }, + { 43, 4, 47, 29, 0 }, + { 49, 4, 52, 29, 0 }, + { 51, 4, 54, 29, 0 }, + /* DV_II_48_0 */ + { 35, 30, 36, 3, 1 }, + { 35, 30, 40, 28, 1 }, + { 63, 0, 64, 5, 1 }, + { 39, 4, 43, 29, 0 }, + { 44, 4, 48, 29, 0 }, + { 50, 4, 53, 29, 0 }, + { 52, 4, 55, 29, 0 }, + /* DV_II_49_0 */ + { 36, 30, 37, 3, 1 }, + { 36, 30, 41, 28, 1 }, + { 37, 4, 41, 29, 0 }, + { 40, 4, 44, 29, 0 }, + { 51, 4, 54, 29, 0 }, + { 53, 4, 56, 29, 0 }, + /* DV_II_49_2 */ + { 50, 1, 51, 6, 1 }, + { 50, 1, 53, 6, 1 }, + { 50, 1, 54, 1, 1 }, + /* DV_II_50_0 */ + { 37, 30, 38, 3, 1 }, + { 37, 30, 42, 28, 1 }, + { 38, 4, 42, 29, 0 }, + { 41, 4, 45, 29, 0 }, + { 52, 4, 55, 29, 0 }, + { 54, 4, 57, 29, 0 }, + /* DV_II_50_2 */ + { 51, 1, 52, 6, 1 }, + { 51, 1, 54, 6, 1 }, + { 51, 1, 55, 1, 1 }, + /* DV_II_51_0 */ + { 38, 30, 39, 3, 1 }, + { 38, 30, 43, 28, 1 }, + { 56, 29, 59, 29, 1 }, + { 39, 4, 43, 29, 0 }, + { 42, 4, 46, 29, 0 }, + { 53, 4, 56, 29, 0 }, + { 55, 4, 58, 29, 0 }, + /* DV_II_51_2 */ + { 52, 1, 53, 6, 1 }, + { 52, 1, 55, 6, 1 }, + { 52, 1, 56, 1, 1 }, + /* DV_II_52_0 */ + { 39, 30, 40, 3, 1 }, + { 39, 30, 44, 28, 1 }, + { 54, 4, 56, 4, 1 }, + { 54, 4, 60, 29, 1 }, + { 56, 29, 59, 29, 1 }, + { 40, 4, 44, 29, 0 }, + { 43, 4, 47, 29, 0 }, + { 54, 4, 57, 29, 0 }, + { 56, 4, 59, 29, 0 }, + /* DV_II_53_0 */ + { 55, 4, 57, 4, 1 }, + { 55, 4, 61, 29, 1 }, + { 41, 3, 45, 28, 0 }, + { 41, 4, 45, 29, 0 }, + { 44, 4, 48, 29, 0 }, + { 49, 4, 52, 29, 0 }, + { 55, 4, 58, 29, 0 }, + { 55, 4, 57, 29, 0 }, + /* DV_II_54_0 */ + { 42, 3, 46, 28, 0 }, + { 42, 4, 46, 29, 0 }, + { 50, 4, 53, 29, 0 }, + { 56, 4, 59, 29, 0 }, + { 56, 4, 58, 29, 0 }, + { 58, 4, 62, 29, 0 }, + /* DV_II_55_0 */ + { 43, 3, 47, 28, 0 }, + { 43, 4, 47, 29, 0 }, + { 51, 4, 54, 29, 0 }, + { 57, 4, 59, 29, 0 }, + { 59, 4, 63, 29, 0 }, + /* DV_II_56_0 */ + { 40, 4, 42, 4, 1 }, + { 44, 3, 48, 28, 0 }, + { 44, 4, 48, 29, 0 }, + { 52, 4, 55, 29, 0 }, + { 60, 4, 64, 29, 0 } +}; + +static const struct tail_span neon_tail_spans[32] = { + { 0, 5 }, /* DV_I_43_0 */ + { 5, 5 }, /* DV_I_44_0 */ + { 10, 3 }, /* DV_I_45_0 */ + { 13, 3 }, /* DV_I_46_0 */ + { 16, 1 }, /* DV_I_46_2 */ + { 17, 4 }, /* DV_I_47_0 */ + { 21, 0 }, /* DV_I_47_2 */ + { 21, 5 }, /* DV_I_48_0 */ + { 26, 0 }, /* DV_I_48_2 */ + { 26, 3 }, /* DV_I_49_0 */ + { 29, 1 }, /* DV_I_49_2 */ + { 30, 6 }, /* DV_I_50_0 */ + { 36, 1 }, /* DV_I_50_2 */ + { 37, 7 }, /* DV_I_51_0 */ + { 44, 4 }, /* DV_I_51_2 */ + { 48, 6 }, /* DV_I_52_0 */ + { 54, 3 }, /* DV_II_45_0 */ + { 57, 4 }, /* DV_II_46_0 */ + { 61, 1 }, /* DV_II_46_2 */ + { 62, 7 }, /* DV_II_47_0 */ + { 69, 7 }, /* DV_II_48_0 */ + { 76, 6 }, /* DV_II_49_0 */ + { 82, 3 }, /* DV_II_49_2 */ + { 85, 6 }, /* DV_II_50_0 */ + { 91, 3 }, /* DV_II_50_2 */ + { 94, 7 }, /* DV_II_51_0 */ + { 101, 3 }, /* DV_II_51_2 */ + { 104, 9 }, /* DV_II_52_0 */ + { 113, 8 }, /* DV_II_53_0 */ + { 121, 6 }, /* DV_II_54_0 */ + { 127, 5 }, /* DV_II_55_0 */ + { 132, 5 } /* DV_II_56_0 */ +}; + +static uint32_t neon_prefix(const uint32_t *w) +{ + uint32x4_t acc = vdupq_n_u32(0); + uint32x2_t folded; + size_t i; + + UNROLL_TABLE + for (i = 0; i < ARRAY_SIZE(neon_groups); i++) { + const struct ubc_group4 *g = &neon_groups[i]; + uint32x4_t lo = vld1q_u32(w + g->lo); + uint32x4_t hi = vld1q_u32(w + g->hi); + uint32x4_t dvs = vld1q_u32(g->dvs); + uint32x4_t set, fail; + + lo = vshlq_u32(lo, vdupq_n_s32(-(int32_t)g->lo_shift)); + hi = vshlq_u32(hi, vdupq_n_s32(-(int32_t)g->hi_shift)); + set = vtstq_u32(veorq_u32(lo, hi), vld1q_u32(g->test)); + /* + * The DVs of the lanes where the bit is not g->want. Each lane + * of set is all ones or zero, so a saturating subtraction keeps + * dvs where the bit is clear, and min keeps it where it is set. + */ + fail = g->want ? vqsubq_u32(dvs, set) : vminq_u32(set, dvs); + acc = vorrq_u32(acc, fail); + } + + folded = vorr_u32(vget_low_u32(acc), vget_high_u32(acc)); + return ~vget_lane_u32(vorr_u32(folded, vdup_lane_u32(folded, 1)), 0); +} + +uint32_t sha1dc_ubc_check_neon(const uint32_t w[80]) +{ + uint32_t mask = neon_prefix(w); + /* Every check only clears bits, so an empty mask settles it. */ + if (!mask) + return 0; + return run_tail(w, mask, neon_tail_checks, neon_tail_spans, 0); +} + +#endif /* SHA1DC_HAVE_NEON */ + +/* sse2 form */ + +#ifdef SHA1DC_HAVE_SSE2 + +static const struct ubc_group4 sse2_groups[] = { + { 35, 0, 36, 5, 1, + { 1u << 1, 1u << 1, 1u << 1, 1u << 0 }, + { DV_I_46_2_BIT | DV_I_49_2_BIT, + DV_I_47_2_BIT | DV_I_50_2_BIT | DV_II_46_2_BIT, + DV_I_48_2_BIT | DV_I_51_2_BIT, + DV_II_51_2_BIT } }, + { 36, 0, 37, 5, 1, + { 1u << 0, 1u << 0, 1u << 1, 1u << 1 }, + { DV_II_49_2_BIT, + DV_II_50_2_BIT, + DV_I_49_2_BIT, + DV_I_46_2_BIT | DV_I_50_2_BIT | DV_II_49_2_BIT } }, + { 36, 0, 38, 0, 1, + { 1u << 4, 1u << 4, 1u << 1, 1u << 4 }, + { DV_II_52_0_BIT | DV_II_54_0_BIT, + DV_I_43_0_BIT | DV_II_53_0_BIT | DV_II_55_0_BIT, + DV_I_49_2_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_II_55_0_BIT } }, + { 37, 0, 40, 25, 0, + { 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_I_43_0_BIT | DV_I_47_0_BIT | DV_II_46_0_BIT | + DV_II_53_0_BIT | DV_II_55_0_BIT, + DV_I_44_0_BIT | DV_I_48_0_BIT | DV_II_47_0_BIT | + DV_II_54_0_BIT | DV_II_56_0_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_49_0_BIT | DV_II_48_0_BIT | + DV_II_55_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_50_0_BIT | DV_II_49_0_BIT | + DV_II_56_0_BIT } }, + { 36, 0, 40, 25, 0, + { 1u << 4, 1u << 5, 1u << 5, 1u << 5 }, + { DV_I_46_0_BIT | DV_I_49_0_BIT | DV_II_45_0_BIT | + DV_II_48_0_BIT, + DV_II_49_2_BIT, + DV_II_50_2_BIT, + DV_II_51_2_BIT } }, + { 38, 0, 40, 0, 1, + { 1u << 4, 1u << 1, 1u << 1, 1u << 1 }, + { DV_I_44_0_BIT | DV_II_54_0_BIT | DV_II_56_0_BIT, + DV_I_50_2_BIT | DV_II_49_2_BIT, + DV_I_51_2_BIT | DV_II_50_2_BIT, + DV_II_46_2_BIT | DV_II_51_2_BIT } }, + { 40, 0, 41, 5, 1, + { 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_I_47_2_BIT | DV_I_51_2_BIT | DV_II_50_2_BIT, + DV_I_48_2_BIT | DV_II_46_2_BIT | DV_II_51_2_BIT, + DV_I_49_2_BIT, + DV_I_50_2_BIT } }, + { 40, 0, 41, 0, 0, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_44_0_BIT | DV_I_47_0_BIT | DV_I_48_0_BIT | DV_II_46_0_BIT | + DV_II_47_0_BIT | DV_II_56_0_BIT, + DV_I_45_0_BIT | DV_I_48_0_BIT | DV_I_49_0_BIT | DV_II_47_0_BIT | + DV_II_48_0_BIT, + DV_I_46_0_BIT | DV_I_49_0_BIT | DV_I_50_0_BIT | DV_II_48_0_BIT | + DV_II_49_0_BIT, + DV_I_47_0_BIT | DV_I_50_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT | + DV_II_49_0_BIT | DV_II_50_0_BIT } }, + { 41, 0, 44, 25, 0, + { 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_51_0_BIT | + DV_II_45_0_BIT | DV_II_50_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_52_0_BIT | + DV_II_46_0_BIT | DV_II_51_0_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_49_0_BIT | + DV_II_47_0_BIT | DV_II_52_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_50_0_BIT | + DV_II_48_0_BIT | DV_II_53_0_BIT } }, + { 42, 0, 44, 0, 0, + { 1u << 6, 1u << 6, 1u << 6, 1u << 6 }, + { DV_I_46_2_BIT | DV_I_48_2_BIT, + DV_I_47_2_BIT | DV_I_49_2_BIT, + DV_I_46_2_BIT | DV_I_48_2_BIT | DV_I_50_2_BIT, + DV_I_47_2_BIT | DV_I_49_2_BIT | DV_I_51_2_BIT } }, + { 40, 0, 44, 0, 0, + { 1u << 6, 1u << 6, 1u << 29, 1u << 29 }, + { DV_I_46_2_BIT, + DV_I_47_2_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT } }, + { 44, 0, 45, 5, 1, + { 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_I_51_2_BIT | DV_II_49_2_BIT, + DV_II_50_2_BIT, + DV_II_51_2_BIT, + DV_II_46_2_BIT } }, + { 44, 0, 45, 0, 0, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_48_0_BIT | DV_I_51_0_BIT | DV_I_52_0_BIT | DV_II_45_0_BIT | + DV_II_46_0_BIT | DV_II_50_0_BIT | DV_II_51_0_BIT, + DV_I_49_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT | + DV_II_47_0_BIT | DV_II_51_0_BIT | DV_II_52_0_BIT, + DV_I_43_0_BIT | DV_I_50_0_BIT | DV_II_47_0_BIT | + DV_II_48_0_BIT | DV_II_52_0_BIT | DV_II_53_0_BIT, + DV_I_44_0_BIT | DV_I_51_0_BIT | DV_II_48_0_BIT | + DV_II_49_0_BIT | DV_II_53_0_BIT | DV_II_54_0_BIT } }, + { 45, 5, 46, 0, 0, + { 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_II_49_2_BIT, + DV_I_46_2_BIT | DV_II_50_2_BIT, + DV_I_47_2_BIT | DV_II_51_2_BIT, + DV_I_48_2_BIT } }, + { 45, 0, 47, 0, 1, + { 1u << 29, 1u << 29, 1u << 29, 1u << 4 }, + { DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_48_0_BIT, + DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_49_0_BIT, + DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_50_0_BIT, + DV_I_50_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT } }, + { 46, 0, 48, 0, 0, + { 1u << 6, 1u << 6, 1u << 6, 1u << 6 }, + { DV_I_48_2_BIT | DV_I_50_2_BIT, + DV_I_49_2_BIT | DV_I_51_2_BIT, + DV_I_50_2_BIT | DV_II_46_2_BIT, + DV_I_51_2_BIT } }, + { 45, 0, 48, 25, 0, + { 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_49_0_BIT | DV_I_51_0_BIT | + DV_II_49_0_BIT | DV_II_54_0_BIT, + DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_50_0_BIT | DV_I_52_0_BIT | + DV_II_50_0_BIT | DV_II_55_0_BIT, + DV_I_47_0_BIT | DV_I_49_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT | + DV_II_51_0_BIT | DV_II_56_0_BIT, + DV_I_48_0_BIT | DV_I_50_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT | + DV_II_52_0_BIT } }, + { 48, 0, 49, 0, 0, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_45_0_BIT | DV_I_52_0_BIT | DV_II_49_0_BIT | + DV_II_50_0_BIT | DV_II_54_0_BIT | DV_II_55_0_BIT, + DV_I_46_0_BIT | DV_II_45_0_BIT | DV_II_50_0_BIT | + DV_II_51_0_BIT | DV_II_55_0_BIT | DV_II_56_0_BIT, + DV_I_47_0_BIT | DV_II_46_0_BIT | DV_II_51_0_BIT | + DV_II_52_0_BIT | DV_II_56_0_BIT, + DV_I_48_0_BIT | DV_II_47_0_BIT | DV_II_52_0_BIT | + DV_II_53_0_BIT } }, + { 49, 5, 50, 0, 0, + { 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_I_49_2_BIT, + DV_I_50_2_BIT | DV_II_46_2_BIT, + DV_I_51_2_BIT, + 0 } }, + { 49, 0, 52, 25, 0, + { 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_I_49_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT | + DV_II_47_0_BIT | DV_II_53_0_BIT, + DV_I_50_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT | + DV_II_48_0_BIT | DV_II_54_0_BIT, + DV_I_51_0_BIT | DV_II_47_0_BIT | DV_II_49_0_BIT | + DV_II_55_0_BIT, + DV_I_52_0_BIT | DV_II_48_0_BIT | DV_II_50_0_BIT | + DV_II_56_0_BIT } }, + { 49, 0, 53, 0, 1, + { 1u << 29, 1u << 1, 1u << 1, 1u << 1 }, + { DV_II_45_0_BIT, + DV_II_49_2_BIT, + DV_II_50_2_BIT, + DV_II_51_2_BIT } }, + { 52, 0, 53, 0, 0, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_49_0_BIT | DV_II_45_0_BIT | DV_II_48_0_BIT | + DV_II_53_0_BIT | DV_II_54_0_BIT, + DV_I_50_0_BIT | DV_II_46_0_BIT | DV_II_49_0_BIT | + DV_II_54_0_BIT | DV_II_55_0_BIT, + DV_I_51_0_BIT | DV_II_47_0_BIT | DV_II_50_0_BIT | + DV_II_55_0_BIT | DV_II_56_0_BIT, + DV_I_52_0_BIT | DV_II_48_0_BIT | DV_II_51_0_BIT | + DV_II_56_0_BIT } }, + { 53, 5, 54, 0, 0, + { 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_II_49_2_BIT, + DV_II_50_2_BIT, + DV_II_51_2_BIT, + 0 } }, + { 52, 0, 55, 0, 1, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_51_0_BIT | DV_II_47_0_BIT | DV_II_48_0_BIT, + DV_I_52_0_BIT | DV_II_48_0_BIT | DV_II_49_0_BIT, + DV_II_49_0_BIT | DV_II_50_0_BIT, + DV_II_50_0_BIT | DV_II_51_0_BIT } }, + { 53, 0, 56, 25, 0, + { 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_II_49_0_BIT | DV_II_51_0_BIT, + DV_II_50_0_BIT | DV_II_52_0_BIT, + DV_II_51_0_BIT | DV_II_53_0_BIT, + DV_II_52_0_BIT | DV_II_54_0_BIT } }, + { 60, 0, 61, 5, 1, + { 1u << 0, 1u << 2, 1u << 2, 1u << 2 }, + { DV_I_45_0_BIT | DV_II_45_0_BIT, + DV_I_46_2_BIT | DV_II_46_2_BIT, + DV_I_47_2_BIT, + DV_I_48_2_BIT } } +}; + +static const struct ubc_cond sse2_tail_checks[] = { + /* DV_I_43_0 */ + { 41, 4, 43, 4, 1 }, + { 58, 0, 59, 5, 1 }, + { 58, 0, 63, 30, 1 }, + { 61, 1, 62, 6, 1 }, + { 43, 4, 47, 29, 0 }, + /* DV_I_44_0 */ + { 59, 0, 60, 5, 1 }, + { 59, 0, 64, 30, 1 }, + { 62, 1, 63, 6, 1 }, + { 38, 4, 42, 4, 0 }, + { 44, 4, 48, 29, 0 }, + /* DV_I_45_0 */ + { 41, 4, 43, 4, 1 }, + { 63, 1, 64, 6, 1 }, + { 35, 4, 39, 29, 0 }, + /* DV_I_46_0 */ + { 61, 0, 62, 5, 1 }, + /* DV_I_47_0 */ + { 41, 4, 43, 4, 1 }, + { 45, 4, 47, 4, 1 }, + { 62, 0, 63, 5, 1 }, + { 37, 4, 41, 29, 0 }, + /* DV_I_48_0 */ + { 46, 4, 48, 4, 1 }, + { 63, 0, 64, 5, 1 }, + { 35, 4, 39, 29, 0 }, + { 38, 4, 42, 29, 0 }, + /* DV_I_49_0 */ + { 45, 4, 47, 4, 1 }, + { 39, 4, 43, 29, 0 }, + { 45, 4, 49, 4, 0 }, + /* DV_I_50_0 */ + { 36, 4, 37, 4, 1 }, + { 46, 4, 48, 4, 1 }, + { 37, 4, 41, 29, 0 }, + { 40, 4, 44, 29, 0 }, + /* DV_I_51_0 */ + { 37, 4, 38, 4, 1 }, + { 45, 4, 47, 4, 1 }, + { 35, 3, 39, 28, 0 }, + { 38, 4, 42, 29, 0 }, + { 41, 4, 45, 29, 0 }, + { 45, 4, 49, 4, 0 }, + /* DV_I_51_2 */ + { 35, 5, 39, 30, 0 }, + { 37, 1, 37, 6, 0 }, + /* DV_I_52_0 */ + { 38, 4, 39, 4, 1 }, + { 46, 4, 48, 4, 1 }, + { 39, 4, 43, 29, 0 }, + { 42, 4, 46, 29, 0 }, + /* DV_II_45_0 */ + { 63, 1, 64, 6, 1 }, + { 41, 4, 45, 29, 0 }, + /* DV_II_46_0 */ + { 61, 0, 62, 5, 1 }, + { 37, 4, 41, 29, 0 }, + { 42, 4, 46, 29, 0 }, + /* DV_II_47_0 */ + { 62, 0, 63, 5, 1 }, + { 35, 3, 39, 28, 0 }, + { 35, 4, 39, 29, 0 }, + { 38, 4, 42, 29, 0 }, + { 43, 4, 47, 29, 0 }, + /* DV_II_48_0 */ + { 35, 30, 36, 3, 1 }, + { 35, 30, 40, 28, 1 }, + { 63, 0, 64, 5, 1 }, + { 39, 4, 43, 29, 0 }, + { 44, 4, 48, 29, 0 }, + /* DV_II_49_0 */ + { 36, 30, 37, 3, 1 }, + { 36, 30, 41, 28, 1 }, + { 37, 4, 41, 29, 0 }, + { 40, 4, 44, 29, 0 }, + /* DV_II_49_2 */ + { 50, 1, 51, 6, 1 }, + /* DV_II_50_0 */ + { 37, 30, 38, 3, 1 }, + { 37, 30, 42, 28, 1 }, + { 38, 4, 42, 29, 0 }, + { 41, 4, 45, 29, 0 }, + /* DV_II_50_2 */ + { 51, 1, 52, 6, 1 }, + /* DV_II_51_0 */ + { 38, 30, 39, 3, 1 }, + { 38, 30, 43, 28, 1 }, + { 56, 29, 59, 29, 1 }, + { 39, 4, 43, 29, 0 }, + { 42, 4, 46, 29, 0 }, + /* DV_II_51_2 */ + { 52, 1, 53, 6, 1 }, + /* DV_II_52_0 */ + { 39, 30, 40, 3, 1 }, + { 39, 30, 44, 28, 1 }, + { 54, 4, 56, 4, 1 }, + { 54, 4, 60, 29, 1 }, + { 56, 29, 59, 29, 1 }, + { 40, 4, 44, 29, 0 }, + { 43, 4, 47, 29, 0 }, + /* DV_II_53_0 */ + { 55, 4, 57, 4, 1 }, + { 55, 4, 61, 29, 1 }, + { 41, 3, 45, 28, 0 }, + { 41, 4, 45, 29, 0 }, + { 44, 4, 48, 29, 0 }, + { 55, 4, 57, 29, 0 }, + /* DV_II_54_0 */ + { 42, 3, 46, 28, 0 }, + { 42, 4, 46, 29, 0 }, + { 56, 4, 58, 29, 0 }, + { 58, 4, 62, 29, 0 }, + /* DV_II_55_0 */ + { 43, 3, 47, 28, 0 }, + { 43, 4, 47, 29, 0 }, + { 57, 4, 59, 29, 0 }, + { 59, 4, 63, 29, 0 }, + /* DV_II_56_0 */ + { 38, 4, 42, 4, 0 }, + { 44, 3, 48, 28, 0 }, + { 44, 4, 48, 29, 0 }, + { 60, 4, 64, 29, 0 } +}; + +static const struct tail_span sse2_tail_spans[32] = { + { 0, 5 }, /* DV_I_43_0 */ + { 5, 5 }, /* DV_I_44_0 */ + { 10, 3 }, /* DV_I_45_0 */ + { 13, 1 }, /* DV_I_46_0 */ + { 14, 0 }, /* DV_I_46_2 */ + { 14, 4 }, /* DV_I_47_0 */ + { 18, 0 }, /* DV_I_47_2 */ + { 18, 4 }, /* DV_I_48_0 */ + { 22, 0 }, /* DV_I_48_2 */ + { 22, 3 }, /* DV_I_49_0 */ + { 25, 0 }, /* DV_I_49_2 */ + { 25, 4 }, /* DV_I_50_0 */ + { 29, 0 }, /* DV_I_50_2 */ + { 29, 6 }, /* DV_I_51_0 */ + { 35, 2 }, /* DV_I_51_2 */ + { 37, 4 }, /* DV_I_52_0 */ + { 41, 2 }, /* DV_II_45_0 */ + { 43, 3 }, /* DV_II_46_0 */ + { 46, 0 }, /* DV_II_46_2 */ + { 46, 5 }, /* DV_II_47_0 */ + { 51, 5 }, /* DV_II_48_0 */ + { 56, 4 }, /* DV_II_49_0 */ + { 60, 1 }, /* DV_II_49_2 */ + { 61, 4 }, /* DV_II_50_0 */ + { 65, 1 }, /* DV_II_50_2 */ + { 66, 5 }, /* DV_II_51_0 */ + { 71, 1 }, /* DV_II_51_2 */ + { 72, 7 }, /* DV_II_52_0 */ + { 79, 6 }, /* DV_II_53_0 */ + { 85, 4 }, /* DV_II_54_0 */ + { 89, 4 }, /* DV_II_55_0 */ + { 93, 4 } /* DV_II_56_0 */ +}; + +SHA1DC_TARGET_SSE2 +static inline uint32_t sse2_prefix(const uint32_t *w, int mirrored) +{ + const __m128i zero = _mm_setzero_si128(); + __m128i acc = zero; + size_t i; + + UNROLL_TABLE + for (i = 0; i < ARRAY_SIZE(sse2_groups); i++) { + const struct ubc_group4 *g = &sse2_groups[i]; + const uint32_t *lo_w = w + (mirrored ? 76 - g->lo : g->lo); + const uint32_t *hi_w = w + (mirrored ? 76 - g->hi : g->hi); + __m128i lo = _mm_loadu_si128((const __m128i *)lo_w); + __m128i hi = _mm_loadu_si128((const __m128i *)hi_w); + __m128i test = mirrored ? + _mm_set_epi32(g->test[0], g->test[1], + g->test[2], g->test[3]) : + _mm_loadu_si128((const __m128i *)g->test); + __m128i dvs = mirrored ? + _mm_set_epi32(g->dvs[0], g->dvs[1], + g->dvs[2], g->dvs[3]) : + _mm_loadu_si128((const __m128i *)g->dvs); + __m128i clear, fail; + + lo = _mm_srl_epi32(lo, _mm_cvtsi32_si128(g->lo_shift)); + hi = _mm_srl_epi32(hi, _mm_cvtsi32_si128(g->hi_shift)); + clear = _mm_and_si128(_mm_xor_si128(lo, hi), test); + clear = _mm_cmpeq_epi32(clear, zero); + /* The DVs of the lanes where the bit is not g->want. */ + fail = g->want ? _mm_and_si128(clear, dvs) : + _mm_andnot_si128(clear, dvs); + acc = _mm_or_si128(acc, fail); + } + + acc = _mm_or_si128(acc, _mm_shuffle_epi32(acc, 0x4E)); + acc = _mm_or_si128(acc, _mm_shuffle_epi32(acc, 0xB1)); + return ~(uint32_t)_mm_cvtsi128_si32(acc); +} + +SHA1DC_TARGET_SSE2 +uint32_t sha1dc_ubc_check_sse2(const uint32_t w[80]) +{ + uint32_t mask = sse2_prefix(w, 0); + /* Every check only clears bits, so an empty mask settles it. */ + if (!mask) + return 0; + return run_tail(w, mask, sse2_tail_checks, sse2_tail_spans, 0); +} + +SHA1DC_TARGET_SSE2 +uint32_t sha1dc_ubc_check_sse2_mirrored(const uint32_t w[80]) +{ + uint32_t mask = sse2_prefix(w, 1); + if (!mask) + return 0; + return run_tail(w, mask, sse2_tail_checks, sse2_tail_spans, 1); +} + +#endif /* SHA1DC_HAVE_SSE2 */ + +/* avx2 form */ + +#ifdef SHA1DC_HAVE_AVX2 + +static const struct ubc_group8 avx2_groups[] = { + { 35, 0, 36, 5, 1, + { 1u << 1, 1u << 1, 1u << 1, 1u << 0, + 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_I_46_2_BIT | DV_I_49_2_BIT, + DV_I_47_2_BIT | DV_I_50_2_BIT | DV_II_46_2_BIT, + DV_I_48_2_BIT | DV_I_51_2_BIT, + DV_II_51_2_BIT, + DV_I_46_2_BIT | DV_I_50_2_BIT | DV_II_49_2_BIT, + DV_I_47_2_BIT | DV_I_51_2_BIT | DV_II_50_2_BIT, + DV_I_48_2_BIT | DV_II_46_2_BIT | DV_II_51_2_BIT, + DV_I_49_2_BIT } }, + { 36, 0, 38, 0, 1, + { 1u << 4, 1u << 4, 1u << 4, 1u << 4, + 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_II_52_0_BIT | DV_II_54_0_BIT, + DV_I_43_0_BIT | DV_II_53_0_BIT | DV_II_55_0_BIT, + DV_I_44_0_BIT | DV_II_54_0_BIT | DV_II_56_0_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_II_55_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_II_56_0_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_47_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_48_0_BIT, + DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_49_0_BIT } }, + { 37, 0, 38, 5, 1, + { 1u << 0, 1u << 1, 0, 0, + 0, 0, 1u << 1, 1u << 1 }, + { DV_II_50_2_BIT, + DV_I_49_2_BIT, + 0, + 0, + 0, + 0, + DV_I_50_2_BIT, + DV_I_51_2_BIT | DV_II_49_2_BIT } }, + { 35, 0, 39, 25, 0, + { 1u << 5, 1u << 4, 1u << 5, 1u << 5, + 1u << 5, 1u << 3, 1u << 3, 1u << 4 }, + { DV_I_51_2_BIT, + DV_I_46_0_BIT | DV_I_49_0_BIT | DV_II_45_0_BIT | + DV_II_48_0_BIT, + DV_II_49_2_BIT, + DV_II_50_2_BIT, + DV_II_51_2_BIT, + DV_II_52_0_BIT, + DV_II_53_0_BIT, + DV_I_52_0_BIT | DV_II_46_0_BIT | DV_II_51_0_BIT | + DV_II_54_0_BIT } }, + { 37, 0, 40, 25, 0, + { 1u << 4, 1u << 4, 1u << 4, 1u << 4, + 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_I_43_0_BIT | DV_I_47_0_BIT | DV_II_46_0_BIT | + DV_II_53_0_BIT | DV_II_55_0_BIT, + DV_I_44_0_BIT | DV_I_48_0_BIT | DV_II_47_0_BIT | + DV_II_54_0_BIT | DV_II_56_0_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_49_0_BIT | DV_II_48_0_BIT | + DV_II_55_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_50_0_BIT | DV_II_49_0_BIT | + DV_II_56_0_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_51_0_BIT | + DV_II_45_0_BIT | DV_II_50_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_52_0_BIT | + DV_II_46_0_BIT | DV_II_51_0_BIT, + DV_I_43_0_BIT | DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_49_0_BIT | + DV_II_47_0_BIT | DV_II_52_0_BIT, + DV_I_44_0_BIT | DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_50_0_BIT | + DV_II_48_0_BIT | DV_II_53_0_BIT } }, + { 39, 5, 40, 0, 0, + { 1u << 1, 1u << 1, 1u << 1, 1u << 1, + 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_I_49_2_BIT, + DV_I_50_2_BIT | DV_II_49_2_BIT, + DV_I_51_2_BIT | DV_II_50_2_BIT, + DV_II_46_2_BIT | DV_II_51_2_BIT, + 0, + 0, + DV_II_49_2_BIT, + DV_I_46_2_BIT | DV_II_50_2_BIT } }, + { 40, 0, 41, 0, 0, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29, + 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_44_0_BIT | DV_I_47_0_BIT | DV_I_48_0_BIT | DV_II_46_0_BIT | + DV_II_47_0_BIT | DV_II_56_0_BIT, + DV_I_45_0_BIT | DV_I_48_0_BIT | DV_I_49_0_BIT | DV_II_47_0_BIT | + DV_II_48_0_BIT, + DV_I_46_0_BIT | DV_I_49_0_BIT | DV_I_50_0_BIT | DV_II_48_0_BIT | + DV_II_49_0_BIT, + DV_I_47_0_BIT | DV_I_50_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT | + DV_II_49_0_BIT | DV_II_50_0_BIT, + DV_I_48_0_BIT | DV_I_51_0_BIT | DV_I_52_0_BIT | DV_II_45_0_BIT | + DV_II_46_0_BIT | DV_II_50_0_BIT | DV_II_51_0_BIT, + DV_I_49_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT | + DV_II_47_0_BIT | DV_II_51_0_BIT | DV_II_52_0_BIT, + DV_I_43_0_BIT | DV_I_50_0_BIT | DV_II_47_0_BIT | + DV_II_48_0_BIT | DV_II_52_0_BIT | DV_II_53_0_BIT, + DV_I_44_0_BIT | DV_I_51_0_BIT | DV_II_48_0_BIT | + DV_II_49_0_BIT | DV_II_53_0_BIT | DV_II_54_0_BIT } }, + { 40, 0, 42, 0, 0, + { 1u << 6, 1u << 6, 1u << 6, 1u << 6, + 1u << 6, 1u << 6, 1u << 6, 1u << 6 }, + { DV_I_46_2_BIT, + DV_I_47_2_BIT, + DV_I_46_2_BIT | DV_I_48_2_BIT, + DV_I_47_2_BIT | DV_I_49_2_BIT, + DV_I_46_2_BIT | DV_I_48_2_BIT | DV_I_50_2_BIT, + DV_I_47_2_BIT | DV_I_49_2_BIT | DV_I_51_2_BIT, + DV_I_48_2_BIT | DV_I_50_2_BIT, + DV_I_49_2_BIT | DV_I_51_2_BIT } }, + { 45, 0, 46, 5, 1, + { 1u << 1, 1u << 1, 1u << 1, 1u << 1, + 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_II_50_2_BIT, + DV_II_51_2_BIT, + DV_II_46_2_BIT, + 0, + 0, + DV_II_49_2_BIT, + DV_II_50_2_BIT, + DV_II_51_2_BIT } }, + { 45, 0, 48, 25, 0, + { 1u << 4, 1u << 4, 1u << 4, 1u << 4, + 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_I_45_0_BIT | DV_I_47_0_BIT | DV_I_49_0_BIT | DV_I_51_0_BIT | + DV_II_49_0_BIT | DV_II_54_0_BIT, + DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_50_0_BIT | DV_I_52_0_BIT | + DV_II_50_0_BIT | DV_II_55_0_BIT, + DV_I_47_0_BIT | DV_I_49_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT | + DV_II_51_0_BIT | DV_II_56_0_BIT, + DV_I_48_0_BIT | DV_I_50_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT | + DV_II_52_0_BIT, + DV_I_49_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT | + DV_II_47_0_BIT | DV_II_53_0_BIT, + DV_I_50_0_BIT | DV_I_52_0_BIT | DV_II_46_0_BIT | + DV_II_48_0_BIT | DV_II_54_0_BIT, + DV_I_51_0_BIT | DV_II_47_0_BIT | DV_II_49_0_BIT | + DV_II_55_0_BIT, + DV_I_52_0_BIT | DV_II_48_0_BIT | DV_II_50_0_BIT | + DV_II_56_0_BIT } }, + { 47, 5, 48, 0, 0, + { 1u << 1, 1u << 1, 1u << 1, 1u << 1, + 1u << 1, 1u << 1, 1u << 1, 1u << 1 }, + { DV_I_47_2_BIT | DV_II_51_2_BIT, + DV_I_48_2_BIT, + DV_I_49_2_BIT, + DV_I_50_2_BIT | DV_II_46_2_BIT, + DV_I_51_2_BIT, + 0, + DV_II_49_2_BIT, + DV_II_50_2_BIT } }, + { 47, 0, 49, 0, 1, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29, + 1u << 4, 1u << 4, 1u << 4, 1u << 4 }, + { DV_I_46_0_BIT | DV_I_48_0_BIT | DV_I_50_0_BIT, + DV_I_47_0_BIT | DV_I_49_0_BIT | DV_I_51_0_BIT, + DV_I_48_0_BIT | DV_I_50_0_BIT | DV_I_52_0_BIT, + DV_I_49_0_BIT | DV_I_51_0_BIT | DV_II_45_0_BIT, + DV_II_49_0_BIT, + DV_II_50_0_BIT, + DV_II_51_0_BIT, + DV_II_52_0_BIT } }, + { 48, 0, 49, 0, 0, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29, + 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_45_0_BIT | DV_I_52_0_BIT | DV_II_49_0_BIT | + DV_II_50_0_BIT | DV_II_54_0_BIT | DV_II_55_0_BIT, + DV_I_46_0_BIT | DV_II_45_0_BIT | DV_II_50_0_BIT | + DV_II_51_0_BIT | DV_II_55_0_BIT | DV_II_56_0_BIT, + DV_I_47_0_BIT | DV_II_46_0_BIT | DV_II_51_0_BIT | + DV_II_52_0_BIT | DV_II_56_0_BIT, + DV_I_48_0_BIT | DV_II_47_0_BIT | DV_II_52_0_BIT | + DV_II_53_0_BIT, + DV_I_49_0_BIT | DV_II_45_0_BIT | DV_II_48_0_BIT | + DV_II_53_0_BIT | DV_II_54_0_BIT, + DV_I_50_0_BIT | DV_II_46_0_BIT | DV_II_49_0_BIT | + DV_II_54_0_BIT | DV_II_55_0_BIT, + DV_I_51_0_BIT | DV_II_47_0_BIT | DV_II_50_0_BIT | + DV_II_55_0_BIT | DV_II_56_0_BIT, + DV_I_52_0_BIT | DV_II_48_0_BIT | DV_II_51_0_BIT | + DV_II_56_0_BIT } }, + { 48, 0, 50, 0, 0, + { 1u << 6, 1u << 6, 1u << 6, 1u << 6, + 1u << 6, 1u << 6, 1u << 6, 1u << 6 }, + { DV_I_50_2_BIT | DV_II_46_2_BIT, + DV_I_51_2_BIT, + 0, + DV_II_49_2_BIT, + DV_II_50_2_BIT, + DV_II_51_2_BIT, + 0, + 0 } }, + { 51, 0, 54, 0, 1, + { 1u << 29, 1u << 29, 1u << 29, 1u << 29, + 1u << 29, 1u << 29, 1u << 29, 1u << 29 }, + { DV_I_50_0_BIT | DV_II_46_0_BIT | DV_II_47_0_BIT, + DV_I_51_0_BIT | DV_II_47_0_BIT | DV_II_48_0_BIT, + DV_I_52_0_BIT | DV_II_48_0_BIT | DV_II_49_0_BIT, + DV_II_49_0_BIT | DV_II_50_0_BIT, + DV_II_50_0_BIT | DV_II_51_0_BIT, + DV_II_51_0_BIT | DV_II_52_0_BIT, + DV_II_52_0_BIT, + DV_II_53_0_BIT } }, + { 58, 0, 59, 5, 1, + { 1u << 0, 1u << 0, 1u << 0, 1u << 2, + 1u << 2, 1u << 2, 0, 0 }, + { DV_I_43_0_BIT, + DV_I_44_0_BIT, + DV_I_45_0_BIT | DV_II_45_0_BIT, + DV_I_46_2_BIT | DV_II_46_2_BIT, + DV_I_47_2_BIT, + DV_I_48_2_BIT, + 0, + 0 } } +}; + +static const struct ubc_cond avx2_tail_checks[] = { + /* DV_I_43_0 */ + { 58, 0, 63, 30, 1 }, + { 61, 1, 62, 6, 1 }, + { 43, 4, 47, 29, 0 }, + /* DV_I_44_0 */ + { 59, 0, 64, 30, 1 }, + { 62, 1, 63, 6, 1 }, + { 44, 4, 48, 29, 0 }, + /* DV_I_45_0 */ + { 63, 1, 64, 6, 1 }, + { 35, 4, 39, 29, 0 }, + /* DV_I_46_0 */ + { 61, 0, 62, 5, 1 }, + /* DV_I_47_0 */ + { 62, 0, 63, 5, 1 }, + { 37, 4, 41, 29, 0 }, + /* DV_I_48_0 */ + { 63, 0, 64, 5, 1 }, + { 35, 4, 39, 29, 0 }, + { 38, 4, 42, 29, 0 }, + /* DV_I_49_0 */ + { 39, 4, 43, 29, 0 }, + /* DV_I_50_0 */ + { 36, 4, 37, 4, 1 }, + { 37, 4, 41, 29, 0 }, + { 40, 4, 44, 29, 0 }, + { 46, 4, 50, 4, 0 }, + /* DV_I_51_0 */ + { 37, 4, 38, 4, 1 }, + { 35, 3, 39, 28, 0 }, + { 38, 4, 42, 29, 0 }, + { 41, 4, 45, 29, 0 }, + /* DV_I_51_2 */ + { 37, 1, 37, 6, 0 }, + /* DV_I_52_0 */ + { 38, 4, 39, 4, 1 }, + { 39, 4, 43, 29, 0 }, + { 46, 4, 50, 4, 0 }, + /* DV_II_45_0 */ + { 63, 1, 64, 6, 1 }, + { 41, 4, 45, 29, 0 }, + /* DV_II_46_0 */ + { 61, 0, 62, 5, 1 }, + { 37, 4, 41, 29, 0 }, + /* DV_II_47_0 */ + { 62, 0, 63, 5, 1 }, + { 35, 3, 39, 28, 0 }, + { 35, 4, 39, 29, 0 }, + { 38, 4, 42, 29, 0 }, + { 43, 4, 47, 29, 0 }, + /* DV_II_48_0 */ + { 35, 30, 36, 3, 1 }, + { 35, 30, 40, 28, 1 }, + { 63, 0, 64, 5, 1 }, + { 39, 4, 43, 29, 0 }, + { 44, 4, 48, 29, 0 }, + /* DV_II_49_0 */ + { 36, 30, 37, 3, 1 }, + { 36, 30, 41, 28, 1 }, + { 37, 4, 41, 29, 0 }, + { 40, 4, 44, 29, 0 }, + /* DV_II_49_2 */ + { 36, 0, 37, 5, 1 }, + /* DV_II_50_0 */ + { 37, 30, 38, 3, 1 }, + { 37, 30, 42, 28, 1 }, + { 38, 4, 42, 29, 0 }, + { 41, 4, 45, 29, 0 }, + { 54, 4, 57, 29, 0 }, + /* DV_II_51_0 */ + { 38, 30, 39, 3, 1 }, + { 38, 30, 43, 28, 1 }, + { 39, 4, 43, 29, 0 }, + { 55, 4, 58, 29, 0 }, + /* DV_II_51_2 */ + { 52, 1, 56, 1, 1 }, + /* DV_II_52_0 */ + { 39, 30, 40, 3, 1 }, + { 40, 4, 44, 29, 0 }, + { 43, 4, 47, 29, 0 }, + { 54, 4, 57, 29, 0 }, + { 56, 4, 59, 29, 0 }, + /* DV_II_53_0 */ + { 55, 4, 57, 4, 1 }, + { 41, 4, 45, 29, 0 }, + { 44, 4, 48, 29, 0 }, + { 55, 4, 58, 29, 0 }, + { 55, 4, 57, 29, 0 }, + /* DV_II_54_0 */ + { 42, 3, 46, 28, 0 }, + { 56, 4, 59, 29, 0 }, + { 56, 4, 58, 29, 0 }, + { 58, 4, 62, 29, 0 }, + /* DV_II_55_0 */ + { 43, 3, 47, 28, 0 }, + { 43, 4, 47, 29, 0 }, + { 57, 4, 59, 29, 0 }, + { 59, 4, 63, 29, 0 }, + /* DV_II_56_0 */ + { 44, 3, 48, 28, 0 }, + { 44, 4, 48, 29, 0 }, + { 60, 4, 64, 29, 0 } +}; + +static const struct tail_span avx2_tail_spans[32] = { + { 0, 3 }, /* DV_I_43_0 */ + { 3, 3 }, /* DV_I_44_0 */ + { 6, 2 }, /* DV_I_45_0 */ + { 8, 1 }, /* DV_I_46_0 */ + { 9, 0 }, /* DV_I_46_2 */ + { 9, 2 }, /* DV_I_47_0 */ + { 11, 0 }, /* DV_I_47_2 */ + { 11, 3 }, /* DV_I_48_0 */ + { 14, 0 }, /* DV_I_48_2 */ + { 14, 1 }, /* DV_I_49_0 */ + { 15, 0 }, /* DV_I_49_2 */ + { 15, 4 }, /* DV_I_50_0 */ + { 19, 0 }, /* DV_I_50_2 */ + { 19, 4 }, /* DV_I_51_0 */ + { 23, 1 }, /* DV_I_51_2 */ + { 24, 3 }, /* DV_I_52_0 */ + { 27, 2 }, /* DV_II_45_0 */ + { 29, 2 }, /* DV_II_46_0 */ + { 31, 0 }, /* DV_II_46_2 */ + { 31, 5 }, /* DV_II_47_0 */ + { 36, 5 }, /* DV_II_48_0 */ + { 41, 4 }, /* DV_II_49_0 */ + { 45, 1 }, /* DV_II_49_2 */ + { 46, 5 }, /* DV_II_50_0 */ + { 51, 0 }, /* DV_II_50_2 */ + { 51, 4 }, /* DV_II_51_0 */ + { 55, 1 }, /* DV_II_51_2 */ + { 56, 5 }, /* DV_II_52_0 */ + { 61, 5 }, /* DV_II_53_0 */ + { 66, 4 }, /* DV_II_54_0 */ + { 70, 4 }, /* DV_II_55_0 */ + { 74, 3 } /* DV_II_56_0 */ +}; + +SHA1DC_TARGET_AVX2 +static inline uint32_t avx2_prefix(const uint32_t *w, int mirrored) +{ + const __m256i zero = _mm256_setzero_si256(); + __m256i acc = zero; + __m128i half; + size_t i; + + UNROLL_TABLE + for (i = 0; i < ARRAY_SIZE(avx2_groups); i++) { + const struct ubc_group8 *g = &avx2_groups[i]; + const uint32_t *lo_w = w + (mirrored ? 72 - g->lo : g->lo); + const uint32_t *hi_w = w + (mirrored ? 72 - g->hi : g->hi); + __m256i lo = _mm256_loadu_si256((const __m256i *)lo_w); + __m256i hi = _mm256_loadu_si256((const __m256i *)hi_w); + __m256i test = mirrored ? + _mm256_set_epi32(g->test[0], g->test[1], g->test[2], + g->test[3], g->test[4], g->test[5], + g->test[6], g->test[7]) : + _mm256_loadu_si256((const __m256i *)g->test); + __m256i dvs = mirrored ? + _mm256_set_epi32(g->dvs[0], g->dvs[1], g->dvs[2], + g->dvs[3], g->dvs[4], g->dvs[5], + g->dvs[6], g->dvs[7]) : + _mm256_loadu_si256((const __m256i *)g->dvs); + __m256i clear, fail; + + lo = _mm256_srl_epi32(lo, _mm_cvtsi32_si128(g->lo_shift)); + hi = _mm256_srl_epi32(hi, _mm_cvtsi32_si128(g->hi_shift)); + clear = _mm256_and_si256(_mm256_xor_si256(lo, hi), test); + clear = _mm256_cmpeq_epi32(clear, zero); + /* The DVs of the lanes where the bit is not g->want. */ + fail = g->want ? _mm256_and_si256(clear, dvs) : + _mm256_andnot_si256(clear, dvs); + acc = _mm256_or_si256(acc, fail); + } + + half = _mm_or_si128(_mm256_castsi256_si128(acc), + _mm256_extracti128_si256(acc, 1)); + half = _mm_or_si128(half, _mm_shuffle_epi32(half, 0x4E)); + half = _mm_or_si128(half, _mm_shuffle_epi32(half, 0xB1)); + return ~(uint32_t)_mm_cvtsi128_si32(half); +} + +SHA1DC_TARGET_AVX2 +uint32_t sha1dc_ubc_check_avx2(const uint32_t w[80]) +{ + uint32_t mask = avx2_prefix(w, 0); + /* Every check only clears bits, so an empty mask settles it. */ + if (!mask) + return 0; + return run_tail(w, mask, avx2_tail_checks, avx2_tail_spans, 0); +} + +SHA1DC_TARGET_AVX2 +uint32_t sha1dc_ubc_check_avx2_mirrored(const uint32_t w[80]) +{ + uint32_t mask = avx2_prefix(w, 1); + if (!mask) + return 0; + return run_tail(w, mask, avx2_tail_checks, avx2_tail_spans, 1); +} + +#endif /* SHA1DC_HAVE_AVX2 */ diff --git a/sha1dc-accel/x86.c b/sha1dc-accel/x86.c new file mode 100644 index 00000000000000..dfdc1fc3b6206d --- /dev/null +++ b/sha1dc-accel/x86.c @@ -0,0 +1,260 @@ +/* + * The x86-64 parts of sha1.c: detecting what the CPU has, and the SHA-1 + * compression and recompression on SHA-NI. + */ + +#include "../git-compat-util.h" +#include "internal.h" + +#ifdef SHA1DC_HAVE_SSE2 + +#include + +/* Leaf 7 of CPUID, subleaf 0, if the CPU has it. */ +static int cpuid_7(unsigned int *ebx) +{ + unsigned int eax, ecx, edx; + + if (__get_cpuid_max(0, NULL) < 7) + return 0; + __cpuid_count(7, 0, eax, *ebx, ecx, edx); + return 1; +} + +int sha1dc_avx2_available(void) +{ + unsigned int eax, ebx, ecx, edx, xcr0_lo, xcr0_hi; + + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) + return 0; + /* OSXSAVE and AVX, then whether the OS saves the YMM registers. */ + if ((ecx & (1u << 27 | 1u << 28)) != (1u << 27 | 1u << 28)) + return 0; + __asm__("xgetbv" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); + if ((xcr0_lo & 6) != 6) + return 0; + return cpuid_7(&ebx) && (ebx & (1u << 5)); +} + +#ifdef SHA1DC_HAVE_SHANI + +int sha1dc_shani_available(void) +{ + unsigned int eax, ebx, ecx, edx; + + if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) + return 0; + /* SSSE3 and SSE4.1 */ + if ((ecx & (1u << 9 | 1u << 19)) != (1u << 9 | 1u << 19)) + return 0; + return cpuid_7(&ebx) && (ebx & (1u << 29)); +} + +/* + * `sha1rnds4` does four steps at a time, on `abcd` held with A in the top + * lane; `sha1nexte` derives the fifth working word for the next group from + * the previous `abcd`, which is why two registers alternate as `live` and + * `held`. The group of schedule words for steps t..t+3 is held reversed too. + * Words 16 to 31 come from `sha1msg1`/`sha1msg2`, the rest from plain SSE + * by the recurrence W[t] = (W[t-6] ^ W[t-16] ^ W[t-28] ^ W[t-32]) <<< 2, in + * which no word of a group depends on another. + */ + +/* Turns a group round, between step order and the order SHA-NI holds. */ +#define REVERSE 0x1B + +#define LOADU(p) _mm_loadu_si128((const __m128i *)(const void *)(p)) +#define STOREU(p, v) _mm_storeu_si128((__m128i *)(void *)(p), (v)) + +/* The group is already reversed; store it in the mirrored schedule. */ +#define SPILL(t, v) STOREU(w + 76 - (t), (v)) + +/* Schedule words 4k..4k+3 for k from 8 on, from groups k-8, k-7, k-4, k-2, k-1. */ +SHA1DC_TARGET_SHANI +static inline __m128i expand_rol2(__m128i v8, __m128i v7, __m128i v4, + __m128i v2, __m128i v1) +{ + __m128i x = _mm_xor_si128(_mm_xor_si128(v8, v7), v4); + /* Words t-6 to t-3: the last two of group k-2, the first two of k-1. */ + x = _mm_xor_si128(x, _mm_alignr_epi8(v2, v1, 8)); + return _mm_or_si128(_mm_slli_epi32(x, 2), _mm_srli_epi32(x, 30)); +} + +/* Schedule words 4k..4k+3 for k from 4 to 7, with the SHA-NI instructions. */ +#define EXPAND_NI(a, b, c, d) \ + _mm_sha1msg2_epu32(_mm_xor_si128(_mm_sha1msg1_epu32((a), (b)), (c)), (d)) + +/* One group of four steps on schedule group `t / 4`, spilling it first. */ +#define ROUNDS(t, k, live, held) \ + do { \ + SPILL(t, v[(t) / 4]); \ + live = _mm_sha1nexte_epu32(live, v[(t) / 4]); \ + held = abcd; \ + abcd = _mm_sha1rnds4_epu32(abcd, live, k); \ + } while (0) + +/* The same, also storing the state [A, B, C, D, E] before it in `at`. */ +#define ROUNDS_AT(t, k, live, held, at) \ + do { \ + SPILL(t, v[(t) / 4]); \ + live = _mm_sha1nexte_epu32(live, v[(t) / 4]); \ + STOREU(at, _mm_shuffle_epi32(abcd, REVERSE)); \ + /* `live` holds E + W[t] in its top lane. */ \ + at[4] = (uint32_t)_mm_extract_epi32(_mm_sub_epi32(live, v[(t) / 4]), 3); \ + held = abcd; \ + abcd = _mm_sha1rnds4_epu32(abcd, live, k); \ + } while (0) + +#define ROL2(k) v[k] = expand_rol2(v[(k) - 8], v[(k) - 7], v[(k) - 4], v[(k) - 2], v[(k) - 1]) + +SHA1DC_TARGET_SHANI +void sha1dc_compress_shani(uint32_t ihv[5], const unsigned char *block, + uint32_t w[80], uint32_t at_60[5], uint32_t at_64[5]) +{ + /* Big-endian words, and the four of a group reversed. */ + const __m128i swap = _mm_set_epi64x(0x0001020304050607LL, + 0x08090A0B0C0D0E0FLL); + __m128i abcd = _mm_shuffle_epi32(LOADU(ihv), REVERSE); + const __m128i abcd_in = abcd; + const __m128i e_in = _mm_set_epi32((int)ihv[4], 0, 0, 0); + __m128i e0, e1, v[20]; + + v[0] = _mm_shuffle_epi8(LOADU(block), swap); + v[1] = _mm_shuffle_epi8(LOADU(block + 16), swap); + v[2] = _mm_shuffle_epi8(LOADU(block + 32), swap); + v[3] = _mm_shuffle_epi8(LOADU(block + 48), swap); + + SPILL(0, v[0]); + e0 = _mm_add_epi32(e_in, v[0]); + e1 = abcd; + abcd = _mm_sha1rnds4_epu32(abcd, e0, 0); + v[4] = EXPAND_NI(v[0], v[1], v[2], v[3]); + + ROUNDS(4, 0, e1, e0); + v[5] = EXPAND_NI(v[1], v[2], v[3], v[4]); + ROUNDS(8, 0, e0, e1); + v[6] = EXPAND_NI(v[2], v[3], v[4], v[5]); + ROUNDS(12, 0, e1, e0); + v[7] = EXPAND_NI(v[3], v[4], v[5], v[6]); + ROUNDS(16, 0, e0, e1); + ROL2(8); + ROUNDS(20, 1, e1, e0); + ROL2(9); + ROUNDS(24, 1, e0, e1); + ROL2(10); + ROUNDS(28, 1, e1, e0); + ROL2(11); + ROUNDS(32, 1, e0, e1); + ROL2(12); + ROUNDS(36, 1, e1, e0); + ROL2(13); + ROUNDS(40, 2, e0, e1); + ROL2(14); + ROUNDS(44, 2, e1, e0); + ROL2(15); + ROUNDS(48, 2, e0, e1); + ROL2(16); + ROUNDS(52, 2, e1, e0); + ROL2(17); + ROUNDS(56, 2, e0, e1); + ROL2(18); + ROUNDS_AT(60, 3, e1, e0, at_60); + ROL2(19); + ROUNDS_AT(64, 3, e0, e1, at_64); + ROUNDS(68, 3, e1, e0); + ROUNDS(72, 3, e0, e1); + ROUNDS(76, 3, e1, e0); + + /* Feed-forward. */ + e0 = _mm_sha1nexte_epu32(e0, e_in); + abcd = _mm_add_epi32(abcd, abcd_in); + STOREU(ihv, _mm_shuffle_epi32(abcd, REVERSE)); + ihv[4] = (uint32_t)_mm_extract_epi32(e0, 3); +} + +/* + * The partner block's schedule words for group `g`, in the order the + * rounds take them. + */ +#define WORDS(g) \ + _mm_shuffle_epi32(_mm_xor_si128(LOADU(m1 + 4 * (g)), LOADU(dm + 4 * (g))), REVERSE) + +/* Four steps. The first of a run adds the fifth word itself. */ +#define GROUP_FIRST(e, g, k) \ + do { \ + __m128i live_ = _mm_add_epi32((e), WORDS(g)); \ + held = abcd; \ + abcd = _mm_sha1rnds4_epu32(abcd, live_, k); \ + } while (0) +#define GROUP(g, k) \ + do { \ + __m128i live_ = _mm_sha1nexte_epu32(held, WORDS(g)); \ + held = abcd; \ + abcd = _mm_sha1rnds4_epu32(abcd, live_, k); \ + } while (0) + +/* + * Whether the partner block, whose state at `from` is `state`, ends on + * `ihv_out`. From its state at step 60 or 64, it runs out to step 80, + * which gives the chaining value an attack would have had to start from, + * and then in from that value, which must arrive back at the same state. + */ +SHA1DC_TARGET_SHANI +int sha1dc_recompress_shani(enum sha1dc_from from, const uint32_t m1[80], + const uint32_t dm[80], const uint32_t state[5], + const uint32_t ihv_out[5]) +{ + uint32_t at[5], reached[5]; + __m128i abcd, held, e_at, e_80, e_in; + + sha1dc_partner_boundary(from, m1, dm, state, at); + abcd = _mm_shuffle_epi32(LOADU(at), REVERSE); + + /* Out to step 80, from 60 or from 64. */ + e_at = _mm_set_epi32((int)at[4], 0, 0, 0); + if (from == SHA1DC_FROM_58) { + GROUP_FIRST(e_at, 15, 3); + GROUP(16, 3); + } else { + GROUP_FIRST(e_at, 16, 3); + } + GROUP(17, 3); + GROUP(18, 3); + GROUP(19, 3); + + /* + * The feed-forward adds the input to the state at 80, so the only + * input that gives this block's output is the output less that state. + */ + e_80 = _mm_sha1nexte_epu32(held, _mm_setzero_si128()); + abcd = _mm_sub_epi32(_mm_shuffle_epi32(LOADU(ihv_out), REVERSE), abcd); + e_in = _mm_sub_epi32(_mm_set_epi32((int)ihv_out[4], 0, 0, 0), e_80); + + /* In from there, as far as the state the way out started from. */ + GROUP_FIRST(e_in, 0, 0); + GROUP(1, 0); + GROUP(2, 0); + GROUP(3, 0); + GROUP(4, 0); + GROUP(5, 1); + GROUP(6, 1); + GROUP(7, 1); + GROUP(8, 1); + GROUP(9, 1); + GROUP(10, 2); + GROUP(11, 2); + GROUP(12, 2); + GROUP(13, 2); + GROUP(14, 2); + if (from == SHA1DC_FROM_65) + GROUP(15, 3); + + STOREU(reached, _mm_shuffle_epi32(abcd, REVERSE)); + reached[4] = (uint32_t)_mm_extract_epi32( + _mm_sha1nexte_epu32(held, _mm_setzero_si128()), 3); + return !memcmp(reached, at, sizeof(at)); +} + +#endif /* SHA1DC_HAVE_SHANI */ + +#endif /* SHA1DC_HAVE_SSE2 */ diff --git a/sha1dc_git.c b/sha1dc_git.c index fe58d7962a30c9..2f20a25a03816a 100644 --- a/sha1dc_git.c +++ b/sha1dc_git.c @@ -1,6 +1,15 @@ +#define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" -#include "sha1dc_git.h" #include "hex.h" +#include "sha1dc_git.h" +#include "config.h" +#include "repository.h" +#include "thread-utils.h" + +#if !defined(DC_SHA1_EXTERNAL) && !defined(DC_SHA1_NO_ACCEL) +#include "sha1dc-accel/sha1.h" +#define USE_SHA1DC_ACCEL +#endif #ifdef DC_SHA1_EXTERNAL /* @@ -16,12 +25,13 @@ void git_SHA1DCInit(SHA1_CTX *ctx) /* * Same as SHA1DCFinal, but convert collision attack case into a verbose die(). */ -void git_SHA1DCFinal(unsigned char hash[20], SHA1_CTX *ctx) +void git_SHA1DCFinal(unsigned char hash[20], SHA1_CTX *ctx, + void (*die_fn)(const char *, ...)) { if (!SHA1DCFinal(hash, ctx)) return; - die("SHA-1 appears to be part of a collision attack: %s", - hash_to_hex_algop(hash, &hash_algos[GIT_HASH_SHA1])); + die_fn("SHA-1 appears to be part of a collision attack: %s", + hash_to_hex_algop(hash, &hash_algos[GIT_HASH_SHA1])); } /* @@ -37,3 +47,133 @@ void git_SHA1DCUpdate(SHA1_CTX *ctx, const void *vdata, size_t len) } SHA1DCUpdate(ctx, data, len); } + +static void sha1dc_c_clone(SHA1_CTX *dst, const SHA1_CTX *src) +{ + *dst = *src; +} + +static void sha1dc_c_discard(SHA1_CTX *ctx UNUSED) +{ + /* The C context owns no resources. */ +} + +#ifdef USE_SHA1DC_ACCEL +static void sha1dc_accel_git_final(unsigned char hash[20], SHA1_CTX *ctx, + void (*die_fn)(const char *, ...)) +{ + if (sha1dc_accel_final(hash, ctx)) + die_fn("SHA-1 appears to be part of a collision attack: %s", + hash_to_hex_algop(hash, &hash_algos[GIT_HASH_SHA1])); +} +#endif + +struct sha1dc_backend { + const char *name; + void (*init)(SHA1_CTX *); + void (*clone)(SHA1_CTX *, const SHA1_CTX *); + void (*update)(SHA1_CTX *, const void *, size_t); + void (*final)(unsigned char [20], SHA1_CTX *, + void (*die_fn)(const char *, ...)); + void (*discard)(SHA1_CTX *); +}; + +static const struct sha1dc_backend backends[] = { + { "c", git_SHA1DCInit, sha1dc_c_clone, git_SHA1DCUpdate, + git_SHA1DCFinal, sha1dc_c_discard }, +#ifdef USE_SHA1DC_ACCEL + { "accel", git_SHA1DCInit, sha1dc_c_clone, sha1dc_accel_update, + sha1dc_accel_git_final, sha1dc_c_discard }, +#endif +#ifdef DC_SHA1_RS + { "rust", sha1dc_rs_init, sha1dc_rs_clone, sha1dc_rs_update, + sha1dc_rs_final, sha1dc_rs_discard }, +#endif +}; + +static const struct sha1dc_backend *selected; + +static int sha1dc_config(const char *key, const char *value, + const struct config_context *ctx UNUSED, void *data) +{ + const struct sha1dc_backend **backend = data; + size_t i; + + if (strcmp(key, "core.sha1dcbackend")) + return 0; + if (!value) + return config_error_nonbool(key); + for (i = 0; i < ARRAY_SIZE(backends); i++) { + if (!strcasecmp(value, backends[i].name)) { + *backend = &backends[i]; + return 0; + } + } + die("unavailable or invalid SHA-1 backend: '%s'", value); +} + +static void sha1dc_choose(void) +{ + const struct sha1dc_backend *backend = &backends[0]; + + read_early_config(the_repository, sha1dc_config, &backend); +#ifdef DC_SHA1_RS + if (backend->init == sha1dc_rs_init && + !sha1dc_rs_ctx_fits(sizeof(SHA1_CTX), SHA1DC_CTX_ALIGNMENT)) + BUG("Rust SHA-1 context does not fit in SHA1_CTX"); +#endif +#ifdef __GNUC__ + __atomic_store_n(&selected, backend, __ATOMIC_RELAXED); +#else + selected = backend; +#endif +} + +static pthread_once_t once = PTHREAD_ONCE_INIT; + +static const struct sha1dc_backend *sha1dc_backend(void) +{ + int ret; +#ifdef __GNUC__ + const struct sha1dc_backend *backend = + __atomic_load_n(&selected, __ATOMIC_RELAXED); + + if (backend) + return backend; +#endif + ret = pthread_once(&once, sha1dc_choose); + if (ret) + die("cannot initialize SHA-1 backend: %s", strerror(ret)); + return selected; +} + +const char *sha1dc_backend_name(void) +{ + return sha1dc_backend()->name; +} + +void sha1dc_init(SHA1_CTX *ctx) +{ + sha1dc_backend()->init(ctx); +} + +void sha1dc_clone(SHA1_CTX *dst, const SHA1_CTX *src) +{ + sha1dc_backend()->clone(dst, src); +} + +void sha1dc_update(SHA1_CTX *ctx, const void *buf, size_t len) +{ + sha1dc_backend()->update(ctx, buf, len); +} + +void sha1dc_final(unsigned char hash[20], SHA1_CTX *ctx, + void (*die_fn)(const char *, ...)) +{ + sha1dc_backend()->final(hash, ctx, die_fn); +} + +void sha1dc_discard(SHA1_CTX *ctx) +{ + sha1dc_backend()->discard(ctx); +} diff --git a/sha1dc_git.h b/sha1dc_git.h index 0bcf1aa84b7241..555bd9b3619986 100644 --- a/sha1dc_git.h +++ b/sha1dc_git.h @@ -14,14 +14,26 @@ void git_SHA1DCInit(SHA1_CTX *); #define git_SHA1DCInit SHA1DCInit #endif -void git_SHA1DCFinal(unsigned char [20], SHA1_CTX *); +void git_SHA1DCFinal(unsigned char [20], SHA1_CTX *, + void (*die_fn)(const char *, ...)); void git_SHA1DCUpdate(SHA1_CTX *ctx, const void *data, size_t len); +const char *sha1dc_backend_name(void); +void sha1dc_init(SHA1_CTX *); +void sha1dc_clone(SHA1_CTX *, const SHA1_CTX *); +void sha1dc_update(SHA1_CTX *, const void *, size_t); +void sha1dc_final(unsigned char [20], SHA1_CTX *, + void (*die_fn)(const char *, ...)); +void sha1dc_discard(SHA1_CTX *); + #define platform_SHA_IS_SHA1DC /* used by "test-tool sha1-is-sha1dc" */ #ifndef platform_SHA_CTX #define platform_SHA_CTX SHA1_CTX -#define platform_SHA1_Init git_SHA1DCInit -#define platform_SHA1_Update git_SHA1DCUpdate -#define platform_SHA1_Final git_SHA1DCFinal +#define platform_SHA1_Init sha1dc_init +#define platform_SHA1_Update sha1dc_update +#define platform_SHA1_Final(hash, ctx) sha1dc_final((hash), (ctx), die) #endif +#define SHA1_NEEDS_CLONE_HELPER +#define platform_SHA1_Clone sha1dc_clone +#define platform_SHA1_Discard sha1dc_discard diff --git a/sha1dc_rs.h b/sha1dc_rs.h new file mode 100644 index 00000000000000..345150f36c3629 --- /dev/null +++ b/sha1dc_rs.h @@ -0,0 +1,26 @@ +#ifndef SHA1DC_RS_H +#define SHA1DC_RS_H + +#include "sha1dc_git.h" + +/* + * Like mem-pool.c, query alignment in C99 using a one-byte prefix. + * The compiler pads before "aligned" as required by SHA1_CTX; its offset + * gives the alignment without allocating a probe object. This avoids + * compiler-specific operators until C11's _Alignof is the baseline. + */ +struct sha1dc_ctx_alignment { + char unalign; + SHA1_CTX aligned; +}; +#define SHA1DC_CTX_ALIGNMENT offsetof(struct sha1dc_ctx_alignment, aligned) + +int sha1dc_rs_ctx_fits(size_t, size_t); +void sha1dc_rs_init(SHA1_CTX *); +void sha1dc_rs_clone(SHA1_CTX *, const SHA1_CTX *); +void sha1dc_rs_update(SHA1_CTX *, const void *, size_t); +void sha1dc_rs_final(unsigned char [20], SHA1_CTX *, + void (*die_fn)(const char *, ...)); +void sha1dc_rs_discard(SHA1_CTX *); + +#endif diff --git a/src/lib.rs b/src/lib.rs index 0c598298b1c6bf..a34f4d489c20cd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,6 @@ pub mod csum_file; pub mod hash; pub mod loose; +#[cfg(feature = "sha1dc-rs")] +mod sha1dc_rs; pub mod varint; diff --git a/src/sha1dc_rs.rs b/src/sha1dc_rs.rs new file mode 100644 index 00000000000000..6bdc41c7afae1e --- /dev/null +++ b/src/sha1dc_rs.rs @@ -0,0 +1,77 @@ +use sha1dc::Hasher; +use std::ffi::{c_void, CString}; +use std::mem::{align_of, size_of}; +use std::os::raw::c_char; +use std::{ptr, slice}; + +/// Check whether C's context storage can hold the Rust hasher. +#[no_mangle] +pub extern "C" fn sha1dc_rs_ctx_fits(size: usize, alignment: usize) -> i32 { + (size >= size_of::() && alignment != 0 && alignment % align_of::() == 0) as i32 +} + +/// Initialize a collision-detecting SHA-1 context. +/// +/// # Safety +/// `ctx` must point to uninitialized storage whose size and alignment have +/// been accepted by `sha1dc_rs_ctx_fits`. +#[no_mangle] +pub unsafe extern "C" fn sha1dc_rs_init(ctx: *mut c_void) { + ctx.cast::().write(Hasher::new()); +} + +/// Replace a SHA-1 context with a clone of another. +/// +/// # Safety +/// Both contexts must be initialized. +#[no_mangle] +pub unsafe extern "C" fn sha1dc_rs_clone(dst: *mut c_void, src: *const c_void) { + *dst.cast::() = (*src.cast::()).clone(); +} + +/// Update the SHA-1 hasher with the given bytes. +/// +/// # Safety +/// `ctx` must be initialized and `data` must point to `len` bytes unless +/// `len` is zero. +#[no_mangle] +pub unsafe extern "C" fn sha1dc_rs_update(ctx: *mut c_void, data: *const c_void, len: usize) { + let ctx = ctx.cast::(); + if len != 0 { + (*ctx).update(slice::from_raw_parts(data.cast::(), len)); + } +} + +/// Finalize SHA-1, reporting detected collisions through `die`. +/// +/// # Safety +/// `ctx` must be initialized, `hash` must point to at least 20 bytes, and +/// `die` must be a non-returning C variadic function. +#[no_mangle] +pub unsafe extern "C" fn sha1dc_rs_final( + hash: *mut u8, + ctx: *mut c_void, + die: unsafe extern "C" fn(*const c_char, ...) -> !, +) { + let hasher = ptr::read(ctx.cast::()); + match hasher.finalize() { + Ok(digest) => ptr::copy_nonoverlapping(digest.as_bytes().as_ptr(), hash, 20), + Err(collision) => { + let message = CString::new(format!( + "SHA-1 appears to be part of a collision attack: {}", + collision.digest() + )) + .expect("collision message contains no NUL"); + die(message.as_ptr()); + } + } +} + +/// Discard a SHA-1 context without producing a digest. +/// +/// # Safety +/// `ctx` must be initialized. +#[no_mangle] +pub unsafe extern "C" fn sha1dc_rs_discard(ctx: *mut c_void) { + ptr::drop_in_place(ctx.cast::()); +} diff --git a/t/.gitattributes b/t/.gitattributes index e867f38c71d69e..f9437159aeca8a 100644 --- a/t/.gitattributes +++ b/t/.gitattributes @@ -2,6 +2,7 @@ t[0-9][0-9][0-9][0-9]/* -whitespace /chainlint/*.expect eol=lf -whitespace /greplint/*.expect eol=lf -whitespace /greplint/*.test eol=lf -whitespace +/t0013/*.bin binary /t0110/url-* binary /t3206/* eol=lf /t3900/*.txt eol=lf diff --git a/t/helper/test-sha1.c b/t/helper/test-sha1.c index 349540c4df8b6a..a7e2aab9322a0f 100644 --- a/t/helper/test-sha1.c +++ b/t/helper/test-sha1.c @@ -1,19 +1,134 @@ +#define USE_THE_REPOSITORY_VARIABLE #include "test-tool.h" #include "hash.h" +#include "config.h" +#include "setup.h" +#include "strbuf.h" + +#if defined(SHA1_DC) && !defined(DC_SHA1_EXTERNAL) && !defined(DC_SHA1_NO_ACCEL) +#include "sha1dc-accel/sha1.h" +#define HAVE_SHA1DC_ACCEL +#endif int cmd__sha1(int ac, const char **av) { +#ifdef HAVE_SHA1DC_ACCEL + const char *backend = getenv("GIT_TEST_SHA1DC_BACKEND"); + + if (backend && sha1dc_accel_select(backend)) + die("sha1dc backend '%s' is not available", backend); + if (backend) + git_config_push_parameter("core.sha1dcBackend=accel"); +#endif return cmd_hash_impl(ac, av, GIT_HASH_SHA1, 0); } -int cmd__sha1_is_sha1dc(int argc UNUSED, const char **argv UNUSED) +int cmd__sha1_is_sha1dc(int argc, const char **argv) { +#ifdef SHA1_DC + if (argc == 2 && !strcmp(argv[1], "--backend")) { + git_SHA_CTX ctx; + int nongit; + + setup_git_directory_gently(the_repository, &nongit); + git_SHA1_Init(&ctx); + puts(sha1dc_backend_name()); + git_SHA1_Discard(&ctx); + return 0; + } +#else + if (argc == 2 && !strcmp(argv[1], "--backend")) + return 1; +#endif #ifdef platform_SHA_IS_SHA1DC return 0; #endif return 1; } +/* + * Lists the implementations of collision-detecting SHA-1 this build and + * CPU can use, which GIT_TEST_SHA1DC_BACKEND selects for "test-tool sha1". + */ +int cmd__sha1dc_backends(int argc UNUSED, const char **argv UNUSED) +{ +#ifdef HAVE_SHA1DC_ACCEL + const char *const *names; + + for (names = sha1dc_accel_backends(); *names; names++) + puts(*names); +#endif + return 0; +} + +/* + * Hashes a file with sha1dc/ and with sha1dc-accel/ (the backend that + * GIT_TEST_SHA1DC_BACKEND selects), under every combination of settings, + * and dies if they ever disagree on the digest or on whether there is a + * collision. sha1dc-accel/ sees the file split at and around every block + * boundary, with an empty update in between. Prints what sha1dc/ found for + * each combination. + */ +int cmd__sha1dc_compare(int argc MAYBE_UNUSED, const char **argv MAYBE_UNUSED) +{ +#ifdef HAVE_SHA1DC_ACCEL + const char *backend = getenv("GIT_TEST_SHA1DC_BACKEND"); + struct strbuf buf = STRBUF_INIT; + int mode; + + if (argc != 2) + die("usage: test-tool sha1dc-compare "); + if (backend && sha1dc_accel_select(backend)) + die("sha1dc backend '%s' is not available", backend); + if (strbuf_read_file(&buf, argv[1], 0) < 0) + die_errno("could not read '%s'", argv[1]); + + for (mode = 0; mode < 16; mode++) { + int detect = !!(mode & 8), safe = !!(mode & 4); + int ubc = !!(mode & 2), reduced = !!(mode & 1); + unsigned char want[20], got[20]; + SHA1_CTX init, ctx; + int want_coll, got_coll, i; + size_t split; + + SHA1DCInit(&init); + SHA1DCSetUseDetectColl(&init, detect); + SHA1DCSetSafeHash(&init, safe); + SHA1DCSetUseUBC(&init, ubc); + SHA1DCSetDetectReducedRoundCollision(&init, reduced); + + ctx = init; + SHA1DCUpdate(&ctx, buf.buf, buf.len); + want_coll = SHA1DCFinal(want, &ctx); + + for (split = 0; split <= buf.len; split++) { + if (split % 64 > 1 && split % 64 < 63 && split != buf.len) + continue; + ctx = init; + sha1dc_accel_update(&ctx, buf.buf, split); + sha1dc_accel_update(&ctx, buf.buf + split, 0); + sha1dc_accel_update(&ctx, buf.buf + split, buf.len - split); + got_coll = sha1dc_accel_final(got, &ctx); + if (got_coll != want_coll || memcmp(got, want, sizeof(got))) + die("%s disagrees with sha1dc/ on '%s' with detect=%d " + "safe=%d ubc=%d reduced=%d, split at %"PRIuMAX, + sha1dc_accel_backend(), argv[1], detect, safe, + ubc, reduced, (uintmax_t)split); + } + + printf("detect=%d safe=%d ubc=%d reduced=%d collision=%d ", + detect, safe, ubc, reduced, want_coll); + for (i = 0; i < 20; i++) + printf("%02x", want[i]); + putchar('\n'); + } + strbuf_release(&buf); + return 0; +#else + die("test-tool sha1dc-compare: not built with sha1dc-accel"); +#endif +} + int cmd__sha1_unsafe(int ac, const char **av) { return cmd_hash_impl(ac, av, GIT_HASH_SHA1, 1); diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index ee16b2cb23719e..1e5c418348587e 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -74,6 +74,8 @@ static struct test_cmd cmds[] = { { "serve-v2", cmd__serve_v2 }, { "sha1", cmd__sha1 }, { "sha1-is-sha1dc", cmd__sha1_is_sha1dc }, + { "sha1dc-backends", cmd__sha1dc_backends }, + { "sha1dc-compare", cmd__sha1dc_compare }, { "sha1-unsafe", cmd__sha1_unsafe }, { "sha256", cmd__sha256 }, { "sigchain", cmd__sigchain }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index 4cf9f935a4cdfa..b5d716a6baa00b 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -67,6 +67,8 @@ int cmd__scrap_cache_tree(int argc, const char **argv); int cmd__serve_v2(int argc, const char **argv); int cmd__sha1(int argc, const char **argv); int cmd__sha1_is_sha1dc(int argc, const char **argv); +int cmd__sha1dc_backends(int argc, const char **argv); +int cmd__sha1dc_compare(int argc, const char **argv); int cmd__sha1_unsafe(int argc, const char **argv); int cmd__sha256(int argc, const char **argv); int cmd__sigchain(int argc, const char **argv); diff --git a/t/meson.build b/t/meson.build index 729de1177e76af..1d9fc8274af697 100644 --- a/t/meson.build +++ b/t/meson.build @@ -22,6 +22,7 @@ clar_test_suites = [ 'unit-tests/u-reftable-stack.c', 'unit-tests/u-reftable-table.c', 'unit-tests/u-reftable-tree.c', + 'unit-tests/u-sha1dc.c', 'unit-tests/u-strbuf.c', 'unit-tests/u-strcmp-offset.c', 'unit-tests/u-string-list.c', diff --git a/t/t0013-sha1dc.sh b/t/t0013-sha1dc.sh index 3ea3169d92ff7c..bcad82a4c3af6d 100755 --- a/t/t0013-sha1dc.sh +++ b/t/t0013-sha1dc.sh @@ -13,10 +13,110 @@ then test_done fi +test_lazy_prereq SHA1DC_RS ' + test rust = "$(GIT_CONFIG_PARAMETERS="${SQ}core.sha1dcBackend=rust${SQ}" \ + test-tool sha1-is-sha1dc --backend)" +' + +test_lazy_prereq SHA1DC_ACCEL ' + test -n "$(test-tool sha1dc-backends)" +' + +test_expect_success 'original C SHA1DC remains the default' ' + test c = "$(test-tool sha1-is-sha1dc --backend)" +' + test_expect_success 'test-sha1 detects shattered pdf' ' test_must_fail test-tool sha1 <"$TEST_DATA/shattered-1.pdf" 2>err && test_grep collision err && - test_grep 38762cf7f55934b34d179ae6a4c80cadccbb7f0a err + test_grep 38762cf7f55934b34d179ae6a4c80cadccbb7f0a err && + if test_have_prereq SHA1DC_RS + then + test_must_fail env \ + GIT_CONFIG_PARAMETERS="${SQ}core.sha1dcBackend=rust${SQ}" \ + test-tool sha1 <"$TEST_DATA/shattered-1.pdf" 2>err && + test_grep collision err && + test_grep 38762cf7f55934b34d179ae6a4c80cadccbb7f0a err + fi ' +test_expect_success SHA1DC_RS 'select SHA1DC backend via config' ' + test_config core.sha1dcBackend rust && + test rust = "$(test-tool sha1-is-sha1dc --backend)" && + test_config core.sha1dcBackend c && + test c = "$(test-tool sha1-is-sha1dc --backend)" +' + +test_expect_success !SHA1DC_RS 'reject uncompiled Rust SHA1DC backend' ' + test_must_fail env \ + GIT_CONFIG_PARAMETERS="${SQ}core.sha1dcBackend=rust${SQ}" \ + test-tool sha1-is-sha1dc --backend 2>err && + test_grep "unavailable or invalid SHA-1 backend" err +' + +test_expect_success SHA1DC_ACCEL 'select accelerated C via config' ' + test_config core.sha1dcBackend accel && + test accel = "$(test-tool sha1-is-sha1dc --backend)" +' + +test_expect_success 'reject invalid SHA1DC backend' ' + test_must_fail env \ + GIT_CONFIG_PARAMETERS="${SQ}core.sha1dcBackend=invalid${SQ}" \ + test-tool sha1-is-sha1dc --backend 2>err && + test_grep "unavailable or invalid SHA-1 backend" err +' + +test_expect_success 'test-sha1 detects SHA-mbles chosen-prefix collision' ' + test_must_fail test-tool sha1 <"$TEST_DATA/sha-mbles-1.bin" 2>err && + test_grep collision err && + test_grep 8ac60ba76f1999a1ab70223f225aefdc78d4ddc0 err +' + +# Each implementation of the detection this build and CPU can use. +for backend in $(test-tool sha1dc-backends) +do + test_expect_success "$backend: detects collisions" ' + test_must_fail env \ + GIT_CONFIG_PARAMETERS="${SQ}core.sha1dcBackend=c${SQ}" \ + GIT_TEST_SHA1DC_BACKEND=$backend \ + test-tool sha1 <"$TEST_DATA/shattered-1.pdf" 2>err && + test_grep 38762cf7f55934b34d179ae6a4c80cadccbb7f0a err && + test_must_fail env \ + GIT_CONFIG_PARAMETERS="${SQ}core.sha1dcBackend=c${SQ}" \ + GIT_TEST_SHA1DC_BACKEND=$backend \ + test-tool sha1 <"$TEST_DATA/sha-mbles-1.bin" 2>err && + test_grep 8ac60ba76f1999a1ab70223f225aefdc78d4ddc0 err + ' + + # In every combination of settings, and with the input split around + # each block, it must find what sha1dc/ finds and give its digest. + test_expect_success "$backend: agrees with sha1dc/ on collisions" ' + test_copy_bytes 320 <"$TEST_DATA/shattered-1.pdf" >shattered && + for f in shattered "$TEST_DATA/sha-mbles-1.bin" + do + GIT_TEST_SHA1DC_BACKEND=$backend \ + test-tool sha1dc-compare "$f" >out && + test_grep ! "detect=1 .* collision=0" out && + test_grep ! "detect=0 .* collision=1" out || return 1 + done && + + # A reduced-round collision counts only when asked for. + GIT_TEST_SHA1DC_BACKEND=$backend test-tool sha1dc-compare \ + "$TEST_DATA/sha1-reduced-round.bin" >out && + grep "collision=1" out >actual && + grep "detect=1 .* reduced=1 collision=1" out >expect && + test_line_count = 4 expect && + test_cmp expect actual + ' + + test_expect_success "$backend: hashes like the others" ' + test-tool genrandom "$backend" 100000 >data && + GIT_CONFIG_PARAMETERS="${SQ}core.sha1dcBackend=c${SQ}" \ + GIT_TEST_SHA1DC_BACKEND=$backend \ + test-tool sha1 actual && + test-tool sha1 expect && + test_cmp expect actual + ' +done + test_done diff --git a/t/t0013/sha-mbles-1.bin b/t/t0013/sha-mbles-1.bin new file mode 100644 index 00000000000000..5a7c30e97646c6 Binary files /dev/null and b/t/t0013/sha-mbles-1.bin differ diff --git a/t/t0013/sha1-reduced-round.bin b/t/t0013/sha1-reduced-round.bin new file mode 100644 index 00000000000000..4623336222bd5c --- /dev/null +++ b/t/t0013/sha1-reduced-round.bin @@ -0,0 +1 @@ +~9:pऄޥVZ-kqpDx$ . (8<+sC>NM]{$,+0ThEC !R6+f^j@,9|<r \ No newline at end of file diff --git a/t/unit-tests/u-hash.c b/t/unit-tests/u-hash.c index 19f4efd410832c..edbf7dbf3e4bd5 100644 --- a/t/unit-tests/u-hash.c +++ b/t/unit-tests/u-hash.c @@ -53,6 +53,38 @@ void test_hash__multi_character(void) "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } +void test_hash__clone_and_discard_sha1(void) +{ + struct git_hash_ctx original, clone; + unsigned char hash[GIT_SHA1_RAWSZ]; + + git_hash_init(&original, &hash_algos[GIT_HASH_SHA1]); + git_hash_update(&original, "ab", 2); + git_hash_init(&clone, &hash_algos[GIT_HASH_SHA1]); + git_hash_update(&clone, "discarded", 9); + git_hash_clone(&clone, &original); + git_hash_clone(&clone, &clone); + git_hash_discard(&original); + git_hash_update(&clone, "c", 1); + git_hash_final(hash, &clone); + cl_assert_equal_s(hash_to_hex_algop(hash, &hash_algos[GIT_HASH_SHA1]), + "a9993e364706816aba3e25717850c26c9cd0d89d"); +} + +void test_hash__rust_context_storage(void) +{ +#ifdef DC_SHA1_RS + size_t alignment = SHA1DC_CTX_ALIGNMENT; + + cl_assert(sha1dc_rs_ctx_fits(sizeof(SHA1_CTX), alignment)); + cl_assert(!sha1dc_rs_ctx_fits(0, alignment)); + cl_assert(!sha1dc_rs_ctx_fits(sizeof(SHA1_CTX), 0)); + cl_assert(!sha1dc_rs_ctx_fits(sizeof(SHA1_CTX), alignment - 1)); +#else + cl_skip(); +#endif +} + void test_hash__message_digest(void) { TEST_HASH_STR("message digest", diff --git a/t/unit-tests/u-sha1dc.c b/t/unit-tests/u-sha1dc.c new file mode 100644 index 00000000000000..c658c219a3a88c --- /dev/null +++ b/t/unit-tests/u-sha1dc.c @@ -0,0 +1,384 @@ +#include "unit-test.h" +#include "hash.h" + +/* + * Tests sha1dc-accel/ against sha1dc/, which it must agree with exactly, + * and its parts against the plain SHA-1 step function written out here. + */ +#if defined(SHA1_DC) && !defined(DC_SHA1_EXTERNAL) && !defined(DC_SHA1_NO_ACCEL) +#define HAVE_SHA1DC_ACCEL + +#if defined(DC_SHA1_SUBMODULE) +#include "sha1collisiondetection/lib/ubc_check.h" +#else +#include "sha1dc/ubc_check.h" +#endif +#include "sha1dc-accel/sha1.h" +#include "sha1dc-accel/internal.h" + +static uint64_t rng_state; + +static void rng_seed(uint64_t seed) +{ + rng_state = seed * 2 + 1; +} + +static uint32_t rng(void) +{ + /* xorshift64* */ + rng_state ^= rng_state >> 12; + rng_state ^= rng_state << 25; + rng_state ^= rng_state >> 27; + return (uint32_t)((rng_state * 0x2545F4914F6CDD1DULL) >> 32); +} + +/* + * Hashes `buf` with sha1dc/ and with sha1dc-accel/ under the same settings + * and checks that both agree. Returns whether they found a collision. + */ +static int check_same(const unsigned char *buf, size_t len, int split, + int safe_hash, int ubc, int reduced_round) +{ + SHA1_CTX want_ctx, got_ctx; + unsigned char want[20], got[20]; + int want_coll, got_coll; + + SHA1DCInit(&want_ctx); + SHA1DCSetSafeHash(&want_ctx, safe_hash); + SHA1DCSetUseUBC(&want_ctx, ubc); + SHA1DCSetDetectReducedRoundCollision(&want_ctx, reduced_round); + got_ctx = want_ctx; + + SHA1DCUpdate(&want_ctx, (const char *)buf, len); + want_coll = SHA1DCFinal(want, &want_ctx); + + sha1dc_accel_update(&got_ctx, buf, split); + sha1dc_accel_update(&got_ctx, buf + split, len - split); + got_coll = sha1dc_accel_final(got, &got_ctx); + + cl_assert_equal_i(got_coll, want_coll); + cl_assert(!memcmp(got, want, sizeof(want))); + return got_coll; +} + +/* + * sha1dc-accel/ relies on sha1dc/'s table of DVs having the 32 DVs below, + * with DV n at bit n of the UBC mask, and each recompressing from a state + * it saves (step 58 or 65). A change there must fail here, rather than + * quietly check the wrong DV or start from the wrong state. + */ +static void dv_table_is_as_expected(void) +{ + static const struct { int type, k, b; } want[] = { + { 1, 43, 0 }, { 1, 44, 0 }, { 1, 45, 0 }, { 1, 46, 0 }, + { 1, 46, 2 }, { 1, 47, 0 }, { 1, 47, 2 }, { 1, 48, 0 }, + { 1, 48, 2 }, { 1, 49, 0 }, { 1, 49, 2 }, { 1, 50, 0 }, + { 1, 50, 2 }, { 1, 51, 0 }, { 1, 51, 2 }, { 1, 52, 0 }, + { 2, 45, 0 }, { 2, 46, 0 }, { 2, 46, 2 }, { 2, 47, 0 }, + { 2, 48, 0 }, { 2, 49, 0 }, { 2, 49, 2 }, { 2, 50, 0 }, + { 2, 50, 2 }, { 2, 51, 0 }, { 2, 51, 2 }, { 2, 52, 0 }, + { 2, 53, 0 }, { 2, 54, 0 }, { 2, 55, 0 }, { 2, 56, 0 }, + }; + int i; + + for (i = 0; sha1_dvs[i].dvType != 0; i++) { + const dv_info_t *dv = &sha1_dvs[i]; + + cl_assert(i < (int)ARRAY_SIZE(want)); + cl_assert_equal_i(dv->dvType, want[i].type); + cl_assert_equal_i(dv->dvK, want[i].k); + cl_assert_equal_i(dv->dvB, want[i].b); + cl_assert_equal_i(dv->maski, 0); + cl_assert_equal_i(dv->maskb, i); + cl_assert(dv->testt == SHA1DC_FROM_58 || dv->testt == SHA1DC_FROM_65); + } + cl_assert_equal_i(i, ARRAY_SIZE(want)); +} + +static void every_backend_agrees_with_sha1dc(void) +{ + const char *const *names = sha1dc_accel_backends(); + unsigned char buf[4200]; + const char *orig = sha1dc_accel_backend(); + + for (; *names; names++) { + size_t i; + + cl_assert_equal_i(sha1dc_accel_select(*names), 0); + rng_seed(1); + for (i = 0; i < sizeof(buf); i++) + buf[i] = rng(); + + for (i = 0; i < 600; i++) { + size_t len = i < 200 ? i : rng() % (sizeof(buf) - 16); + size_t off = rng() % 16; + size_t split = len ? rng() % (len + 1) : 0; + /* + * Without the filter, every block is recompressed + * for all 32 DVs, which must not find anything in + * random data either. + */ + int ubc = i % 5 != 4; + + cl_assert_equal_i(check_same(buf + off, len, split, + i & 1, ubc, i % 3 == 0), 0); + } + } + cl_assert_equal_i(sha1dc_accel_select(orig), 0); +} + +/* Asserts only on a mismatch; clar's assertions are too slow to run 10^5 times. */ +#define check_form(got, want) do { \ + uint32_t got_ = (got); \ + if (got_ != (want)) \ + cl_assert_equal_i(got_, (want)); \ + } while (0) + +/* + * Checks every form of the UBC check against sha1dc/'s on `w`, and next to + * it, with each bit flipped in turn if `flip`. + */ +static void check_ubc_forms(uint32_t w[80], int flip, int avx2) +{ + int k; + + for (k = 0; k <= (flip ? 80 * 32 : 0); k++) { + uint32_t want; + + if (k) + w[(k - 1) / 32] ^= 1u << ((k - 1) % 32); + ubc_check(w, &want); + check_form(sha1dc_ubc_check_scalar(w), want); +#ifdef SHA1DC_HAVE_SSE2 + check_form(sha1dc_ubc_check_sse2(w), want); + { + uint32_t mirrored[80]; + int t; + + for (t = 0; t < 80; t++) + mirrored[t] = w[79 - t]; + check_form(sha1dc_ubc_check_sse2_mirrored(mirrored), + want); +#ifdef SHA1DC_HAVE_AVX2 + if (avx2) + check_form( + sha1dc_ubc_check_avx2_mirrored( + mirrored), want); +#endif + } +#endif +#ifdef SHA1DC_HAVE_AVX2 + if (avx2) + check_form(sha1dc_ubc_check_avx2(w), want); +#endif +#ifdef SHA1DC_HAVE_NEON + check_form(sha1dc_ubc_check_neon(w), want); +#endif + (void)avx2; + if (k) + w[(k - 1) / 32] ^= 1u << ((k - 1) % 32); + } +} + +static void ubc_forms_agree_with_sha1dc(void) +{ + uint32_t w[80]; + int i, k, dv, avx2 = 0; + +#ifdef SHA1DC_HAVE_AVX2 + /* Once: CPUID can take microseconds under a hypervisor. */ + avx2 = sha1dc_avx2_available(); +#endif + rng_seed(2); + for (i = 0; i < 20000; i++) { + for (k = 0; k < 80; k++) + w[k] = rng(); + check_ubc_forms(w, 0, avx2); + } + + /* + * Random schedules rarely get past the vector prefix of a form, so + * look for one that keeps each DV alive, and check around it. + */ + for (dv = 0; dv < 32; dv++) { + uint32_t mask; + + do { + for (k = 0; k < 80; k++) + w[k] = rng(); + ubc_check(w, &mask); + } while (!(mask & (1u << dv))); + check_ubc_forms(w, 1, avx2); + } +} + +#if defined(SHA1DC_HAVE_SHANI) || defined(SHA1DC_HAVE_ARMV8) + +static uint32_t rol(uint32_t x, int n) +{ + return (x << n) | (x >> (32 - n)); +} + +static uint32_t f_k(int t, uint32_t b, uint32_t c, uint32_t d) +{ + if (t < 20) + return ((b & c) | (~b & d)) + 0x5A827999; + if (t < 40) + return (b ^ c ^ d) + 0x6ED9EBA1; + if (t < 60) + return ((b & c) | (b & d) | (c & d)) + 0x8F1BBCDC; + return (b ^ c ^ d) + 0xCA62C1D6; +} + +/* One SHA-1 step forwards, from the state before step t. */ +static void step(uint32_t s[5], int t, uint32_t w) +{ + uint32_t a = rol(s[0], 5) + f_k(t, s[1], s[2], s[3]) + s[4] + w; + s[4] = s[3]; + s[3] = s[2]; + s[2] = rol(s[1], 30); + s[1] = s[0]; + s[0] = a; +} + +/* One SHA-1 step backwards, to the state before step t. */ +static void unstep(uint32_t s[5], int t, uint32_t w) +{ + uint32_t a = s[1], b = rol(s[2], 2), c = s[3], d = s[4]; + s[4] = s[0] - (rol(a, 5) + f_k(t, b, c, d) + w); + s[0] = a; + s[1] = b; + s[2] = c; + s[3] = d; +} + +typedef void (*compress_fn)(uint32_t ihv[5], const unsigned char *block, + uint32_t w[80], uint32_t at_60[5], uint32_t at_64[5]); +typedef int (*recompress_fn)(enum sha1dc_from from, const uint32_t m1[80], + const uint32_t dm[80], const uint32_t state[5], + const uint32_t ihv_out[5]); + +static void check_compress(compress_fn compress, int mirrored) +{ + int i, t; + + rng_seed(3); + for (i = 0; i < 2000; i++) { + unsigned char block[64]; + uint32_t ihv[5], got[5], w[80], expected[80]; + uint32_t at_60[5], at_64[5], s[5]; + + for (t = 0; t < 64; t++) + block[t] = rng(); + for (t = 0; t < 5; t++) + ihv[t] = got[t] = rng(); + compress(got, block, w, at_60, at_64); + + memcpy(s, ihv, sizeof(s)); + for (t = 0; t < 80; t++) { + expected[t] = t < 16 ? get_be32(block + 4 * t) : + rol(expected[t - 3] ^ expected[t - 8] ^ + expected[t - 14] ^ expected[t - 16], 1); + cl_assert_equal_i(w[mirrored ? 79 - t : t], + expected[t]); + if (t == 60) + cl_assert(!memcmp(at_60, s, sizeof(s))); + if (t == 64) + cl_assert(!memcmp(at_64, s, sizeof(s))); + step(s, t, expected[t]); + } + for (t = 0; t < 5; t++) + cl_assert_equal_i(got[t], ihv[t] + s[t]); + } +} + +/* + * A recompression must accept exactly the chaining value the partner + * block ends on, and nothing else. + */ +static void check_recompress(recompress_fn recompress) +{ + int i, t, d; + + rng_seed(4); + for (i = 0; i < 50; i++) { + uint32_t w[80], state[5]; + + for (t = 0; t < 80; t++) + w[t] = rng(); + for (t = 0; t < 5; t++) + state[t] = rng(); + + for (d = 0; sha1_dvs[d].dvType; d++) { + const dv_info_t *dv = &sha1_dvs[d]; + enum sha1dc_from from = dv->testt == 58 ? SHA1DC_FROM_58 : SHA1DC_FROM_65; + uint32_t in[5], out[5]; + unsigned nudge = rng(); + + memcpy(in, state, sizeof(in)); + for (t = dv->testt - 1; t >= 0; t--) + unstep(in, t, w[t] ^ dv->dm[t]); + memcpy(out, state, sizeof(out)); + for (t = dv->testt; t < 80; t++) + step(out, t, w[t] ^ dv->dm[t]); + for (t = 0; t < 5; t++) + out[t] += in[t]; + + cl_assert(recompress(from, w, dv->dm, state, out)); + out[nudge % 5] ^= 1u << (nudge / 5 % 32); + cl_assert(!recompress(from, w, dv->dm, state, out)); + } + } +} + +#endif + +static void hardware_compression(void) +{ + int tested = 0; + +#ifdef SHA1DC_HAVE_SHANI + if (sha1dc_shani_available()) { + check_compress(sha1dc_compress_shani, 1); + check_recompress(sha1dc_recompress_shani); + tested = 1; + } +#endif +#ifdef SHA1DC_HAVE_ARMV8 + if (sha1dc_armv8_available()) { + check_compress(sha1dc_compress_armv8, 0); + check_recompress(sha1dc_recompress_armv8); + tested = 1; + } +#endif + if (!tested) + cl_skip(); +} + +#endif /* HAVE_SHA1DC_ACCEL */ + +#ifdef HAVE_SHA1DC_ACCEL +#define RUN_OR_SKIP(fn) fn() +#else +#define RUN_OR_SKIP(fn) cl_skip() +#endif + +void test_sha1dc__dv_table_is_as_expected(void) +{ + RUN_OR_SKIP(dv_table_is_as_expected); +} + +void test_sha1dc__every_backend_agrees_with_sha1dc(void) +{ + RUN_OR_SKIP(every_backend_agrees_with_sha1dc); +} + +void test_sha1dc__ubc_forms_agree_with_sha1dc(void) +{ + RUN_OR_SKIP(ubc_forms_agree_with_sha1dc); +} + +void test_sha1dc__hardware_compression(void) +{ + RUN_OR_SKIP(hardware_compression); +} diff --git a/thread-utils.h b/thread-utils.h index 4961487ed914f4..98b574eff41e64 100644 --- a/thread-utils.h +++ b/thread-utils.h @@ -19,6 +19,22 @@ #define pthread_mutex_t int #define pthread_cond_t int #define pthread_key_t int +#define pthread_once_t int +#undef PTHREAD_ONCE_INIT +#define PTHREAD_ONCE_INIT 0 + +static inline int dummy_pthread_once(pthread_once_t *once_control, + void (*init_routine)(void)) +{ + if (!*once_control) { + init_routine(); + *once_control = 1; + } + return 0; +} + +#define pthread_once(once_control, init_routine) \ + dummy_pthread_once((once_control), (init_routine)) #define pthread_mutex_init(mutex, attr) dummy_pthread_init(mutex) #define pthread_mutex_lock(mutex) diff --git a/trace2/tr2_sid.c b/trace2/tr2_sid.c index 131b4f5a620ee3..54e3675a141eb0 100644 --- a/trace2/tr2_sid.c +++ b/trace2/tr2_sid.c @@ -32,7 +32,11 @@ static void tr2_sid_append_my_sid_component(void) { const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; struct tr2_tbuf tb_now; +#ifdef SHA1_DC + SHA1_CTX ctx; +#else struct git_hash_ctx ctx; +#endif pid_t pid = getpid(); unsigned char hash[GIT_MAX_RAWSZ + 1]; char hex[GIT_MAX_HEXSZ + 1]; @@ -45,9 +49,16 @@ static void tr2_sid_append_my_sid_component(void) if (xgethostname(hostname, sizeof(hostname))) strbuf_add(&tr2sid_buf, "Localhost", 9); else { +#ifdef SHA1_DC + /* Startup tracing must not select a configurable SHA-1 backend. */ + git_SHA1DCInit(&ctx); + git_SHA1DCUpdate(&ctx, hostname, strlen(hostname)); + git_SHA1DCFinal(hash, &ctx, die); +#else git_hash_init(&ctx, algo); git_hash_update(&ctx, hostname, strlen(hostname)); git_hash_final(hash, &ctx); +#endif hash_to_hex_algop_r(hex, hash, algo); strbuf_addch(&tr2sid_buf, 'H'); strbuf_add(&tr2sid_buf, hex, 8);