ed/raddebugger
1
0
Fork 0
mirror of https://github.com/Ed94/raddebugger.git synced 2026-08-14 17:28:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

updated hashing libraries

Browse Source
This commit is contained in:
Nikita Smith
2025-11-13 11:45:34 -08:00
parent 7cafde3376
commit 08f794527c
3 changed files with 748 additions and 733 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/third_party/martins_hash/sha1.h
Vendored
+104 -120
View File
@@ -137,17 +137,17 @@ static inline int sha1_cpuid(void)
SHA1_TARGET("ssse3,sha") SHA1_TARGET("ssse3,sha")
static void sha1_process_shani(uint32_t* state, const uint8_t* block, size_t count) static void sha1_process_shani(uint32_t* state, const uint8_t* block, size_t count)
{ {
// in SHA1 each round has two parts: // in sha1 each round has two parts:
// 1) calculate message schedule dwords in w[i] // 1) calculate message schedule dwords in w[i]
// 2) do round functions to update a/b/c/d/e state values using w[i] // 2) do round functions to update a/b/c/d/e state values using w[i]
// w[i] in first 16 rounds is just loaded from block bytes, as 32-bit big-endian load // initial message schedule is loaded as 32-bit big-endian 16 dwords
// for next rounds it is done as: // for next rounds it is done as:
// w[i] = ROL(w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) // w[i] = ROL(w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16])
// where ROL(x) = 32-bit rotate left by 1 // where ROL(x) = 32-bit rotate left by 1
// this means it is possible to keep just the last 16 of w's in circular buffer // this means it is possible to keep just the last 16 of w's
// and every new w calculated will need to update 1 to 3 previous w's // and every new w calculated will need to update 1 to 3 previous w's
// unrolling round calculations by 4 we get: // unrolling round calculations by 4 we get:
@@ -156,7 +156,7 @@ static void sha1_process_shani(uint32_t* state, const uint8_t* block, size_t cou
// w[i+2] = ROL(w[i-1] ^ w[i-6] ^ w[i-12] ^ w[i-14]) // w[i+2] = ROL(w[i-1] ^ w[i-6] ^ w[i-12] ^ w[i-14])
// w[i+3] = ROL(w[i+0] ^ w[i-5] ^ w[i-11] ^ w[i-13]) // w[i+3] = ROL(w[i+0] ^ w[i-5] ^ w[i-11] ^ w[i-13])
// now if you store 4 w[..] values in 128-bit SSE register, then // now use 4 w[..] values in 128-bit SSE register W(i), then
// W(i) = ROL( r0 ^ r1 ^ r2 ^ r3 ) // W(i) = ROL( r0 ^ r1 ^ r2 ^ r3 )
// with caveat that r0 lane 3 depends on W(i) lane 0 // with caveat that r0 lane 3 depends on W(i) lane 0
@@ -166,43 +166,31 @@ static void sha1_process_shani(uint32_t* state, const uint8_t* block, size_t cou
// r2 = [ w[i-11], w[i-12], w[i-13], w[i-14] ] // r2 = [ w[i-11], w[i-12], w[i-13], w[i-14] ]
// r3 = [ w[i-13], w[i-14], w[i-15], w[i-16] ] // r3 = [ w[i-13], w[i-14], w[i-15], w[i-16] ]
// in each 4-round i'th step it is possible to incrementally update new W(..) value when // in each 4-round i'th step it is possible to incrementally update W's that will be
// keeping W(i) values in 4 xmm element circular buffer // used in later rounds
// rounds i>0: W(i-1) = r2 ^ r3 = _mm_sha1msg1_epu32(W(i-1), W(i)) // rounds i>0: m3 = r2 ^ r3 = _mm_sha1msg1_epu32(m3, m0)
// rounds i>1: W(i-2) = W(i-2) ^ r1 = _mm_xor_si128 (W(i-2), W(i)) // rounds i>1: m2 = m2 ^ r1 = _mm_xor_si128 (m2, m0)
// rounds i>2: W(i-3) = ROL(W(i-3) ^ r0) = _mm_sha1msg2_epu32(W(i-3), W(i)) // rounds i>2: m1 = ROL(m1 ^ r0) = _mm_sha1msg2_epu32(m1, m0)
// then the new W(i) can be used in round function calculations // then the new m0 can be used in round function calculations
// _mm_sha1msg2_epu32 correctly handles r0 lane 3 dependency on W(i) lane 0 // _mm_sha1msg2_epu32 correctly handles r0 lane 3 dependency on lane 0 output
// to perform round functions on two SIMD registers with state as: // to perform round functions on two SIMD registers with state as:
// abcd = [a,b,c,d] // s0 = [a,b,c,d]
// e0 = [e,0,0,0] // s1 = [e,0,0,0]
// use the following code to get next abcd/e0 state 4 rounds at a time: // use _mm_sha1rnds4_epu32 and _mm_sha1nexte_epu32 ops and swap both state variables between rounds
// tmp = _mm_sha1nexte_epu32(e0, W(i)) // rotates e0 and adds message dwords
// abcd_next = _mm_sha1rnds4_epu32(abcd, tmp, Fn) // with Fn = 0..3 round function selection
// e0_next = abcd
// sha1nexte is not needed on first round, just regular add32(e0, W(i)) should be used
// after last round need to do extra rotation, which sha1nexte takes care when adding to last_e0
#define W(i) w[(i)%4]
// 4 wide round calculations // 4 wide round calculations
#define QROUND(i) do { \ #define QROUND(i,s0,s1,m0,m1,m2,m3) do { \
/* first 4 rounds load input block */ \
if (i < 4) W(i) = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[i]), bswap); \
/* update message schedule */ \ /* update message schedule */ \
if (i > 0 && i < 17) W(i-1) = _mm_sha1msg1_epu32(W(i-1), W(i)); \ if (i > 0 && i < 17) m3 = _mm_sha1msg1_epu32(m3, m0); \
if (i > 1 && i < 18) W(i-2) = _mm_xor_si128 (W(i-2), W(i)); \ if (i > 1 && i < 18) m2 = _mm_xor_si128 (m2, m0); \
if (i > 2 && i < 19) W(i-3) = _mm_sha1msg2_epu32(W(i-3), W(i)); \ if (i > 2 && i < 19) m1 = _mm_sha1msg2_epu32(m1, m0); \
/* calculate E plus message schedule */ \ /* calculate E plus message schedule */ \
if (i == 0) tmp = _mm_add_epi32 (e0, W(i)); \ if (i == 0) tmp = _mm_add_epi32 (s1, m0); \
if (i != 0) tmp = _mm_sha1nexte_epu32(e0, W(i)); \ if (i != 0) tmp = _mm_sha1nexte_epu32(s1, m0); \
/* 4 round functions */ \ /* 4 round functions */ \
e0 = abcd; \ s1 = _mm_sha1rnds4_epu32(s0, tmp, i/5); \
abcd = _mm_sha1rnds4_epu32(abcd, tmp, i/5); \
} while(0) } while(0)
const __m128i* buffer = (const __m128i*)block; const __m128i* buffer = (const __m128i*)block;
@@ -213,63 +201,67 @@ static void sha1_process_shani(uint32_t* state, const uint8_t* block, size_t cou
const __m128i bswap = _mm_setr_epi8(15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0); const __m128i bswap = _mm_setr_epi8(15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0);
// load initial state // load initial state
__m128i abcd = _mm_loadu_si128((const __m128i*)state); // [d,c,b,a] __m128i s0 = _mm_loadu_si128((const __m128i*)state); // [d,c,b,a]
__m128i e0 = _mm_loadu_si32(&state[4]); // [0,0,0,e] __m128i s1 = _mm_loadu_si32(&state[4]); // [0,0,0,e]
// flip dword order, to what sha1 instructions use // flip dword order, to what sha1 instructions use
abcd = _mm_shuffle_epi32(abcd, _MM_SHUFFLE(0,1,2,3)); // [a,b,c,d] where a is in the top lane s0 = _mm_shuffle_epi32(s0, _MM_SHUFFLE(0,1,2,3)); // [a,b,c,d]
e0 = _mm_slli_si128(e0, 12); // [e,0,0,0] where e is in top lane s1 = _mm_shuffle_epi32(s1, _MM_SHUFFLE(0,1,2,3)); // [e,0,0,0]
do do
{ {
// remember current state // remember current state
__m128i last_abcd = abcd; __m128i last0 = s0;
__m128i last_e0 = e0; __m128i last1 = s1;
__m128i tmp, w[4]; // load initial message schedule, 64-byte block
__m128i w0 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[0]), bswap);
__m128i w1 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[1]), bswap);
__m128i w2 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[2]), bswap);
__m128i w3 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[3]), bswap);
buffer += 4;
QROUND(0); __m128i tmp;
QROUND(1);
QROUND(2);
QROUND(3);
QROUND(4);
QROUND(5); QROUND( 0, s0, s1, w0, w1, w2, w3);
QROUND(6); QROUND( 1, s1, s0, w1, w2, w3, w0);
QROUND(7); QROUND( 2, s0, s1, w2, w3, w0, w1);
QROUND(8); QROUND( 3, s1, s0, w3, w0, w1, w2);
QROUND(9); QROUND( 4, s0, s1, w0, w1, w2, w3);
QROUND(10); QROUND( 5, s1, s0, w1, w2, w3, w0);
QROUND(11); QROUND( 6, s0, s1, w2, w3, w0, w1);
QROUND(12); QROUND( 7, s1, s0, w3, w0, w1, w2);
QROUND(13); QROUND( 8, s0, s1, w0, w1, w2, w3);
QROUND(14); QROUND( 9, s1, s0, w1, w2, w3, w0);
QROUND(15); QROUND(10, s0, s1, w2, w3, w0, w1);
QROUND(16); QROUND(11, s1, s0, w3, w0, w1, w2);
QROUND(17); QROUND(12, s0, s1, w0, w1, w2, w3);
QROUND(18); QROUND(13, s1, s0, w1, w2, w3, w0);
QROUND(19); QROUND(14, s0, s1, w2, w3, w0, w1);
QROUND(15, s1, s0, w3, w0, w1, w2);
QROUND(16, s0, s1, w0, w1, w2, w3);
QROUND(17, s1, s0, w1, w2, w3, w0);
QROUND(18, s0, s1, w2, w3, w0, w1);
QROUND(19, s1, s0, w3, w0, w1, w2);
// update next state // update next state
abcd = _mm_add_epi32(abcd, last_abcd); s0 = _mm_add_epi32 (s0, last0);
e0 = _mm_sha1nexte_epu32(e0, last_e0); s1 = _mm_sha1nexte_epu32(s1, last1);
buffer += 4;
} }
while (--count); while (--count);
// restore dword order // restore dword order
abcd = _mm_shuffle_epi32(abcd, _MM_SHUFFLE(0,1,2,3)); s0 = _mm_shuffle_epi32(s0, _MM_SHUFFLE(0,1,2,3));
e0 = _mm_shuffle_epi32(e0, _MM_SHUFFLE(0,1,2,3)); s1 = _mm_shuffle_epi32(s1, _MM_SHUFFLE(0,1,2,3));
// save the new state // save the new state
_mm_storeu_si128((__m128i*)state, abcd); _mm_storeu_si128((__m128i*)state, s0);
_mm_storeu_si32(&state[4], e0); _mm_storeu_si32(&state[4], s1);
#undef QROUND #undef QROUND
#undef W
} }
#endif // defined(__x86_64__) || defined(_M_AMD64) #endif // defined(__x86_64__) || defined(_M_AMD64)
@@ -343,21 +335,16 @@ static void sha1_process_arm64(uint32_t* state, const uint8_t* block, size_t cou
{ {
// code here is similar to x64 shani implementation // code here is similar to x64 shani implementation
// message array is 16 element circular buffer #define QROUND(i,m0,m1,m2,m3,k,F) do { \
// each iteration updates 4 rounds at the same time
#define W(i) w[(i)%4]
#define QROUND(i,F,k) do { \
/* update message schedule */ \ /* update message schedule */ \
if (i >= 4) W(i) = vsha1su0q_u32(W(i), W(i-3), W(i-2)); \ if (i >= 4) m0 = vsha1su0q_u32(m0, m1, m2); \
if (i >= 4) W(i) = vsha1su1q_u32(W(i), W(i-1)); \ if (i >= 4) m0 = vsha1su1q_u32(m0, m3); \
/* add round constant */ \ /* add round constant */ \
uint32x4_t tmp = vaddq_u32(W(i), k); \ uint32x4_t tmp = vaddq_u32(m0, k); \
/* 4 round functions */ \ /* 4 round functions */ \
uint32_t x = e0; \ uint32_t e = vgetq_lane_u32(s0, 0); \
e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); \ s0 = F(s0, s1, tmp); \
abcd = F(abcd, x, tmp); \ s1 = vsha1h_u32(e); \
} while (0) } while (0)
const uint32x4_t k0 = vdupq_n_u32(0x5a827999); const uint32x4_t k0 = vdupq_n_u32(0x5a827999);
@@ -365,64 +352,61 @@ static void sha1_process_arm64(uint32_t* state, const uint8_t* block, size_t cou
const uint32x4_t k2 = vdupq_n_u32(0x8f1bbcdc); const uint32x4_t k2 = vdupq_n_u32(0x8f1bbcdc);
const uint32x4_t k3 = vdupq_n_u32(0xca62c1d6); const uint32x4_t k3 = vdupq_n_u32(0xca62c1d6);
// load state - a,b,c,d,e // load initial state
uint32x4_t abcd = vld1q_u32(state); uint32x4_t s0 = vld1q_u32(state); // [d,c,b,a]
uint32_t e0 = state[4]; uint32_t s1 = state[4]; // e
do do
{ {
// remember current state // remember current state
uint32x4_t last_abcd = abcd; uint32x4_t last0 = s0;
uint32_t last_e0 = e0; uint32_t last1 = s1;
// load 64-byte block and advance pointer to next block // load initial message schedule, 64-byte block
uint8x16x4_t msg = vld1q_u8_x4(block); uint8x16x4_t msg = vld1q_u8_x4(block);
block += SHA1_BLOCK_SIZE; block += SHA1_BLOCK_SIZE;
uint32x4_t w[4]; // reverse the byte order in each 32-bit lane
uint32x4_t w0 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[0]));
uint32x4_t w1 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[1]));
uint32x4_t w2 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[2]));
uint32x4_t w3 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[3]));
// for first 16 w's reverse the byte order in each 32-bit lane QROUND( 0, w0, w1, w2, w3, k0, vsha1cq_u32);
W(0) = vreinterpretq_u32_u8(vrev32q_u8(msg.val[0])); QROUND( 1, w1, w2, w3, w0, k0, vsha1cq_u32);
W(1) = vreinterpretq_u32_u8(vrev32q_u8(msg.val[1])); QROUND( 2, w2, w3, w0, w1, k0, vsha1cq_u32);
W(2) = vreinterpretq_u32_u8(vrev32q_u8(msg.val[2])); QROUND( 3, w3, w0, w1, w2, k0, vsha1cq_u32);
W(3) = vreinterpretq_u32_u8(vrev32q_u8(msg.val[3])); QROUND( 4, w0, w1, w2, w3, k0, vsha1cq_u32);
QROUND( 0, vsha1cq_u32, k0); QROUND( 5, w1, w2, w3, w0, k1, vsha1pq_u32);
QROUND( 1, vsha1cq_u32, k0); QROUND( 6, w2, w3, w0, w1, k1, vsha1pq_u32);
QROUND( 2, vsha1cq_u32, k0); QROUND( 7, w3, w0, w1, w2, k1, vsha1pq_u32);
QROUND( 3, vsha1cq_u32, k0); QROUND( 8, w0, w1, w2, w3, k1, vsha1pq_u32);
QROUND( 4, vsha1cq_u32, k0); QROUND( 9, w1, w2, w3, w0, k1, vsha1pq_u32);
QROUND( 5, vsha1pq_u32, k1); QROUND(10, w2, w3, w0, w1, k2, vsha1mq_u32);
QROUND( 6, vsha1pq_u32, k1); QROUND(11, w3, w0, w1, w2, k2, vsha1mq_u32);
QROUND( 7, vsha1pq_u32, k1); QROUND(12, w0, w1, w2, w3, k2, vsha1mq_u32);
QROUND( 8, vsha1pq_u32, k1); QROUND(13, w1, w2, w3, w0, k2, vsha1mq_u32);
QROUND( 9, vsha1pq_u32, k1); QROUND(14, w2, w3, w0, w1, k2, vsha1mq_u32);
QROUND(10, vsha1mq_u32, k2); QROUND(15, w3, w0, w1, w2, k3, vsha1pq_u32);
QROUND(11, vsha1mq_u32, k2); QROUND(16, w0, w1, w2, w3, k3, vsha1pq_u32);
QROUND(12, vsha1mq_u32, k2); QROUND(17, w1, w2, w3, w0, k3, vsha1pq_u32);
QROUND(13, vsha1mq_u32, k2); QROUND(18, w2, w3, w0, w1, k3, vsha1pq_u32);
QROUND(14, vsha1mq_u32, k2); QROUND(19, w3, w0, w1, w2, k3, vsha1pq_u32);
QROUND(15, vsha1pq_u32, k3);
QROUND(16, vsha1pq_u32, k3);
QROUND(17, vsha1pq_u32, k3);
QROUND(18, vsha1pq_u32, k3);
QROUND(19, vsha1pq_u32, k3);
// update next state // update next state
abcd = vaddq_u32(abcd, last_abcd); s0 = vaddq_u32(s0, last0);
e0 += last_e0; s1 += last1;
} }
while (--count); while (--count);
// save state // save state
vst1q_u32(state, abcd); vst1q_u32(state, s0);
state[4] = e0; state[4] = s1;
#undef QROUND #undef QROUND
#undef W
} }
#endif // defined(__aarch64__) || defined(_M_ARM64) #endif // defined(__aarch64__) || defined(_M_ARM64)
@@ -452,7 +436,7 @@ static void sha1_process(uint32_t* state, const uint8_t* block, size_t count)
#define F3(x,y,z) (0x8f1bbcdc + ((x & y) | (z & (x | y)))) #define F3(x,y,z) (0x8f1bbcdc + ((x & y) | (z & (x | y))))
#define F4(x,y,z) (0xca62c1d6 + (x ^ y ^ z)) #define F4(x,y,z) (0xca62c1d6 + (x ^ y ^ z))
#define W(i) w[(i)%16] #define W(i) w[(i+16)%16]
#define ROUND(i,a,b,c,d,e,F) do \ #define ROUND(i,a,b,c,d,e,F) do \
{ \ { \
src/third_party/martins_hash/sha256.h
Vendored
+92 -91
View File
@@ -167,9 +167,7 @@ static void sha256_process_shani(uint32_t* state, const uint8_t* block, size_t c
{ {
// similar way how sha1 works in with shani // similar way how sha1 works in with shani
// first 16 rounds loads message schedule dwords as 32-bit big endian values // rounds message schedule is updated as:
// for next rounds message schedule is prepared as:
// w[i] = SSig1(w[i-2]) + w[i-7] + SSig0(w[i-15]) + w[i-16] // w[i] = SSig1(w[i-2]) + w[i-7] + SSig0(w[i-15]) + w[i-16]
// unrolled by 4: // unrolled by 4:
@@ -190,32 +188,22 @@ static void sha256_process_shani(uint32_t* state, const uint8_t* block, size_t c
// r2 = [ w[i-12], w[i-13], w[i-14], w[i-15] ] // r2 = [ w[i-12], w[i-13], w[i-14], w[i-15] ]
// r3 = [ w[i-13], w[i-14], w[i-15], w[i-16] ] // r3 = [ w[i-13], w[i-14], w[i-15], w[i-16] ]
// rN's can be calculated from previous W(..) values: // rounds i>2: m1 = _mm_sha256msg2_epu32(_mm_add_epi32(m1, _mm_alignr_epi8(m0, m3, 4) ), m0)
// r0 from W(i) // rounds i>0: m3 = _mm_sha256msg1_epu32(m3, m0)
// r1 from _mm_alignr_epi8(W(i), W(i-1), 4)
// r2 from W(i-1) and W(i)
// r3 from W(i-1)
// rounds i>2: W(i-3) = _mm_sha256msg2_epu32(_mm_add_epi32( W(i-3), _mm_alignr_epi8(W(i), W(i-1), 4) ), W(i))
// rounds i>0: W(i-1) = _mm_sha256msg1_epu32(W(i-1), W(i))
// round functions are done with _mm_sha256rnds2_epu32 which performs it for 2 rounds // round functions are done with _mm_sha256rnds2_epu32 which performs it for 2 rounds
// thus repeat it two times, as input use W(i) + K(i) - message schedule added with sha256 constants // thus repeat it two times, as input use m0 + K(i) - message schedule added with sha256 constants
#define W(i) w[(i)%4]
// 4 wide round calculations // 4 wide round calculations
#define QROUND(i) do { \ #define QROUND(i,m0,m1,m2,m3) do { \
/* first 4 rounds load input block */ \
if (i < 4) W(i) = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[i]), bswap); \
/* update message schedule */ \ /* update message schedule */ \
if (i > 2 && i < 15) W(i-3) = _mm_sha256msg2_epu32(_mm_add_epi32(W(i-3), _mm_alignr_epi8(W(i), W(i-1), 4)), W(i)); \ if (i > 2 && i < 15) m1 = _mm_sha256msg2_epu32(_mm_add_epi32(m1, _mm_alignr_epi8(m0, m3, 4)), m0); \
if (i > 0 && i < 13) W(i-1) = _mm_sha256msg1_epu32(W(i-1), W(i)); \ if (i > 0 && i < 13) m3 = _mm_sha256msg1_epu32(m3, m0); \
/* add round constants */ \ /* add round constants */ \
__m128i tmp = _mm_add_epi32(W(i), _mm_loadu_si128((const __m128i*)&SHA256_K[4*i])); \ __m128i tmp = _mm_add_epi32(m0, _mm_loadu_si128((const __m128i*)&SHA256_K[4*i])); \
/* 4 round functions */ \ /* 4 round functions */ \
state1 = _mm_sha256rnds2_epu32(state1, state0, tmp); \ s1 = _mm_sha256rnds2_epu32(s1, s0, tmp); \
state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(tmp, _MM_SHUFFLE(0,0,3,2))); \ s0 = _mm_sha256rnds2_epu32(s0, s1, _mm_shuffle_epi32(tmp, _MM_SHUFFLE(0,0,3,2))); \
} while(0) } while(0)
const __m128i* buffer = (const __m128i*)block; const __m128i* buffer = (const __m128i*)block;
@@ -228,52 +216,57 @@ static void sha256_process_shani(uint32_t* state, const uint8_t* block, size_t c
__m128i efgh = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)&state[4]), _MM_SHUFFLE(0,1,2,3)); // [e,f,g,h] __m128i efgh = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)&state[4]), _MM_SHUFFLE(0,1,2,3)); // [e,f,g,h]
// dword order for sha256rnds2 instruction // dword order for sha256rnds2 instruction
__m128i state0 = _mm_unpackhi_epi64(efgh, abcd); // [a,b,e,f] __m128i s0 = _mm_unpackhi_epi64(efgh, abcd); // [a,b,e,f]
__m128i state1 = _mm_unpacklo_epi64(efgh, abcd); // [c,d,g,h] __m128i s1 = _mm_unpacklo_epi64(efgh, abcd); // [c,d,g,h]
do do
{ {
// remember current state // remember current state
__m128i last0 = state0; __m128i last0 = s0;
__m128i last1 = state1; __m128i last1 = s1;
__m128i w[4]; // load initial message schedule, 64-byte block
__m128i w0 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[0]), bswap);
__m128i w1 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[1]), bswap);
__m128i w2 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[2]), bswap);
__m128i w3 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[3]), bswap);
buffer += 4;
QROUND( 0); QROUND( 0, w0, w1, w2, w3);
QROUND( 1); QROUND( 1, w1, w2, w3, w0);
QROUND( 2); QROUND( 2, w2, w3, w0, w1);
QROUND( 3); QROUND( 3, w3, w0, w1, w2);
QROUND( 4);
QROUND( 5); QROUND( 4, w0, w1, w2, w3);
QROUND( 6); QROUND( 5, w1, w2, w3, w0);
QROUND( 7); QROUND( 6, w2, w3, w0, w1);
QROUND( 8); QROUND( 7, w3, w0, w1, w2);
QROUND( 9);
QROUND(10); QROUND( 8, w0, w1, w2, w3);
QROUND(11); QROUND( 9, w1, w2, w3, w0);
QROUND(12); QROUND(10, w2, w3, w0, w1);
QROUND(13); QROUND(11, w3, w0, w1, w2);
QROUND(14);
QROUND(15); QROUND(12, w0, w1, w2, w3);
QROUND(13, w1, w2, w3, w0);
QROUND(14, w2, w3, w0, w1);
QROUND(15, w3, w0, w1, w2);
// update next state // update next state
state0 = _mm_add_epi32(state0, last0); s0 = _mm_add_epi32(s0, last0);
state1 = _mm_add_epi32(state1, last1); s1 = _mm_add_epi32(s1, last1);
buffer += 4;
} }
while (--count); while (--count);
// restore dword order // restore dword order
abcd = _mm_unpackhi_epi64(state1, state0); abcd = _mm_unpackhi_epi64(s1, s0);
efgh = _mm_unpacklo_epi64(state1, state0); efgh = _mm_unpacklo_epi64(s1, s0);
// save the new state // save the new state
_mm_storeu_si128((__m128i*)&state[0], _mm_shuffle_epi32(abcd, _MM_SHUFFLE(0,1,2,3))); _mm_storeu_si128((__m128i*)&state[0], _mm_shuffle_epi32(abcd, _MM_SHUFFLE(0,1,2,3)));
_mm_storeu_si128((__m128i*)&state[4], _mm_shuffle_epi32(efgh, _MM_SHUFFLE(0,1,2,3))); _mm_storeu_si128((__m128i*)&state[4], _mm_shuffle_epi32(efgh, _MM_SHUFFLE(0,1,2,3)));
#undef QROUND #undef QROUND
#undef W
} }
#endif // defined(__x86_64__) || defined(_M_AMD64) #endif // defined(__x86_64__) || defined(_M_AMD64)
@@ -344,70 +337,71 @@ static inline int sha256_cpuid(void)
SHA256_TARGET SHA256_TARGET
static void sha256_process_arm64(uint32_t* state, const uint8_t* block, size_t count) static void sha256_process_arm64(uint32_t* state, const uint8_t* block, size_t count)
{ {
// code here is similar to x64 shani implementation #define QROUND(i,m0,m1,m2,m3) do { \
#define W(i) w[(i)%4]
#define QROUND(i) do { \
/* load 16 round constants */ \
if ((i % 4) == 0) rk = vld1q_u32_x4(&SHA256_K[4*i]); \
/* first 4 rounds reverse byte order in each 32-bit lane of input block */ \
if (i < 4) W(i) = vreinterpretq_u32_u8(vrev32q_u8(msg.val[i])); \
/* update message schedule */ \ /* update message schedule */ \
if (i >= 4) W(i) = vsha256su0q_u32(W(i), W(i-3)); \ if (i >= 4) m0 = vsha256su1q_u32(vsha256su0q_u32(m0, m1), m2, m3); \
if (i >= 4) W(i) = vsha256su1q_u32(W(i), W(i-2), W(i-1)); \
/* add round constants */ \ /* add round constants */ \
uint32x4_t tmp = vaddq_u32(W(i), rk.val[i%4]); \ uint32x4_t tmp = vaddq_u32(m0, rk.val[i%4]); \
/* 4 round functions */ \ /* 4 round functions */ \
uint32x4_t x = vstate.val[0]; \ uint32x4x2_t x = s; \
vstate.val[0] = vsha256hq_u32(vstate.val[0], vstate.val[1], tmp); \ s.val[0] = vsha256hq_u32(x.val[0], x.val[1], tmp); \
vstate.val[1] = vsha256h2q_u32(vstate.val[1], x, tmp); \ s.val[1] = vsha256h2q_u32(x.val[1], x.val[0], tmp); \
} while (0) } while (0)
// load initial state // load initial state
uint32x4x2_t vstate = vld1q_u32_x2(state); uint32x4x2_t s = vld1q_u32_x2(state);
do do
{ {
// remember current state // remember current state
uint32x4x2_t vlast = vstate; uint32x4x2_t last = s;
// load 64-byte block // load initial message schedule, 64-byte block
uint8x16x4_t msg = vld1q_u8_x4(block); uint8x16x4_t msg = vld1q_u8_x4(block);
block += SHA256_BLOCK_SIZE;
// reverse the byte order in each 32-bit lane
uint32x4_t w0 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[0]));
uint32x4_t w1 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[1]));
uint32x4_t w2 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[2]));
uint32x4_t w3 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[3]));
uint32x4x4_t rk; uint32x4x4_t rk;
uint32x4_t w[4];
QROUND( 0); rk = vld1q_u32_x4(&SHA256_K[0]);
QROUND( 1); QROUND( 0, w0, w1, w2, w3);
QROUND( 2); QROUND( 1, w1, w2, w3, w0);
QROUND( 3); QROUND( 2, w2, w3, w0, w1);
QROUND( 4); QROUND( 3, w3, w0, w1, w2);
QROUND( 5);
QROUND( 6); rk = vld1q_u32_x4(&SHA256_K[16]);
QROUND( 7); QROUND( 4, w0, w1, w2, w3);
QROUND( 8); QROUND( 5, w1, w2, w3, w0);
QROUND( 9); QROUND( 6, w2, w3, w0, w1);
QROUND(10); QROUND( 7, w3, w0, w1, w2);
QROUND(11);
QROUND(12); rk = vld1q_u32_x4(&SHA256_K[32]);
QROUND(13); QROUND( 8, w0, w1, w2, w3);
QROUND(14); QROUND( 9, w1, w2, w3, w0);
QROUND(15); QROUND(10, w2, w3, w0, w1);
QROUND(11, w3, w0, w1, w2);
rk = vld1q_u32_x4(&SHA256_K[48]);
QROUND(12, w0, w1, w2, w3);
QROUND(13, w1, w2, w3, w0);
QROUND(14, w2, w3, w0, w1);
QROUND(15, w3, w0, w1, w2);
// update next state // update next state
vstate.val[0] = vaddq_u32(vstate.val[0], vlast.val[0]); s.val[0] = vaddq_u32(s.val[0], last.val[0]);
vstate.val[1] = vaddq_u32(vstate.val[1], vlast.val[1]); s.val[1] = vaddq_u32(s.val[1], last.val[1]);
block += SHA256_BLOCK_SIZE;
} }
while (--count); while (--count);
// save the new state // save the new state
vst1q_u32_x2(state, vstate); vst1q_u32_x2(state, s);
#undef QROUND #undef QROUND
#undef W
} }
#endif // defined(__aarch64__) || defined(_M_ARM64) #endif // defined(__aarch64__) || defined(_M_ARM64)
@@ -475,6 +469,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count)
ROUND( 5, d, e, f, g, h, a, b, c); ROUND( 5, d, e, f, g, h, a, b, c);
ROUND( 6, c, d, e, f, g, h, a, b); ROUND( 6, c, d, e, f, g, h, a, b);
ROUND( 7, b, c, d, e, f, g, h, a); ROUND( 7, b, c, d, e, f, g, h, a);
ROUND( 8, a, b, c, d, e, f, g, h); ROUND( 8, a, b, c, d, e, f, g, h);
ROUND( 9, h, a, b, c, d, e, f, g); ROUND( 9, h, a, b, c, d, e, f, g);
ROUND(10, g, h, a, b, c, d, e, f); ROUND(10, g, h, a, b, c, d, e, f);
@@ -483,6 +478,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count)
ROUND(13, d, e, f, g, h, a, b, c); ROUND(13, d, e, f, g, h, a, b, c);
ROUND(14, c, d, e, f, g, h, a, b); ROUND(14, c, d, e, f, g, h, a, b);
ROUND(15, b, c, d, e, f, g, h, a); ROUND(15, b, c, d, e, f, g, h, a);
ROUND(16, a, b, c, d, e, f, g, h); ROUND(16, a, b, c, d, e, f, g, h);
ROUND(17, h, a, b, c, d, e, f, g); ROUND(17, h, a, b, c, d, e, f, g);
ROUND(18, g, h, a, b, c, d, e, f); ROUND(18, g, h, a, b, c, d, e, f);
@@ -491,6 +487,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count)
ROUND(21, d, e, f, g, h, a, b, c); ROUND(21, d, e, f, g, h, a, b, c);
ROUND(22, c, d, e, f, g, h, a, b); ROUND(22, c, d, e, f, g, h, a, b);
ROUND(23, b, c, d, e, f, g, h, a); ROUND(23, b, c, d, e, f, g, h, a);
ROUND(24, a, b, c, d, e, f, g, h); ROUND(24, a, b, c, d, e, f, g, h);
ROUND(25, h, a, b, c, d, e, f, g); ROUND(25, h, a, b, c, d, e, f, g);
ROUND(26, g, h, a, b, c, d, e, f); ROUND(26, g, h, a, b, c, d, e, f);
@@ -499,6 +496,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count)
ROUND(29, d, e, f, g, h, a, b, c); ROUND(29, d, e, f, g, h, a, b, c);
ROUND(30, c, d, e, f, g, h, a, b); ROUND(30, c, d, e, f, g, h, a, b);
ROUND(31, b, c, d, e, f, g, h, a); ROUND(31, b, c, d, e, f, g, h, a);
ROUND(32, a, b, c, d, e, f, g, h); ROUND(32, a, b, c, d, e, f, g, h);
ROUND(33, h, a, b, c, d, e, f, g); ROUND(33, h, a, b, c, d, e, f, g);
ROUND(34, g, h, a, b, c, d, e, f); ROUND(34, g, h, a, b, c, d, e, f);
@@ -507,6 +505,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count)
ROUND(37, d, e, f, g, h, a, b, c); ROUND(37, d, e, f, g, h, a, b, c);
ROUND(38, c, d, e, f, g, h, a, b); ROUND(38, c, d, e, f, g, h, a, b);
ROUND(39, b, c, d, e, f, g, h, a); ROUND(39, b, c, d, e, f, g, h, a);
ROUND(40, a, b, c, d, e, f, g, h); ROUND(40, a, b, c, d, e, f, g, h);
ROUND(41, h, a, b, c, d, e, f, g); ROUND(41, h, a, b, c, d, e, f, g);
ROUND(42, g, h, a, b, c, d, e, f); ROUND(42, g, h, a, b, c, d, e, f);
@@ -515,6 +514,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count)
ROUND(45, d, e, f, g, h, a, b, c); ROUND(45, d, e, f, g, h, a, b, c);
ROUND(46, c, d, e, f, g, h, a, b); ROUND(46, c, d, e, f, g, h, a, b);
ROUND(47, b, c, d, e, f, g, h, a); ROUND(47, b, c, d, e, f, g, h, a);
ROUND(48, a, b, c, d, e, f, g, h); ROUND(48, a, b, c, d, e, f, g, h);
ROUND(49, h, a, b, c, d, e, f, g); ROUND(49, h, a, b, c, d, e, f, g);
ROUND(50, g, h, a, b, c, d, e, f); ROUND(50, g, h, a, b, c, d, e, f);
@@ -523,6 +523,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count)
ROUND(53, d, e, f, g, h, a, b, c); ROUND(53, d, e, f, g, h, a, b, c);
ROUND(54, c, d, e, f, g, h, a, b); ROUND(54, c, d, e, f, g, h, a, b);
ROUND(55, b, c, d, e, f, g, h, a); ROUND(55, b, c, d, e, f, g, h, a);
ROUND(56, a, b, c, d, e, f, g, h); ROUND(56, a, b, c, d, e, f, g, h);
ROUND(57, h, a, b, c, d, e, f, g); ROUND(57, h, a, b, c, d, e, f, g);
ROUND(58, g, h, a, b, c, d, e, f); ROUND(58, g, h, a, b, c, d, e, f);
src/third_party/martins_hash/sha512.h
Vendored
+200 -170
View File
@@ -62,28 +62,28 @@ static inline void sha384_finish(sha384_ctx* ctx, uint8_t digest[SHA384_DIGEST_S
# define SHA512_SET64BE(ptr,x) *((__unaligned uint64_t*)(ptr)) = _byteswap_uint64(x) # define SHA512_SET64BE(ptr,x) *((__unaligned uint64_t*)(ptr)) = _byteswap_uint64(x)
#else #else
# define SHA512_GET64BE(ptr) \ # define SHA512_GET64BE(ptr) \
( \ ( \
((uint64_t)((ptr)[0]) << 56) | \ ((uint64_t)((ptr)[0]) << 56) | \
((uint64_t)((ptr)[1]) << 48) | \ ((uint64_t)((ptr)[1]) << 48) | \
((uint64_t)((ptr)[2]) << 40) | \ ((uint64_t)((ptr)[2]) << 40) | \
((uint64_t)((ptr)[3]) << 32) | \ ((uint64_t)((ptr)[3]) << 32) | \
((uint64_t)((ptr)[4]) << 24) | \ ((uint64_t)((ptr)[4]) << 24) | \
((uint64_t)((ptr)[5]) << 16) | \ ((uint64_t)((ptr)[5]) << 16) | \
((uint64_t)((ptr)[6]) << 8) | \ ((uint64_t)((ptr)[6]) << 8) | \
((uint64_t)((ptr)[7]) << 0) \ ((uint64_t)((ptr)[7]) << 0) \
) )
# define SHA512_SET64BE(ptr, x) do \ # define SHA512_SET64BE(ptr, x) do \
{ \ { \
(ptr)[0] = (uint8_t)((x) >> 56); \ (ptr)[0] = (uint8_t)((x) >> 56); \
(ptr)[1] = (uint8_t)((x) >> 48); \ (ptr)[1] = (uint8_t)((x) >> 48); \
(ptr)[2] = (uint8_t)((x) >> 40); \ (ptr)[2] = (uint8_t)((x) >> 40); \
(ptr)[3] = (uint8_t)((x) >> 32); \ (ptr)[3] = (uint8_t)((x) >> 32); \
(ptr)[4] = (uint8_t)((x) >> 24); \ (ptr)[4] = (uint8_t)((x) >> 24); \
(ptr)[5] = (uint8_t)((x) >> 16); \ (ptr)[5] = (uint8_t)((x) >> 16); \
(ptr)[6] = (uint8_t)((x) >> 8); \ (ptr)[6] = (uint8_t)((x) >> 8); \
(ptr)[7] = (uint8_t)((x) >> 0); \ (ptr)[7] = (uint8_t)((x) >> 0); \
} \ } \
while (0) while (0)
#endif #endif
static const uint64_t SHA512_K[80] = static const uint64_t SHA512_K[80] =
@@ -181,21 +181,17 @@ static void sha512_process_vsha512(uint64_t* state, const uint8_t* block, size_t
// message qwords are loaded as 64-bit big-endian values // message qwords are loaded as 64-bit big-endian values
#define W(i) w[(i)%4]
// 4 wide round calculations // 4 wide round calculations
#define QROUND(i) do { \ #define QROUND(i,m0,m1,m2,m3) do { \
/* first 4 rounds load input block */ \ /* update message schedule */ \
if (i < 4) W(i) = _mm256_shuffle_epi8(_mm256_loadu_si256(&buffer[i]), bswap); \ if (i > 2 && i < 19) m1 = _mm256_sha512msg2_epi64(_mm256_add_epi64(m1, _mm256_permute4x64_epi64(_mm256_blend_epi32(m3, m0, 3), _MM_SHUFFLE(0,3,2,1))), m0); \
/* update message schedule */ \ if (i > 0 && i < 17) m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0)); \
if (i > 2 && i < 19) W(i-3) = _mm256_sha512msg2_epi64(_mm256_add_epi64(W(i-3), _mm256_permute4x64_epi64(_mm256_blend_epi32(W(i-1), W(i), 3), _MM_SHUFFLE(0,3,2,1))), W(i)); \ /* add round constants */ \
if (i > 0 && i < 17) W(i-1) = _mm256_sha512msg1_epi64(W(i-1), _mm256_castsi256_si128(W(i))); \ __m256i tmp = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i*)&SHA512_K[4*i])); \
/* add round constants */ \ /* 4 round functions */ \
__m256i tmp = _mm256_add_epi64(W(i), _mm256_loadu_si256((const __m256i*)&SHA512_K[4*i])); \ s1 = _mm256_sha512rnds2_epi64(s1, s0, _mm256_castsi256_si128(tmp)); \
/* round functions */ \ s0 = _mm256_sha512rnds2_epi64(s0, s1, _mm256_extracti128_si256(tmp, 1)); \
state1 = _mm256_sha512rnds2_epi64(state1, state0, _mm256_castsi256_si128(tmp)); \ } while(0)
state0 = _mm256_sha512rnds2_epi64(state0, state1, _mm256_extracti128_si256(tmp, 1)); \
} while(0)
const __m256i* buffer = (const __m256i*)block; const __m256i* buffer = (const __m256i*)block;
@@ -207,56 +203,62 @@ state0 = _mm256_sha512rnds2_epi64(state0, state1, _mm256_extracti128_si256(tmp,
__m256i efgh = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)&state[4]), _MM_SHUFFLE(0,1,2,3)); // [e,f,g,h] __m256i efgh = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)&state[4]), _MM_SHUFFLE(0,1,2,3)); // [e,f,g,h]
// qword order for vsha512rnds2 instruction // qword order for vsha512rnds2 instruction
__m256i state0 = _mm256_permute2x128_si256(efgh, abcd, (3 << 4) | 1); // [a,b,e,f] __m256i s0 = _mm256_permute2x128_si256(efgh, abcd, 0x31); // [a,b,e,f]
__m256i state1 = _mm256_permute2x128_si256(efgh, abcd, (2 << 4) | 0); // [c,d,g,h] __m256i s1 = _mm256_permute2x128_si256(efgh, abcd, 0x20); // [c,d,g,h]
do do
{ {
// remember current state // remember current state
__m256i last0 = state0; __m256i last0 = s0;
__m256i last1 = state1; __m256i last1 = s1;
__m256i w[4]; // load initial message schedule, 128-byte block
__m256i w0 = _mm256_shuffle_epi8(_mm256_loadu_si256(&buffer[0]), bswap);
__m256i w1 = _mm256_shuffle_epi8(_mm256_loadu_si256(&buffer[1]), bswap);
__m256i w2 = _mm256_shuffle_epi8(_mm256_loadu_si256(&buffer[2]), bswap);
__m256i w3 = _mm256_shuffle_epi8(_mm256_loadu_si256(&buffer[3]), bswap);
buffer += 4;
QROUND(0); QROUND( 0, w0, w1, w2, w3);
QROUND(1); QROUND( 1, w1, w2, w3, w0);
QROUND(2); QROUND( 2, w2, w3, w0, w1);
QROUND(3); QROUND( 3, w3, w0, w1, w2);
QROUND(4);
QROUND(5); QROUND( 4, w0, w1, w2, w3);
QROUND(6); QROUND( 5, w1, w2, w3, w0);
QROUND(7); QROUND( 6, w2, w3, w0, w1);
QROUND(8); QROUND( 7, w3, w0, w1, w2);
QROUND(9);
QROUND(10); QROUND( 8, w0, w1, w2, w3);
QROUND(11); QROUND( 9, w1, w2, w3, w0);
QROUND(12); QROUND(10, w2, w3, w0, w1);
QROUND(13); QROUND(11, w3, w0, w1, w2);
QROUND(14);
QROUND(15); QROUND(12, w0, w1, w2, w3);
QROUND(16); QROUND(13, w1, w2, w3, w0);
QROUND(17); QROUND(14, w2, w3, w0, w1);
QROUND(18); QROUND(15, w3, w0, w1, w2);
QROUND(19);
QROUND(16, w0, w1, w2, w3);
QROUND(17, w1, w2, w3, w0);
QROUND(18, w2, w3, w0, w1);
QROUND(19, w3, w0, w1, w2);
// update next state // update next state
state0 = _mm256_add_epi64(state0, last0); s0 = _mm256_add_epi64(s0, last0);
state1 = _mm256_add_epi64(state1, last1); s1 = _mm256_add_epi64(s1, last1);
buffer += 4;
} }
while (--count); while (--count);
// restore qword order // restore qword order
abcd = _mm256_permute2x128_si256(state1, state0, (3 << 4) | 1); abcd = _mm256_permute2x128_si256(s1, s0, 0x31);
efgh = _mm256_permute2x128_si256(state1, state0, (2 << 4) | 0); efgh = _mm256_permute2x128_si256(s1, s0, 0x20);
// save the new state // save the new state
_mm256_storeu_si256((__m256i*)&state[0], _mm256_permute4x64_epi64(abcd, _MM_SHUFFLE(0,1,2,3))); _mm256_storeu_si256((__m256i*)&state[0], _mm256_permute4x64_epi64(abcd, _MM_SHUFFLE(0,1,2,3)));
_mm256_storeu_si256((__m256i*)&state[4], _mm256_permute4x64_epi64(efgh, _MM_SHUFFLE(0,1,2,3))); _mm256_storeu_si256((__m256i*)&state[4], _mm256_permute4x64_epi64(efgh, _MM_SHUFFLE(0,1,2,3)));
#undef QROUND #undef QROUND
#undef W
} }
#endif // defined(__x86_64__) || defined(_M_AMD64) #endif // defined(__x86_64__) || defined(_M_AMD64)
@@ -337,109 +339,128 @@ static inline int sha512_cpuid(void)
SHA512_TARGET SHA512_TARGET
static void sha512_process_arm64(uint64_t* state, const uint8_t* block, size_t count) static void sha512_process_arm64(uint64_t* state, const uint8_t* block, size_t count)
{ {
#define W(i) w[(i)%8] #define DROUND(i, ab,cd,ef,gh, m0,m1,m2,m3,m4,m5,m6,m7) do { \
#define S(i) vstate.val[3-(i)%4] /* update message schedule */ \
if (i >= 8) m0 = vsha512su1q_u64(vsha512su0q_u64(m0, m1), m7, vextq_u64(m4, m5, 1)); \
#define DROUND(i) do { \ /* add round constants */ \
/* load 8 round constants */ \ uint64x2_t tmp = vaddq_u64(m0, rk.val[i%4]); \
if ((i % 4) == 0) rk = vld1q_u64_x4(&SHA512_K[2*i]); \ /* 2 round functions */ \
/* first 8 rounds reverse byte order in each 64-bit lane of input block */ \ uint64x2_t gf = vextq_u64(ef, gh, 1); \
if (i < 8) W(i) = vreinterpretq_u64_u8(vrev64q_u8(msg[(i/4)%2].val[i%4])); \ uint64x2_t ed = vextq_u64(cd, ef, 1); \
/* update message schedule for next rounds */ \ uint64x2_t x0 = vaddq_u64(vextq_u64(tmp, tmp, 1), gh); \
if (i >= 8) W(i) = vsha512su1q_u64(vsha512su0q_u64(W(i), W(i-7)), W(i-1), vextq_u64(W(i-4), W(i-3), 1)); \ uint64x2_t x1 = vsha512hq_u64(x0, gf, ed); \
/* add round constants */ \ gh = vsha512h2q_u64(x1, cd, ab); \
uint64x2_t tmp = vaddq_u64(W(i), rk.val[i%4]); \ cd = vaddq_u64(cd, x1); \
/* 2 round functions */ \ } while (0)
uint64x2_t x0 = vaddq_u64(vextq_u64(tmp, tmp, 1), S(i+0)); \
uint64x2_t x1 = vsha512hq_u64(x0, vextq_u64(S(i+1), S(i+0), 1), vextq_u64(S(i+2), S(i+1), 1)); \
S(i+0) = vsha512h2q_u64(x1, S(i+2), S(i+3)); \
S(i+2) = vaddq_u64(S(i+2), x1); \
} while (0)
// load initial state // load initial state
uint64x2x4_t vstate = vld1q_u64_x4(state); uint64x2x4_t s = vld1q_u64_x4(state);
uint64x2_t s0 = s.val[0]; // ab
uint64x2_t s1 = s.val[1]; // cd
uint64x2_t s2 = s.val[2]; // ef
uint64x2_t s3 = s.val[3]; // gh
do do
{ {
// remember current state // remember current state
uint64x2x4_t vlast = vstate; uint64x2_t last0 = s0;
uint64x2_t last1 = s1;
uint64x2_t last2 = s2;
uint64x2_t last3 = s3;
// load 128-byte block // load initial message schedule, 128-byte block
uint8x16x4_t msg[2] = uint8x16x4_t msg0 = vld1q_u8_x4(block + 0 * 16);
{ uint8x16x4_t msg1 = vld1q_u8_x4(block + 4 * 16);
vld1q_u8_x4(block + 0 * 16), block += SHA512_BLOCK_SIZE;
vld1q_u8_x4(block + 4 * 16),
}; // reverse the byte order in each 64-bit lane
uint64x2_t w0 = vreinterpretq_u64_u8(vrev64q_u8(msg0.val[0]));
uint64x2_t w1 = vreinterpretq_u64_u8(vrev64q_u8(msg0.val[1]));
uint64x2_t w2 = vreinterpretq_u64_u8(vrev64q_u8(msg0.val[2]));
uint64x2_t w3 = vreinterpretq_u64_u8(vrev64q_u8(msg0.val[3]));
uint64x2_t w4 = vreinterpretq_u64_u8(vrev64q_u8(msg1.val[0]));
uint64x2_t w5 = vreinterpretq_u64_u8(vrev64q_u8(msg1.val[1]));
uint64x2_t w6 = vreinterpretq_u64_u8(vrev64q_u8(msg1.val[2]));
uint64x2_t w7 = vreinterpretq_u64_u8(vrev64q_u8(msg1.val[3]));
uint64x2x4_t rk; uint64x2x4_t rk;
uint64x2_t w[8];
DROUND( 0); rk = vld1q_u64_x4(&SHA512_K[0]);
DROUND( 1); DROUND( 0, s0,s1,s2,s3, w0,w1,w2,w3,w4,w5,w6,w7);
DROUND( 2); DROUND( 1, s3,s0,s1,s2, w1,w2,w3,w4,w5,w6,w7,w0);
DROUND( 3); DROUND( 2, s2,s3,s0,s1, w2,w3,w4,w5,w6,w7,w0,w1);
DROUND( 3, s1,s2,s3,s0, w3,w4,w5,w6,w7,w0,w1,w2);
DROUND( 4); rk = vld1q_u64_x4(&SHA512_K[8]);
DROUND( 5); DROUND( 4, s0,s1,s2,s3, w4,w5,w6,w7,w0,w1,w2,w3);
DROUND( 6); DROUND( 5, s3,s0,s1,s2, w5,w6,w7,w0,w1,w2,w3,w4);
DROUND( 7); DROUND( 6, s2,s3,s0,s1, w6,w7,w0,w1,w2,w3,w4,w5);
DROUND( 7, s1,s2,s3,s0, w7,w0,w1,w2,w3,w4,w5,w6);
DROUND( 8); rk = vld1q_u64_x4(&SHA512_K[16]);
DROUND( 9); DROUND( 8, s0,s1,s2,s3, w0,w1,w2,w3,w4,w5,w6,w7);
DROUND(10); DROUND( 9, s3,s0,s1,s2, w1,w2,w3,w4,w5,w6,w7,w0);
DROUND(11); DROUND(10, s2,s3,s0,s1, w2,w3,w4,w5,w6,w7,w0,w1);
DROUND(11, s1,s2,s3,s0, w3,w4,w5,w6,w7,w0,w1,w2);
DROUND(12); rk = vld1q_u64_x4(&SHA512_K[24]);
DROUND(13); DROUND(12, s0,s1,s2,s3, w4,w5,w6,w7,w0,w1,w2,w3);
DROUND(14); DROUND(13, s3,s0,s1,s2, w5,w6,w7,w0,w1,w2,w3,w4);
DROUND(15); DROUND(14, s2,s3,s0,s1, w6,w7,w0,w1,w2,w3,w4,w5);
DROUND(15, s1,s2,s3,s0, w7,w0,w1,w2,w3,w4,w5,w6);
DROUND(16); rk = vld1q_u64_x4(&SHA512_K[32]);
DROUND(17); DROUND(16, s0,s1,s2,s3, w0,w1,w2,w3,w4,w5,w6,w7);
DROUND(18); DROUND(17, s3,s0,s1,s2, w1,w2,w3,w4,w5,w6,w7,w0);
DROUND(19); DROUND(18, s2,s3,s0,s1, w2,w3,w4,w5,w6,w7,w0,w1);
DROUND(19, s1,s2,s3,s0, w3,w4,w5,w6,w7,w0,w1,w2);
DROUND(20); rk = vld1q_u64_x4(&SHA512_K[40]);
DROUND(21); DROUND(20, s0,s1,s2,s3, w4,w5,w6,w7,w0,w1,w2,w3);
DROUND(22); DROUND(21, s3,s0,s1,s2, w5,w6,w7,w0,w1,w2,w3,w4);
DROUND(23); DROUND(22, s2,s3,s0,s1, w6,w7,w0,w1,w2,w3,w4,w5);
DROUND(23, s1,s2,s3,s0, w7,w0,w1,w2,w3,w4,w5,w6);
DROUND(24); rk = vld1q_u64_x4(&SHA512_K[48]);
DROUND(25); DROUND(24, s0,s1,s2,s3, w0,w1,w2,w3,w4,w5,w6,w7);
DROUND(26); DROUND(25, s3,s0,s1,s2, w1,w2,w3,w4,w5,w6,w7,w0);
DROUND(27); DROUND(26, s2,s3,s0,s1, w2,w3,w4,w5,w6,w7,w0,w1);
DROUND(27, s1,s2,s3,s0, w3,w4,w5,w6,w7,w0,w1,w2);
DROUND(28); rk = vld1q_u64_x4(&SHA512_K[56]);
DROUND(29); DROUND(28, s0,s1,s2,s3, w4,w5,w6,w7,w0,w1,w2,w3);
DROUND(30); DROUND(29, s3,s0,s1,s2, w5,w6,w7,w0,w1,w2,w3,w4);
DROUND(31); DROUND(30, s2,s3,s0,s1, w6,w7,w0,w1,w2,w3,w4,w5);
DROUND(31, s1,s2,s3,s0, w7,w0,w1,w2,w3,w4,w5,w6);
DROUND(32); rk = vld1q_u64_x4(&SHA512_K[64]);
DROUND(33); DROUND(32, s0,s1,s2,s3, w0,w1,w2,w3,w4,w5,w6,w7);
DROUND(34); DROUND(33, s3,s0,s1,s2, w1,w2,w3,w4,w5,w6,w7,w0);
DROUND(35); DROUND(34, s2,s3,s0,s1, w2,w3,w4,w5,w6,w7,w0,w1);
DROUND(35, s1,s2,s3,s0, w3,w4,w5,w6,w7,w0,w1,w2);
DROUND(36); rk = vld1q_u64_x4(&SHA512_K[72]);
DROUND(37); DROUND(36, s0,s1,s2,s3, w4,w5,w6,w7,w0,w1,w2,w3);
DROUND(38); DROUND(37, s3,s0,s1,s2, w5,w6,w7,w0,w1,w2,w3,w4);
DROUND(39); DROUND(38, s2,s3,s0,s1, w6,w7,w0,w1,w2,w3,w4,w5);
DROUND(39, s1,s2,s3,s0, w7,w0,w1,w2,w3,w4,w5,w6);
// update next state // update next state
vstate.val[0] = vaddq_u64(vstate.val[0], vlast.val[0]); s0 = vaddq_u64(s0, last0);
vstate.val[1] = vaddq_u64(vstate.val[1], vlast.val[1]); s1 = vaddq_u64(s1, last1);
vstate.val[2] = vaddq_u64(vstate.val[2], vlast.val[2]); s2 = vaddq_u64(s2, last2);
vstate.val[3] = vaddq_u64(vstate.val[3], vlast.val[3]); s3 = vaddq_u64(s3, last3);
block += SHA512_BLOCK_SIZE;
} }
while (--count); while (--count);
// save the new state // save the new state
vst1q_u64_x4(state, vstate); s.val[0] = s0;
s.val[1] = s1;
s.val[2] = s2;
s.val[3] = s3;
vst1q_u64_x4(state, s);
#undef DROUND #undef DROUND
#undef S
#undef W
} }
#endif // defined(__aarch64__) || defined(_M_ARM64) #endif // defined(__aarch64__) || defined(_M_ARM64)
@@ -464,27 +485,27 @@ static void sha512_process(uint64_t* state, const uint8_t* block, size_t count)
} }
#endif #endif
#define Ch(x,y,z) ((x & (y ^ z)) ^ z) #define Ch(x,y,z) ((x & (y ^ z)) ^ z)
#define Maj(x,y,z) ((x & y) | (z & (x | y))) #define Maj(x,y,z) ((x & y) | (z & (x | y)))
#define BSig0(x) (SHA512_ROR64(x, 28) ^ SHA512_ROR64(x, 34) ^ SHA512_ROR64(x, 39)) #define BSig0(x) (SHA512_ROR64(x, 28) ^ SHA512_ROR64(x, 34) ^ SHA512_ROR64(x, 39))
#define BSig1(x) (SHA512_ROR64(x, 14) ^ SHA512_ROR64(x, 18) ^ SHA512_ROR64(x, 41)) #define BSig1(x) (SHA512_ROR64(x, 14) ^ SHA512_ROR64(x, 18) ^ SHA512_ROR64(x, 41))
#define SSig0(x) (SHA512_ROR64(x, 1) ^ SHA512_ROR64(x, 8) ^ (x >> 7)) #define SSig0(x) (SHA512_ROR64(x, 1) ^ SHA512_ROR64(x, 8) ^ (x >> 7))
#define SSig1(x) (SHA512_ROR64(x, 19) ^ SHA512_ROR64(x, 61) ^ (x >> 6)) #define SSig1(x) (SHA512_ROR64(x, 19) ^ SHA512_ROR64(x, 61) ^ (x >> 6))
#define W(i) w[(i+16)%16] #define W(i) w[(i+16)%16]
#define ROUND(i,a,b,c,d,e,f,g,h) do \ #define ROUND(i,a,b,c,d,e,f,g,h) do \
{ \ { \
uint64_t w0; \ uint64_t w0; \
if (i < 16) W(i) = w0 = SHA512_GET64BE(block + i*sizeof(uint64_t)); \ if (i < 16) W(i) = w0 = SHA512_GET64BE(block + i*sizeof(uint64_t)); \
if (i >= 16) W(i) = w0 = SSig1(W(i-2)) + W(i-7) + SSig0(W(i-15)) + W(i-16); \ if (i >= 16) W(i) = w0 = SSig1(W(i-2)) + W(i-7) + SSig0(W(i-15)) + W(i-16); \
\ \
uint64_t t1 = h + BSig1(e) + Ch(e,f,g) + SHA512_K[i] + w0; \ uint64_t t1 = h + BSig1(e) + Ch(e,f,g) + SHA512_K[i] + w0; \
uint64_t t2 = BSig0(a) + Maj(a,b,c); \ uint64_t t2 = BSig0(a) + Maj(a,b,c); \
d += t1; \ d += t1; \
h = t1 + t2; \ h = t1 + t2; \
} while (0) } while (0)
do do
{ {
@@ -507,6 +528,7 @@ h = t1 + t2; \
ROUND( 5, d, e, f, g, h, a, b, c); ROUND( 5, d, e, f, g, h, a, b, c);
ROUND( 6, c, d, e, f, g, h, a, b); ROUND( 6, c, d, e, f, g, h, a, b);
ROUND( 7, b, c, d, e, f, g, h, a); ROUND( 7, b, c, d, e, f, g, h, a);
ROUND( 8, a, b, c, d, e, f, g, h); ROUND( 8, a, b, c, d, e, f, g, h);
ROUND( 9, h, a, b, c, d, e, f, g); ROUND( 9, h, a, b, c, d, e, f, g);
ROUND(10, g, h, a, b, c, d, e, f); ROUND(10, g, h, a, b, c, d, e, f);
@@ -515,6 +537,7 @@ h = t1 + t2; \
ROUND(13, d, e, f, g, h, a, b, c); ROUND(13, d, e, f, g, h, a, b, c);
ROUND(14, c, d, e, f, g, h, a, b); ROUND(14, c, d, e, f, g, h, a, b);
ROUND(15, b, c, d, e, f, g, h, a); ROUND(15, b, c, d, e, f, g, h, a);
ROUND(16, a, b, c, d, e, f, g, h); ROUND(16, a, b, c, d, e, f, g, h);
ROUND(17, h, a, b, c, d, e, f, g); ROUND(17, h, a, b, c, d, e, f, g);
ROUND(18, g, h, a, b, c, d, e, f); ROUND(18, g, h, a, b, c, d, e, f);
@@ -523,6 +546,7 @@ h = t1 + t2; \
ROUND(21, d, e, f, g, h, a, b, c); ROUND(21, d, e, f, g, h, a, b, c);
ROUND(22, c, d, e, f, g, h, a, b); ROUND(22, c, d, e, f, g, h, a, b);
ROUND(23, b, c, d, e, f, g, h, a); ROUND(23, b, c, d, e, f, g, h, a);
ROUND(24, a, b, c, d, e, f, g, h); ROUND(24, a, b, c, d, e, f, g, h);
ROUND(25, h, a, b, c, d, e, f, g); ROUND(25, h, a, b, c, d, e, f, g);
ROUND(26, g, h, a, b, c, d, e, f); ROUND(26, g, h, a, b, c, d, e, f);
@@ -531,6 +555,7 @@ h = t1 + t2; \
ROUND(29, d, e, f, g, h, a, b, c); ROUND(29, d, e, f, g, h, a, b, c);
ROUND(30, c, d, e, f, g, h, a, b); ROUND(30, c, d, e, f, g, h, a, b);
ROUND(31, b, c, d, e, f, g, h, a); ROUND(31, b, c, d, e, f, g, h, a);
ROUND(32, a, b, c, d, e, f, g, h); ROUND(32, a, b, c, d, e, f, g, h);
ROUND(33, h, a, b, c, d, e, f, g); ROUND(33, h, a, b, c, d, e, f, g);
ROUND(34, g, h, a, b, c, d, e, f); ROUND(34, g, h, a, b, c, d, e, f);
@@ -539,6 +564,7 @@ h = t1 + t2; \
ROUND(37, d, e, f, g, h, a, b, c); ROUND(37, d, e, f, g, h, a, b, c);
ROUND(38, c, d, e, f, g, h, a, b); ROUND(38, c, d, e, f, g, h, a, b);
ROUND(39, b, c, d, e, f, g, h, a); ROUND(39, b, c, d, e, f, g, h, a);
ROUND(40, a, b, c, d, e, f, g, h); ROUND(40, a, b, c, d, e, f, g, h);
ROUND(41, h, a, b, c, d, e, f, g); ROUND(41, h, a, b, c, d, e, f, g);
ROUND(42, g, h, a, b, c, d, e, f); ROUND(42, g, h, a, b, c, d, e, f);
@@ -547,6 +573,7 @@ h = t1 + t2; \
ROUND(45, d, e, f, g, h, a, b, c); ROUND(45, d, e, f, g, h, a, b, c);
ROUND(46, c, d, e, f, g, h, a, b); ROUND(46, c, d, e, f, g, h, a, b);
ROUND(47, b, c, d, e, f, g, h, a); ROUND(47, b, c, d, e, f, g, h, a);
ROUND(48, a, b, c, d, e, f, g, h); ROUND(48, a, b, c, d, e, f, g, h);
ROUND(49, h, a, b, c, d, e, f, g); ROUND(49, h, a, b, c, d, e, f, g);
ROUND(50, g, h, a, b, c, d, e, f); ROUND(50, g, h, a, b, c, d, e, f);
@@ -555,6 +582,7 @@ h = t1 + t2; \
ROUND(53, d, e, f, g, h, a, b, c); ROUND(53, d, e, f, g, h, a, b, c);
ROUND(54, c, d, e, f, g, h, a, b); ROUND(54, c, d, e, f, g, h, a, b);
ROUND(55, b, c, d, e, f, g, h, a); ROUND(55, b, c, d, e, f, g, h, a);
ROUND(56, a, b, c, d, e, f, g, h); ROUND(56, a, b, c, d, e, f, g, h);
ROUND(57, h, a, b, c, d, e, f, g); ROUND(57, h, a, b, c, d, e, f, g);
ROUND(58, g, h, a, b, c, d, e, f); ROUND(58, g, h, a, b, c, d, e, f);
@@ -563,6 +591,7 @@ h = t1 + t2; \
ROUND(61, d, e, f, g, h, a, b, c); ROUND(61, d, e, f, g, h, a, b, c);
ROUND(62, c, d, e, f, g, h, a, b); ROUND(62, c, d, e, f, g, h, a, b);
ROUND(63, b, c, d, e, f, g, h, a); ROUND(63, b, c, d, e, f, g, h, a);
ROUND(64, a, b, c, d, e, f, g, h); ROUND(64, a, b, c, d, e, f, g, h);
ROUND(65, h, a, b, c, d, e, f, g); ROUND(65, h, a, b, c, d, e, f, g);
ROUND(66, g, h, a, b, c, d, e, f); ROUND(66, g, h, a, b, c, d, e, f);
@@ -571,6 +600,7 @@ h = t1 + t2; \
ROUND(69, d, e, f, g, h, a, b, c); ROUND(69, d, e, f, g, h, a, b, c);
ROUND(70, c, d, e, f, g, h, a, b); ROUND(70, c, d, e, f, g, h, a, b);
ROUND(71, b, c, d, e, f, g, h, a); ROUND(71, b, c, d, e, f, g, h, a);
ROUND(72, a, b, c, d, e, f, g, h); ROUND(72, a, b, c, d, e, f, g, h);
ROUND(73, h, a, b, c, d, e, f, g); ROUND(73, h, a, b, c, d, e, f, g);
ROUND(74, g, h, a, b, c, d, e, f); ROUND(74, g, h, a, b, c, d, e, f);
@@ -593,14 +623,14 @@ h = t1 + t2; \
} }
while (--count); while (--count);
#undef ROUND #undef ROUND
#undef W #undef W
#undef Ch #undef Ch
#undef Maj #undef Maj
#undef BSig0 #undef BSig0
#undef BSig1 #undef BSig1
#undef SSig0 #undef SSig0
#undef SSig1 #undef SSig1
} }
void sha512_init(sha512_ctx* ctx) void sha512_init(sha512_ctx* ctx)