From 08f794527cb721e290082f3b60947b62a4a5f150 Mon Sep 17 00:00:00 2001 From: Nikita Smith Date: Thu, 13 Nov 2025 11:45:34 -0800 Subject: [PATCH] updated hashing libraries --- src/third_party/martins_hash/sha1.h | 236 +++--- src/third_party/martins_hash/sha256.h | 195 ++--- src/third_party/martins_hash/sha512.h | 1050 +++++++++++++------------ 3 files changed, 748 insertions(+), 733 deletions(-) diff --git a/src/third_party/martins_hash/sha1.h b/src/third_party/martins_hash/sha1.h index 5fd59362..b61e36dd 100644 --- a/src/third_party/martins_hash/sha1.h +++ b/src/third_party/martins_hash/sha1.h @@ -137,17 +137,17 @@ static inline int sha1_cpuid(void) SHA1_TARGET("ssse3,sha") static void sha1_process_shani(uint32_t* state, const uint8_t* block, size_t count) { - // in SHA1 each round has two parts: + // in sha1 each round has two parts: // 1) calculate message schedule dwords in w[i] // 2) do round functions to update a/b/c/d/e state values using w[i] - // w[i] in first 16 rounds is just loaded from block bytes, as 32-bit big-endian load + // initial message schedule is loaded as 32-bit big-endian 16 dwords // for next rounds it is done as: // w[i] = ROL(w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) // where ROL(x) = 32-bit rotate left by 1 - // this means it is possible to keep just the last 16 of w's in circular buffer + // this means it is possible to keep just the last 16 of w's // and every new w calculated will need to update 1 to 3 previous w's // unrolling round calculations by 4 we get: @@ -156,7 +156,7 @@ static void sha1_process_shani(uint32_t* state, const uint8_t* block, size_t cou // w[i+2] = ROL(w[i-1] ^ w[i-6] ^ w[i-12] ^ w[i-14]) // w[i+3] = ROL(w[i+0] ^ w[i-5] ^ w[i-11] ^ w[i-13]) - // now if you store 4 w[..] values in 128-bit SSE register, then + // now use 4 w[..] values in 128-bit SSE register W(i), then // W(i) = ROL( r0 ^ r1 ^ r2 ^ r3 ) // with caveat that r0 lane 3 depends on W(i) lane 0 @@ -166,43 +166,31 @@ static void sha1_process_shani(uint32_t* state, const uint8_t* block, size_t cou // r2 = [ w[i-11], w[i-12], w[i-13], w[i-14] ] // r3 = [ w[i-13], w[i-14], w[i-15], w[i-16] ] - // in each 4-round i'th step it is possible to incrementally update new W(..) value when - // keeping W(i) values in 4 xmm element circular buffer + // in each 4-round i'th step it is possible to incrementally update W's that will be + // used in later rounds - // rounds i>0: W(i-1) = r2 ^ r3 = _mm_sha1msg1_epu32(W(i-1), W(i)) - // rounds i>1: W(i-2) = W(i-2) ^ r1 = _mm_xor_si128 (W(i-2), W(i)) - // rounds i>2: W(i-3) = ROL(W(i-3) ^ r0) = _mm_sha1msg2_epu32(W(i-3), W(i)) - // then the new W(i) can be used in round function calculations - // _mm_sha1msg2_epu32 correctly handles r0 lane 3 dependency on W(i) lane 0 + // rounds i>0: m3 = r2 ^ r3 = _mm_sha1msg1_epu32(m3, m0) + // rounds i>1: m2 = m2 ^ r1 = _mm_xor_si128 (m2, m0) + // rounds i>2: m1 = ROL(m1 ^ r0) = _mm_sha1msg2_epu32(m1, m0) + // then the new m0 can be used in round function calculations + // _mm_sha1msg2_epu32 correctly handles r0 lane 3 dependency on lane 0 output // to perform round functions on two SIMD registers with state as: - // abcd = [a,b,c,d] - // e0 = [e,0,0,0] - // use the following code to get next abcd/e0 state 4 rounds at a time: - - // tmp = _mm_sha1nexte_epu32(e0, W(i)) // rotates e0 and adds message dwords - // abcd_next = _mm_sha1rnds4_epu32(abcd, tmp, Fn) // with Fn = 0..3 round function selection - // e0_next = abcd - - // sha1nexte is not needed on first round, just regular add32(e0, W(i)) should be used - // after last round need to do extra rotation, which sha1nexte takes care when adding to last_e0 - - #define W(i) w[(i)%4] + // s0 = [a,b,c,d] + // s1 = [e,0,0,0] + // use _mm_sha1rnds4_epu32 and _mm_sha1nexte_epu32 ops and swap both state variables between rounds // 4 wide round calculations - #define QROUND(i) do { \ - /* first 4 rounds load input block */ \ - if (i < 4) W(i) = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[i]), bswap); \ - /* update message schedule */ \ - if (i > 0 && i < 17) W(i-1) = _mm_sha1msg1_epu32(W(i-1), W(i)); \ - if (i > 1 && i < 18) W(i-2) = _mm_xor_si128 (W(i-2), W(i)); \ - if (i > 2 && i < 19) W(i-3) = _mm_sha1msg2_epu32(W(i-3), W(i)); \ - /* calculate E plus message schedule */ \ - if (i == 0) tmp = _mm_add_epi32 (e0, W(i)); \ - if (i != 0) tmp = _mm_sha1nexte_epu32(e0, W(i)); \ - /* 4 round functions */ \ - e0 = abcd; \ - abcd = _mm_sha1rnds4_epu32(abcd, tmp, i/5); \ + #define QROUND(i,s0,s1,m0,m1,m2,m3) do { \ + /* update message schedule */ \ + if (i > 0 && i < 17) m3 = _mm_sha1msg1_epu32(m3, m0); \ + if (i > 1 && i < 18) m2 = _mm_xor_si128 (m2, m0); \ + if (i > 2 && i < 19) m1 = _mm_sha1msg2_epu32(m1, m0); \ + /* calculate E plus message schedule */ \ + if (i == 0) tmp = _mm_add_epi32 (s1, m0); \ + if (i != 0) tmp = _mm_sha1nexte_epu32(s1, m0); \ + /* 4 round functions */ \ + s1 = _mm_sha1rnds4_epu32(s0, tmp, i/5); \ } while(0) const __m128i* buffer = (const __m128i*)block; @@ -213,63 +201,67 @@ static void sha1_process_shani(uint32_t* state, const uint8_t* block, size_t cou const __m128i bswap = _mm_setr_epi8(15,14,13,12, 11,10,9,8, 7,6,5,4, 3,2,1,0); // load initial state - __m128i abcd = _mm_loadu_si128((const __m128i*)state); // [d,c,b,a] - __m128i e0 = _mm_loadu_si32(&state[4]); // [0,0,0,e] + __m128i s0 = _mm_loadu_si128((const __m128i*)state); // [d,c,b,a] + __m128i s1 = _mm_loadu_si32(&state[4]); // [0,0,0,e] // flip dword order, to what sha1 instructions use - abcd = _mm_shuffle_epi32(abcd, _MM_SHUFFLE(0,1,2,3)); // [a,b,c,d] where a is in the top lane - e0 = _mm_slli_si128(e0, 12); // [e,0,0,0] where e is in top lane + s0 = _mm_shuffle_epi32(s0, _MM_SHUFFLE(0,1,2,3)); // [a,b,c,d] + s1 = _mm_shuffle_epi32(s1, _MM_SHUFFLE(0,1,2,3)); // [e,0,0,0] do { // remember current state - __m128i last_abcd = abcd; - __m128i last_e0 = e0; + __m128i last0 = s0; + __m128i last1 = s1; - __m128i tmp, w[4]; + // load initial message schedule, 64-byte block + __m128i w0 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[0]), bswap); + __m128i w1 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[1]), bswap); + __m128i w2 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[2]), bswap); + __m128i w3 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[3]), bswap); + buffer += 4; - QROUND(0); - QROUND(1); - QROUND(2); - QROUND(3); - QROUND(4); + __m128i tmp; - QROUND(5); - QROUND(6); - QROUND(7); - QROUND(8); - QROUND(9); + QROUND( 0, s0, s1, w0, w1, w2, w3); + QROUND( 1, s1, s0, w1, w2, w3, w0); + QROUND( 2, s0, s1, w2, w3, w0, w1); + QROUND( 3, s1, s0, w3, w0, w1, w2); + QROUND( 4, s0, s1, w0, w1, w2, w3); - QROUND(10); - QROUND(11); - QROUND(12); - QROUND(13); - QROUND(14); + QROUND( 5, s1, s0, w1, w2, w3, w0); + QROUND( 6, s0, s1, w2, w3, w0, w1); + QROUND( 7, s1, s0, w3, w0, w1, w2); + QROUND( 8, s0, s1, w0, w1, w2, w3); + QROUND( 9, s1, s0, w1, w2, w3, w0); - QROUND(15); - QROUND(16); - QROUND(17); - QROUND(18); - QROUND(19); + QROUND(10, s0, s1, w2, w3, w0, w1); + QROUND(11, s1, s0, w3, w0, w1, w2); + QROUND(12, s0, s1, w0, w1, w2, w3); + QROUND(13, s1, s0, w1, w2, w3, w0); + QROUND(14, s0, s1, w2, w3, w0, w1); + + QROUND(15, s1, s0, w3, w0, w1, w2); + QROUND(16, s0, s1, w0, w1, w2, w3); + QROUND(17, s1, s0, w1, w2, w3, w0); + QROUND(18, s0, s1, w2, w3, w0, w1); + QROUND(19, s1, s0, w3, w0, w1, w2); // update next state - abcd = _mm_add_epi32(abcd, last_abcd); - e0 = _mm_sha1nexte_epu32(e0, last_e0); - - buffer += 4; + s0 = _mm_add_epi32 (s0, last0); + s1 = _mm_sha1nexte_epu32(s1, last1); } while (--count); // restore dword order - abcd = _mm_shuffle_epi32(abcd, _MM_SHUFFLE(0,1,2,3)); - e0 = _mm_shuffle_epi32(e0, _MM_SHUFFLE(0,1,2,3)); + s0 = _mm_shuffle_epi32(s0, _MM_SHUFFLE(0,1,2,3)); + s1 = _mm_shuffle_epi32(s1, _MM_SHUFFLE(0,1,2,3)); // save the new state - _mm_storeu_si128((__m128i*)state, abcd); - _mm_storeu_si32(&state[4], e0); + _mm_storeu_si128((__m128i*)state, s0); + _mm_storeu_si32(&state[4], s1); #undef QROUND - #undef W } #endif // defined(__x86_64__) || defined(_M_AMD64) @@ -343,21 +335,16 @@ static void sha1_process_arm64(uint32_t* state, const uint8_t* block, size_t cou { // code here is similar to x64 shani implementation - // message array is 16 element circular buffer - // each iteration updates 4 rounds at the same time - - #define W(i) w[(i)%4] - - #define QROUND(i,F,k) do { \ - /* update message schedule */ \ - if (i >= 4) W(i) = vsha1su0q_u32(W(i), W(i-3), W(i-2)); \ - if (i >= 4) W(i) = vsha1su1q_u32(W(i), W(i-1)); \ - /* add round constant */ \ - uint32x4_t tmp = vaddq_u32(W(i), k); \ - /* 4 round functions */ \ - uint32_t x = e0; \ - e0 = vsha1h_u32(vgetq_lane_u32(abcd, 0)); \ - abcd = F(abcd, x, tmp); \ + #define QROUND(i,m0,m1,m2,m3,k,F) do { \ + /* update message schedule */ \ + if (i >= 4) m0 = vsha1su0q_u32(m0, m1, m2); \ + if (i >= 4) m0 = vsha1su1q_u32(m0, m3); \ + /* add round constant */ \ + uint32x4_t tmp = vaddq_u32(m0, k); \ + /* 4 round functions */ \ + uint32_t e = vgetq_lane_u32(s0, 0); \ + s0 = F(s0, s1, tmp); \ + s1 = vsha1h_u32(e); \ } while (0) const uint32x4_t k0 = vdupq_n_u32(0x5a827999); @@ -365,64 +352,61 @@ static void sha1_process_arm64(uint32_t* state, const uint8_t* block, size_t cou const uint32x4_t k2 = vdupq_n_u32(0x8f1bbcdc); const uint32x4_t k3 = vdupq_n_u32(0xca62c1d6); - // load state - a,b,c,d,e - uint32x4_t abcd = vld1q_u32(state); - uint32_t e0 = state[4]; + // load initial state + uint32x4_t s0 = vld1q_u32(state); // [d,c,b,a] + uint32_t s1 = state[4]; // e do { // remember current state - uint32x4_t last_abcd = abcd; - uint32_t last_e0 = e0; + uint32x4_t last0 = s0; + uint32_t last1 = s1; - // load 64-byte block and advance pointer to next block + // load initial message schedule, 64-byte block uint8x16x4_t msg = vld1q_u8_x4(block); block += SHA1_BLOCK_SIZE; - uint32x4_t w[4]; + // reverse the byte order in each 32-bit lane + uint32x4_t w0 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[0])); + uint32x4_t w1 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[1])); + uint32x4_t w2 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[2])); + uint32x4_t w3 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[3])); - // for first 16 w's reverse the byte order in each 32-bit lane - W(0) = vreinterpretq_u32_u8(vrev32q_u8(msg.val[0])); - W(1) = vreinterpretq_u32_u8(vrev32q_u8(msg.val[1])); - W(2) = vreinterpretq_u32_u8(vrev32q_u8(msg.val[2])); - W(3) = vreinterpretq_u32_u8(vrev32q_u8(msg.val[3])); + QROUND( 0, w0, w1, w2, w3, k0, vsha1cq_u32); + QROUND( 1, w1, w2, w3, w0, k0, vsha1cq_u32); + QROUND( 2, w2, w3, w0, w1, k0, vsha1cq_u32); + QROUND( 3, w3, w0, w1, w2, k0, vsha1cq_u32); + QROUND( 4, w0, w1, w2, w3, k0, vsha1cq_u32); - QROUND( 0, vsha1cq_u32, k0); - QROUND( 1, vsha1cq_u32, k0); - QROUND( 2, vsha1cq_u32, k0); - QROUND( 3, vsha1cq_u32, k0); - QROUND( 4, vsha1cq_u32, k0); + QROUND( 5, w1, w2, w3, w0, k1, vsha1pq_u32); + QROUND( 6, w2, w3, w0, w1, k1, vsha1pq_u32); + QROUND( 7, w3, w0, w1, w2, k1, vsha1pq_u32); + QROUND( 8, w0, w1, w2, w3, k1, vsha1pq_u32); + QROUND( 9, w1, w2, w3, w0, k1, vsha1pq_u32); - QROUND( 5, vsha1pq_u32, k1); - QROUND( 6, vsha1pq_u32, k1); - QROUND( 7, vsha1pq_u32, k1); - QROUND( 8, vsha1pq_u32, k1); - QROUND( 9, vsha1pq_u32, k1); + QROUND(10, w2, w3, w0, w1, k2, vsha1mq_u32); + QROUND(11, w3, w0, w1, w2, k2, vsha1mq_u32); + QROUND(12, w0, w1, w2, w3, k2, vsha1mq_u32); + QROUND(13, w1, w2, w3, w0, k2, vsha1mq_u32); + QROUND(14, w2, w3, w0, w1, k2, vsha1mq_u32); - QROUND(10, vsha1mq_u32, k2); - QROUND(11, vsha1mq_u32, k2); - QROUND(12, vsha1mq_u32, k2); - QROUND(13, vsha1mq_u32, k2); - QROUND(14, vsha1mq_u32, k2); - - QROUND(15, vsha1pq_u32, k3); - QROUND(16, vsha1pq_u32, k3); - QROUND(17, vsha1pq_u32, k3); - QROUND(18, vsha1pq_u32, k3); - QROUND(19, vsha1pq_u32, k3); + QROUND(15, w3, w0, w1, w2, k3, vsha1pq_u32); + QROUND(16, w0, w1, w2, w3, k3, vsha1pq_u32); + QROUND(17, w1, w2, w3, w0, k3, vsha1pq_u32); + QROUND(18, w2, w3, w0, w1, k3, vsha1pq_u32); + QROUND(19, w3, w0, w1, w2, k3, vsha1pq_u32); // update next state - abcd = vaddq_u32(abcd, last_abcd); - e0 += last_e0; + s0 = vaddq_u32(s0, last0); + s1 += last1; } while (--count); // save state - vst1q_u32(state, abcd); - state[4] = e0; + vst1q_u32(state, s0); + state[4] = s1; #undef QROUND - #undef W } #endif // defined(__aarch64__) || defined(_M_ARM64) @@ -452,7 +436,7 @@ static void sha1_process(uint32_t* state, const uint8_t* block, size_t count) #define F3(x,y,z) (0x8f1bbcdc + ((x & y) | (z & (x | y)))) #define F4(x,y,z) (0xca62c1d6 + (x ^ y ^ z)) - #define W(i) w[(i)%16] + #define W(i) w[(i+16)%16] #define ROUND(i,a,b,c,d,e,F) do \ { \ diff --git a/src/third_party/martins_hash/sha256.h b/src/third_party/martins_hash/sha256.h index b70a72a0..56cbc5c5 100644 --- a/src/third_party/martins_hash/sha256.h +++ b/src/third_party/martins_hash/sha256.h @@ -167,9 +167,7 @@ static void sha256_process_shani(uint32_t* state, const uint8_t* block, size_t c { // similar way how sha1 works in with shani - // first 16 rounds loads message schedule dwords as 32-bit big endian values - - // for next rounds message schedule is prepared as: + // rounds message schedule is updated as: // w[i] = SSig1(w[i-2]) + w[i-7] + SSig0(w[i-15]) + w[i-16] // unrolled by 4: @@ -190,32 +188,22 @@ static void sha256_process_shani(uint32_t* state, const uint8_t* block, size_t c // r2 = [ w[i-12], w[i-13], w[i-14], w[i-15] ] // r3 = [ w[i-13], w[i-14], w[i-15], w[i-16] ] - // rN's can be calculated from previous W(..) values: - // r0 from W(i) - // r1 from _mm_alignr_epi8(W(i), W(i-1), 4) - // r2 from W(i-1) and W(i) - // r3 from W(i-1) - - // rounds i>2: W(i-3) = _mm_sha256msg2_epu32(_mm_add_epi32( W(i-3), _mm_alignr_epi8(W(i), W(i-1), 4) ), W(i)) - // rounds i>0: W(i-1) = _mm_sha256msg1_epu32(W(i-1), W(i)) + // rounds i>2: m1 = _mm_sha256msg2_epu32(_mm_add_epi32(m1, _mm_alignr_epi8(m0, m3, 4) ), m0) + // rounds i>0: m3 = _mm_sha256msg1_epu32(m3, m0) // round functions are done with _mm_sha256rnds2_epu32 which performs it for 2 rounds - // thus repeat it two times, as input use W(i) + K(i) - message schedule added with sha256 constants - - #define W(i) w[(i)%4] + // thus repeat it two times, as input use m0 + K(i) - message schedule added with sha256 constants // 4 wide round calculations - #define QROUND(i) do { \ - /* first 4 rounds load input block */ \ - if (i < 4) W(i) = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[i]), bswap); \ - /* update message schedule */ \ - if (i > 2 && i < 15) W(i-3) = _mm_sha256msg2_epu32(_mm_add_epi32(W(i-3), _mm_alignr_epi8(W(i), W(i-1), 4)), W(i)); \ - if (i > 0 && i < 13) W(i-1) = _mm_sha256msg1_epu32(W(i-1), W(i)); \ - /* add round constants */ \ - __m128i tmp = _mm_add_epi32(W(i), _mm_loadu_si128((const __m128i*)&SHA256_K[4*i])); \ - /* 4 round functions */ \ - state1 = _mm_sha256rnds2_epu32(state1, state0, tmp); \ - state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(tmp, _MM_SHUFFLE(0,0,3,2))); \ + #define QROUND(i,m0,m1,m2,m3) do { \ + /* update message schedule */ \ + if (i > 2 && i < 15) m1 = _mm_sha256msg2_epu32(_mm_add_epi32(m1, _mm_alignr_epi8(m0, m3, 4)), m0); \ + if (i > 0 && i < 13) m3 = _mm_sha256msg1_epu32(m3, m0); \ + /* add round constants */ \ + __m128i tmp = _mm_add_epi32(m0, _mm_loadu_si128((const __m128i*)&SHA256_K[4*i])); \ + /* 4 round functions */ \ + s1 = _mm_sha256rnds2_epu32(s1, s0, tmp); \ + s0 = _mm_sha256rnds2_epu32(s0, s1, _mm_shuffle_epi32(tmp, _MM_SHUFFLE(0,0,3,2))); \ } while(0) const __m128i* buffer = (const __m128i*)block; @@ -228,52 +216,57 @@ static void sha256_process_shani(uint32_t* state, const uint8_t* block, size_t c __m128i efgh = _mm_shuffle_epi32(_mm_loadu_si128((const __m128i*)&state[4]), _MM_SHUFFLE(0,1,2,3)); // [e,f,g,h] // dword order for sha256rnds2 instruction - __m128i state0 = _mm_unpackhi_epi64(efgh, abcd); // [a,b,e,f] - __m128i state1 = _mm_unpacklo_epi64(efgh, abcd); // [c,d,g,h] + __m128i s0 = _mm_unpackhi_epi64(efgh, abcd); // [a,b,e,f] + __m128i s1 = _mm_unpacklo_epi64(efgh, abcd); // [c,d,g,h] do { // remember current state - __m128i last0 = state0; - __m128i last1 = state1; + __m128i last0 = s0; + __m128i last1 = s1; - __m128i w[4]; + // load initial message schedule, 64-byte block + __m128i w0 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[0]), bswap); + __m128i w1 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[1]), bswap); + __m128i w2 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[2]), bswap); + __m128i w3 = _mm_shuffle_epi8(_mm_loadu_si128(&buffer[3]), bswap); + buffer += 4; - QROUND( 0); - QROUND( 1); - QROUND( 2); - QROUND( 3); - QROUND( 4); - QROUND( 5); - QROUND( 6); - QROUND( 7); - QROUND( 8); - QROUND( 9); - QROUND(10); - QROUND(11); - QROUND(12); - QROUND(13); - QROUND(14); - QROUND(15); + QROUND( 0, w0, w1, w2, w3); + QROUND( 1, w1, w2, w3, w0); + QROUND( 2, w2, w3, w0, w1); + QROUND( 3, w3, w0, w1, w2); + + QROUND( 4, w0, w1, w2, w3); + QROUND( 5, w1, w2, w3, w0); + QROUND( 6, w2, w3, w0, w1); + QROUND( 7, w3, w0, w1, w2); + + QROUND( 8, w0, w1, w2, w3); + QROUND( 9, w1, w2, w3, w0); + QROUND(10, w2, w3, w0, w1); + QROUND(11, w3, w0, w1, w2); + + QROUND(12, w0, w1, w2, w3); + QROUND(13, w1, w2, w3, w0); + QROUND(14, w2, w3, w0, w1); + QROUND(15, w3, w0, w1, w2); // update next state - state0 = _mm_add_epi32(state0, last0); - state1 = _mm_add_epi32(state1, last1); - - buffer += 4; + s0 = _mm_add_epi32(s0, last0); + s1 = _mm_add_epi32(s1, last1); } while (--count); // restore dword order - abcd = _mm_unpackhi_epi64(state1, state0); - efgh = _mm_unpacklo_epi64(state1, state0); + abcd = _mm_unpackhi_epi64(s1, s0); + efgh = _mm_unpacklo_epi64(s1, s0); // save the new state _mm_storeu_si128((__m128i*)&state[0], _mm_shuffle_epi32(abcd, _MM_SHUFFLE(0,1,2,3))); _mm_storeu_si128((__m128i*)&state[4], _mm_shuffle_epi32(efgh, _MM_SHUFFLE(0,1,2,3))); #undef QROUND - #undef W } #endif // defined(__x86_64__) || defined(_M_AMD64) @@ -344,70 +337,71 @@ static inline int sha256_cpuid(void) SHA256_TARGET static void sha256_process_arm64(uint32_t* state, const uint8_t* block, size_t count) { - // code here is similar to x64 shani implementation - - #define W(i) w[(i)%4] - - #define QROUND(i) do { \ - /* load 16 round constants */ \ - if ((i % 4) == 0) rk = vld1q_u32_x4(&SHA256_K[4*i]); \ - /* first 4 rounds reverse byte order in each 32-bit lane of input block */ \ - if (i < 4) W(i) = vreinterpretq_u32_u8(vrev32q_u8(msg.val[i])); \ - /* update message schedule */ \ - if (i >= 4) W(i) = vsha256su0q_u32(W(i), W(i-3)); \ - if (i >= 4) W(i) = vsha256su1q_u32(W(i), W(i-2), W(i-1)); \ - /* add round constants */ \ - uint32x4_t tmp = vaddq_u32(W(i), rk.val[i%4]); \ - /* 4 round functions */ \ - uint32x4_t x = vstate.val[0]; \ - vstate.val[0] = vsha256hq_u32(vstate.val[0], vstate.val[1], tmp); \ - vstate.val[1] = vsha256h2q_u32(vstate.val[1], x, tmp); \ + #define QROUND(i,m0,m1,m2,m3) do { \ + /* update message schedule */ \ + if (i >= 4) m0 = vsha256su1q_u32(vsha256su0q_u32(m0, m1), m2, m3); \ + /* add round constants */ \ + uint32x4_t tmp = vaddq_u32(m0, rk.val[i%4]); \ + /* 4 round functions */ \ + uint32x4x2_t x = s; \ + s.val[0] = vsha256hq_u32(x.val[0], x.val[1], tmp); \ + s.val[1] = vsha256h2q_u32(x.val[1], x.val[0], tmp); \ } while (0) // load initial state - uint32x4x2_t vstate = vld1q_u32_x2(state); + uint32x4x2_t s = vld1q_u32_x2(state); do { // remember current state - uint32x4x2_t vlast = vstate; + uint32x4x2_t last = s; - // load 64-byte block + // load initial message schedule, 64-byte block uint8x16x4_t msg = vld1q_u8_x4(block); + block += SHA256_BLOCK_SIZE; + + // reverse the byte order in each 32-bit lane + uint32x4_t w0 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[0])); + uint32x4_t w1 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[1])); + uint32x4_t w2 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[2])); + uint32x4_t w3 = vreinterpretq_u32_u8(vrev32q_u8(msg.val[3])); uint32x4x4_t rk; - uint32x4_t w[4]; - QROUND( 0); - QROUND( 1); - QROUND( 2); - QROUND( 3); - QROUND( 4); - QROUND( 5); - QROUND( 6); - QROUND( 7); - QROUND( 8); - QROUND( 9); - QROUND(10); - QROUND(11); - QROUND(12); - QROUND(13); - QROUND(14); - QROUND(15); + rk = vld1q_u32_x4(&SHA256_K[0]); + QROUND( 0, w0, w1, w2, w3); + QROUND( 1, w1, w2, w3, w0); + QROUND( 2, w2, w3, w0, w1); + QROUND( 3, w3, w0, w1, w2); + + rk = vld1q_u32_x4(&SHA256_K[16]); + QROUND( 4, w0, w1, w2, w3); + QROUND( 5, w1, w2, w3, w0); + QROUND( 6, w2, w3, w0, w1); + QROUND( 7, w3, w0, w1, w2); + + rk = vld1q_u32_x4(&SHA256_K[32]); + QROUND( 8, w0, w1, w2, w3); + QROUND( 9, w1, w2, w3, w0); + QROUND(10, w2, w3, w0, w1); + QROUND(11, w3, w0, w1, w2); + + rk = vld1q_u32_x4(&SHA256_K[48]); + QROUND(12, w0, w1, w2, w3); + QROUND(13, w1, w2, w3, w0); + QROUND(14, w2, w3, w0, w1); + QROUND(15, w3, w0, w1, w2); // update next state - vstate.val[0] = vaddq_u32(vstate.val[0], vlast.val[0]); - vstate.val[1] = vaddq_u32(vstate.val[1], vlast.val[1]); - - block += SHA256_BLOCK_SIZE; + s.val[0] = vaddq_u32(s.val[0], last.val[0]); + s.val[1] = vaddq_u32(s.val[1], last.val[1]); } while (--count); // save the new state - vst1q_u32_x2(state, vstate); + vst1q_u32_x2(state, s); #undef QROUND - #undef W } #endif // defined(__aarch64__) || defined(_M_ARM64) @@ -475,6 +469,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count) ROUND( 5, d, e, f, g, h, a, b, c); ROUND( 6, c, d, e, f, g, h, a, b); ROUND( 7, b, c, d, e, f, g, h, a); + ROUND( 8, a, b, c, d, e, f, g, h); ROUND( 9, h, a, b, c, d, e, f, g); ROUND(10, g, h, a, b, c, d, e, f); @@ -483,6 +478,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count) ROUND(13, d, e, f, g, h, a, b, c); ROUND(14, c, d, e, f, g, h, a, b); ROUND(15, b, c, d, e, f, g, h, a); + ROUND(16, a, b, c, d, e, f, g, h); ROUND(17, h, a, b, c, d, e, f, g); ROUND(18, g, h, a, b, c, d, e, f); @@ -491,6 +487,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count) ROUND(21, d, e, f, g, h, a, b, c); ROUND(22, c, d, e, f, g, h, a, b); ROUND(23, b, c, d, e, f, g, h, a); + ROUND(24, a, b, c, d, e, f, g, h); ROUND(25, h, a, b, c, d, e, f, g); ROUND(26, g, h, a, b, c, d, e, f); @@ -499,6 +496,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count) ROUND(29, d, e, f, g, h, a, b, c); ROUND(30, c, d, e, f, g, h, a, b); ROUND(31, b, c, d, e, f, g, h, a); + ROUND(32, a, b, c, d, e, f, g, h); ROUND(33, h, a, b, c, d, e, f, g); ROUND(34, g, h, a, b, c, d, e, f); @@ -507,6 +505,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count) ROUND(37, d, e, f, g, h, a, b, c); ROUND(38, c, d, e, f, g, h, a, b); ROUND(39, b, c, d, e, f, g, h, a); + ROUND(40, a, b, c, d, e, f, g, h); ROUND(41, h, a, b, c, d, e, f, g); ROUND(42, g, h, a, b, c, d, e, f); @@ -515,6 +514,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count) ROUND(45, d, e, f, g, h, a, b, c); ROUND(46, c, d, e, f, g, h, a, b); ROUND(47, b, c, d, e, f, g, h, a); + ROUND(48, a, b, c, d, e, f, g, h); ROUND(49, h, a, b, c, d, e, f, g); ROUND(50, g, h, a, b, c, d, e, f); @@ -523,6 +523,7 @@ static void sha256_process(uint32_t* state, const uint8_t* block, size_t count) ROUND(53, d, e, f, g, h, a, b, c); ROUND(54, c, d, e, f, g, h, a, b); ROUND(55, b, c, d, e, f, g, h, a); + ROUND(56, a, b, c, d, e, f, g, h); ROUND(57, h, a, b, c, d, e, f, g); ROUND(58, g, h, a, b, c, d, e, f); diff --git a/src/third_party/martins_hash/sha512.h b/src/third_party/martins_hash/sha512.h index 4a48c6f1..10a4cb9e 100644 --- a/src/third_party/martins_hash/sha512.h +++ b/src/third_party/martins_hash/sha512.h @@ -15,9 +15,9 @@ #define SHA512_BLOCK_SIZE 128 typedef struct { - uint8_t buffer[SHA512_BLOCK_SIZE]; - uint64_t count[2]; - uint64_t state[8]; + uint8_t buffer[SHA512_BLOCK_SIZE]; + uint64_t count[2]; + uint64_t state[8]; } sha512_ctx; typedef sha512_ctx sha384_ctx; @@ -62,52 +62,52 @@ static inline void sha384_finish(sha384_ctx* ctx, uint8_t digest[SHA384_DIGEST_S # define SHA512_SET64BE(ptr,x) *((__unaligned uint64_t*)(ptr)) = _byteswap_uint64(x) #else # define SHA512_GET64BE(ptr) \ -( \ -((uint64_t)((ptr)[0]) << 56) | \ -((uint64_t)((ptr)[1]) << 48) | \ -((uint64_t)((ptr)[2]) << 40) | \ -((uint64_t)((ptr)[3]) << 32) | \ -((uint64_t)((ptr)[4]) << 24) | \ -((uint64_t)((ptr)[5]) << 16) | \ -((uint64_t)((ptr)[6]) << 8) | \ -((uint64_t)((ptr)[7]) << 0) \ -) + ( \ + ((uint64_t)((ptr)[0]) << 56) | \ + ((uint64_t)((ptr)[1]) << 48) | \ + ((uint64_t)((ptr)[2]) << 40) | \ + ((uint64_t)((ptr)[3]) << 32) | \ + ((uint64_t)((ptr)[4]) << 24) | \ + ((uint64_t)((ptr)[5]) << 16) | \ + ((uint64_t)((ptr)[6]) << 8) | \ + ((uint64_t)((ptr)[7]) << 0) \ + ) # define SHA512_SET64BE(ptr, x) do \ -{ \ -(ptr)[0] = (uint8_t)((x) >> 56); \ -(ptr)[1] = (uint8_t)((x) >> 48); \ -(ptr)[2] = (uint8_t)((x) >> 40); \ -(ptr)[3] = (uint8_t)((x) >> 32); \ -(ptr)[4] = (uint8_t)((x) >> 24); \ -(ptr)[5] = (uint8_t)((x) >> 16); \ -(ptr)[6] = (uint8_t)((x) >> 8); \ -(ptr)[7] = (uint8_t)((x) >> 0); \ -} \ -while (0) + { \ + (ptr)[0] = (uint8_t)((x) >> 56); \ + (ptr)[1] = (uint8_t)((x) >> 48); \ + (ptr)[2] = (uint8_t)((x) >> 40); \ + (ptr)[3] = (uint8_t)((x) >> 32); \ + (ptr)[4] = (uint8_t)((x) >> 24); \ + (ptr)[5] = (uint8_t)((x) >> 16); \ + (ptr)[6] = (uint8_t)((x) >> 8); \ + (ptr)[7] = (uint8_t)((x) >> 0); \ + } \ + while (0) #endif static const uint64_t SHA512_K[80] = { - 0x428a2f98d728ae22, 0x7137449123ef65cd, 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc, - 0x3956c25bf348b538, 0x59f111f1b605d019, 0x923f82a4af194f9b, 0xab1c5ed5da6d8118, - 0xd807aa98a3030242, 0x12835b0145706fbe, 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2, - 0x72be5d74f27b896f, 0x80deb1fe3b1696b1, 0x9bdc06a725c71235, 0xc19bf174cf692694, - 0xe49b69c19ef14ad2, 0xefbe4786384f25e3, 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65, - 0x2de92c6f592b0275, 0x4a7484aa6ea6e483, 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5, - 0x983e5152ee66dfab, 0xa831c66d2db43210, 0xb00327c898fb213f, 0xbf597fc7beef0ee4, - 0xc6e00bf33da88fc2, 0xd5a79147930aa725, 0x06ca6351e003826f, 0x142929670a0e6e70, - 0x27b70a8546d22ffc, 0x2e1b21385c26c926, 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df, - 0x650a73548baf63de, 0x766a0abb3c77b2a8, 0x81c2c92e47edaee6, 0x92722c851482353b, - 0xa2bfe8a14cf10364, 0xa81a664bbc423001, 0xc24b8b70d0f89791, 0xc76c51a30654be30, - 0xd192e819d6ef5218, 0xd69906245565a910, 0xf40e35855771202a, 0x106aa07032bbd1b8, - 0x19a4c116b8d2d0c8, 0x1e376c085141ab53, 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8, - 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb, 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3, - 0x748f82ee5defb2fc, 0x78a5636f43172f60, 0x84c87814a1f0ab72, 0x8cc702081a6439ec, - 0x90befffa23631e28, 0xa4506cebde82bde9, 0xbef9a3f7b2c67915, 0xc67178f2e372532b, - 0xca273eceea26619c, 0xd186b8c721c0c207, 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178, - 0x06f067aa72176fba, 0x0a637dc5a2c898a6, 0x113f9804bef90dae, 0x1b710b35131c471b, - 0x28db77f523047d84, 0x32caab7b40c72493, 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c, - 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a, 0x5fcb6fab3ad6faec, 0x6c44198c4a475817, + 0x428a2f98d728ae22, 0x7137449123ef65cd, 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc, + 0x3956c25bf348b538, 0x59f111f1b605d019, 0x923f82a4af194f9b, 0xab1c5ed5da6d8118, + 0xd807aa98a3030242, 0x12835b0145706fbe, 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2, + 0x72be5d74f27b896f, 0x80deb1fe3b1696b1, 0x9bdc06a725c71235, 0xc19bf174cf692694, + 0xe49b69c19ef14ad2, 0xefbe4786384f25e3, 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65, + 0x2de92c6f592b0275, 0x4a7484aa6ea6e483, 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5, + 0x983e5152ee66dfab, 0xa831c66d2db43210, 0xb00327c898fb213f, 0xbf597fc7beef0ee4, + 0xc6e00bf33da88fc2, 0xd5a79147930aa725, 0x06ca6351e003826f, 0x142929670a0e6e70, + 0x27b70a8546d22ffc, 0x2e1b21385c26c926, 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df, + 0x650a73548baf63de, 0x766a0abb3c77b2a8, 0x81c2c92e47edaee6, 0x92722c851482353b, + 0xa2bfe8a14cf10364, 0xa81a664bbc423001, 0xc24b8b70d0f89791, 0xc76c51a30654be30, + 0xd192e819d6ef5218, 0xd69906245565a910, 0xf40e35855771202a, 0x106aa07032bbd1b8, + 0x19a4c116b8d2d0c8, 0x1e376c085141ab53, 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8, + 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb, 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3, + 0x748f82ee5defb2fc, 0x78a5636f43172f60, 0x84c87814a1f0ab72, 0x8cc702081a6439ec, + 0x90befffa23631e28, 0xa4506cebde82bde9, 0xbef9a3f7b2c67915, 0xc67178f2e372532b, + 0xca273eceea26619c, 0xd186b8c721c0c207, 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178, + 0x06f067aa72176fba, 0x0a637dc5a2c898a6, 0x113f9804bef90dae, 0x1b710b35131c471b, + 0x28db77f523047d84, 0x32caab7b40c72493, 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c, + 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a, 0x5fcb6fab3ad6faec, 0x6c44198c4a475817, }; #if defined(__x86_64__) || defined(_M_AMD64) @@ -134,129 +134,131 @@ static const uint64_t SHA512_K[80] = SHA512_TARGET("xsave") static inline int sha512_cpuid(void) { - static int cpuid; - - int result = cpuid; - if (result == 0) - { - int info[4]; - - SHA512_CPUID(1, info); - int has_xsave = info[2] & (1 << 26); - - int has_ymm = 0; - if (has_xsave) + static int cpuid; + + int result = cpuid; + if (result == 0) { - uint64_t xcr0 = SHA512_XGETBV(0); - has_ymm = xcr0 & (1 << 2); + int info[4]; + + SHA512_CPUID(1, info); + int has_xsave = info[2] & (1 << 26); + + int has_ymm = 0; + if (has_xsave) + { + uint64_t xcr0 = SHA512_XGETBV(0); + has_ymm = xcr0 & (1 << 2); + } + + SHA512_CPUID_EX(7, 0, info); + int has_avx2 = info[1] & (1 << 5); + + SHA512_CPUID_EX(7, 1, info); + int has_sha512 = info[0] & (1 << 0); + + result |= SHA512_CPUID_INIT; + if (has_ymm && has_avx2 && has_sha512) + { + result |= SHA512_CPUID_VSHA512; + } + + cpuid = result; } - - SHA512_CPUID_EX(7, 0, info); - int has_avx2 = info[1] & (1 << 5); - - SHA512_CPUID_EX(7, 1, info); - int has_sha512 = info[0] & (1 << 0); - - result |= SHA512_CPUID_INIT; - if (has_ymm && has_avx2 && has_sha512) - { - result |= SHA512_CPUID_VSHA512; - } - - cpuid = result; - } - + #if defined(SHA512_CPUID_MASK) - result &= SHA512_CPUID_MASK; + result &= SHA512_CPUID_MASK; #endif - - return result; + + return result; } SHA512_TARGET("avx2,sha512") static void sha512_process_vsha512(uint64_t* state, const uint8_t* block, size_t count) { - // pretty much same way how sha256 works, only with avx2 registers and 64-bit additions - // state is kept as two 256-bit ymm registers (8 qwords) - - // message qwords are loaded as 64-bit big-endian values - -#define W(i) w[(i)%4] - - // 4 wide round calculations -#define QROUND(i) do { \ -/* first 4 rounds load input block */ \ -if (i < 4) W(i) = _mm256_shuffle_epi8(_mm256_loadu_si256(&buffer[i]), bswap); \ -/* update message schedule */ \ -if (i > 2 && i < 19) W(i-3) = _mm256_sha512msg2_epi64(_mm256_add_epi64(W(i-3), _mm256_permute4x64_epi64(_mm256_blend_epi32(W(i-1), W(i), 3), _MM_SHUFFLE(0,3,2,1))), W(i)); \ -if (i > 0 && i < 17) W(i-1) = _mm256_sha512msg1_epi64(W(i-1), _mm256_castsi256_si128(W(i))); \ -/* add round constants */ \ -__m256i tmp = _mm256_add_epi64(W(i), _mm256_loadu_si256((const __m256i*)&SHA512_K[4*i])); \ -/* round functions */ \ -state1 = _mm256_sha512rnds2_epi64(state1, state0, _mm256_castsi256_si128(tmp)); \ -state0 = _mm256_sha512rnds2_epi64(state0, state1, _mm256_extracti128_si256(tmp, 1)); \ -} while(0) - - const __m256i* buffer = (const __m256i*)block; - - // to byteswap when doing big-ending load for message qwords - const __m256i bswap = _mm256_broadcastsi128_si256(_mm_setr_epi8(7,6,5,4,3,2,1,0, 15,14,13,12,11,10,9,8)); - - // load initial state - __m256i abcd = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)&state[0]), _MM_SHUFFLE(0,1,2,3)); // [a,b,c,d] - __m256i efgh = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)&state[4]), _MM_SHUFFLE(0,1,2,3)); // [e,f,g,h] - - // qword order for vsha512rnds2 instruction - __m256i state0 = _mm256_permute2x128_si256(efgh, abcd, (3 << 4) | 1); // [a,b,e,f] - __m256i state1 = _mm256_permute2x128_si256(efgh, abcd, (2 << 4) | 0); // [c,d,g,h] - - do - { - // remember current state - __m256i last0 = state0; - __m256i last1 = state1; - - __m256i w[4]; - - QROUND(0); - QROUND(1); - QROUND(2); - QROUND(3); - QROUND(4); - QROUND(5); - QROUND(6); - QROUND(7); - QROUND(8); - QROUND(9); - QROUND(10); - QROUND(11); - QROUND(12); - QROUND(13); - QROUND(14); - QROUND(15); - QROUND(16); - QROUND(17); - QROUND(18); - QROUND(19); - - // update next state - state0 = _mm256_add_epi64(state0, last0); - state1 = _mm256_add_epi64(state1, last1); - - buffer += 4; - } - while (--count); - - // restore qword order - abcd = _mm256_permute2x128_si256(state1, state0, (3 << 4) | 1); - efgh = _mm256_permute2x128_si256(state1, state0, (2 << 4) | 0); - - // save the new state - _mm256_storeu_si256((__m256i*)&state[0], _mm256_permute4x64_epi64(abcd, _MM_SHUFFLE(0,1,2,3))); - _mm256_storeu_si256((__m256i*)&state[4], _mm256_permute4x64_epi64(efgh, _MM_SHUFFLE(0,1,2,3))); - -#undef QROUND -#undef W + // pretty much same way how sha256 works, only with avx2 registers and 64-bit additions + // state is kept as two 256-bit ymm registers (8 qwords) + + // message qwords are loaded as 64-bit big-endian values + + // 4 wide round calculations + #define QROUND(i,m0,m1,m2,m3) do { \ + /* update message schedule */ \ + if (i > 2 && i < 19) m1 = _mm256_sha512msg2_epi64(_mm256_add_epi64(m1, _mm256_permute4x64_epi64(_mm256_blend_epi32(m3, m0, 3), _MM_SHUFFLE(0,3,2,1))), m0); \ + if (i > 0 && i < 17) m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0)); \ + /* add round constants */ \ + __m256i tmp = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i*)&SHA512_K[4*i])); \ + /* 4 round functions */ \ + s1 = _mm256_sha512rnds2_epi64(s1, s0, _mm256_castsi256_si128(tmp)); \ + s0 = _mm256_sha512rnds2_epi64(s0, s1, _mm256_extracti128_si256(tmp, 1)); \ + } while(0) + + const __m256i* buffer = (const __m256i*)block; + + // to byteswap when doing big-ending load for message qwords + const __m256i bswap = _mm256_broadcastsi128_si256(_mm_setr_epi8(7,6,5,4,3,2,1,0, 15,14,13,12,11,10,9,8)); + + // load initial state + __m256i abcd = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)&state[0]), _MM_SHUFFLE(0,1,2,3)); // [a,b,c,d] + __m256i efgh = _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)&state[4]), _MM_SHUFFLE(0,1,2,3)); // [e,f,g,h] + + // qword order for vsha512rnds2 instruction + __m256i s0 = _mm256_permute2x128_si256(efgh, abcd, 0x31); // [a,b,e,f] + __m256i s1 = _mm256_permute2x128_si256(efgh, abcd, 0x20); // [c,d,g,h] + + do + { + // remember current state + __m256i last0 = s0; + __m256i last1 = s1; + + // load initial message schedule, 128-byte block + __m256i w0 = _mm256_shuffle_epi8(_mm256_loadu_si256(&buffer[0]), bswap); + __m256i w1 = _mm256_shuffle_epi8(_mm256_loadu_si256(&buffer[1]), bswap); + __m256i w2 = _mm256_shuffle_epi8(_mm256_loadu_si256(&buffer[2]), bswap); + __m256i w3 = _mm256_shuffle_epi8(_mm256_loadu_si256(&buffer[3]), bswap); + buffer += 4; + + QROUND( 0, w0, w1, w2, w3); + QROUND( 1, w1, w2, w3, w0); + QROUND( 2, w2, w3, w0, w1); + QROUND( 3, w3, w0, w1, w2); + + QROUND( 4, w0, w1, w2, w3); + QROUND( 5, w1, w2, w3, w0); + QROUND( 6, w2, w3, w0, w1); + QROUND( 7, w3, w0, w1, w2); + + QROUND( 8, w0, w1, w2, w3); + QROUND( 9, w1, w2, w3, w0); + QROUND(10, w2, w3, w0, w1); + QROUND(11, w3, w0, w1, w2); + + QROUND(12, w0, w1, w2, w3); + QROUND(13, w1, w2, w3, w0); + QROUND(14, w2, w3, w0, w1); + QROUND(15, w3, w0, w1, w2); + + QROUND(16, w0, w1, w2, w3); + QROUND(17, w1, w2, w3, w0); + QROUND(18, w2, w3, w0, w1); + QROUND(19, w3, w0, w1, w2); + + // update next state + s0 = _mm256_add_epi64(s0, last0); + s1 = _mm256_add_epi64(s1, last1); + } + while (--count); + + // restore qword order + abcd = _mm256_permute2x128_si256(s1, s0, 0x31); + efgh = _mm256_permute2x128_si256(s1, s0, 0x20); + + // save the new state + _mm256_storeu_si256((__m256i*)&state[0], _mm256_permute4x64_epi64(abcd, _MM_SHUFFLE(0,1,2,3))); + _mm256_storeu_si256((__m256i*)&state[4], _mm256_permute4x64_epi64(efgh, _MM_SHUFFLE(0,1,2,3))); + + #undef QROUND } #endif // defined(__x86_64__) || defined(_M_AMD64) @@ -293,153 +295,172 @@ state0 = _mm256_sha512rnds2_epi64(state0, state1, _mm256_extracti128_si256(tmp, static inline int sha512_cpuid(void) { #if defined(__ARM_FEATURE_SHA512) - int result = SHA512_CPUID_ARM64; + int result = SHA512_CPUID_ARM64; #else - static int cpuid; - - int result = cpuid; - if (result == 0) - { + static int cpuid; + + int result = cpuid; + if (result == 0) + { #if defined(_WIN32) - // no sha512 bit in IsProcessorFeaturePresent function :( - uint64_t bits; - DWORD bitsize = sizeof(bits); - RegGetValueA(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "CP 4030", RRF_RT_QWORD | RRF_ZEROONFAILURE, NULL, &bits, &bitsize); - // bits from ID_AA64ISAR0_EL1 - int has_arm64 = ((bits >> 12) & 0xf) == 0x2; + // no sha512 bit in IsProcessorFeaturePresent function :( + uint64_t bits; + DWORD bitsize = sizeof(bits); + RegGetValueA(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "CP 4030", RRF_RT_QWORD | RRF_ZEROONFAILURE, NULL, &bits, &bitsize); + // bits from ID_AA64ISAR0_EL1 + int has_arm64 = ((bits >> 12) & 0xf) == 0x2; #elif defined(__linux__) - unsigned long hwcap = getauxval(AT_HWCAP); - int has_arm64 = hwcap & HWCAP_SHA512; + unsigned long hwcap = getauxval(AT_HWCAP); + int has_arm64 = hwcap & HWCAP_SHA512; #elif defined(__APPLE__) - int value = 0; - size_t valuelen = sizeof(value); - int has_arm64 = sysctlbyname("hw.optional.arm.FEAT_SHA512", &value, &valuelen, NULL, 0) == 0 && value != 0; + int value = 0; + size_t valuelen = sizeof(value); + int has_arm64 = sysctlbyname("hw.optional.arm.FEAT_SHA512", &value, &valuelen, NULL, 0) == 0 && value != 0; #else #error unknown platform #endif - result |= SHA512_CPUID_INIT; - if (has_arm64) - { - result |= SHA512_CPUID_ARM64; + result |= SHA512_CPUID_INIT; + if (has_arm64) + { + result |= SHA512_CPUID_ARM64; + } + + cpuid = result; } - - cpuid = result; - } #endif - + #if defined(SHA512_CPUID_MASK) - result &= SHA512_CPUID_MASK; + result &= SHA512_CPUID_MASK; #endif - - return result; + + return result; } SHA512_TARGET static void sha512_process_arm64(uint64_t* state, const uint8_t* block, size_t count) { -#define W(i) w[(i)%8] -#define S(i) vstate.val[3-(i)%4] - -#define DROUND(i) do { \ -/* load 8 round constants */ \ -if ((i % 4) == 0) rk = vld1q_u64_x4(&SHA512_K[2*i]); \ -/* first 8 rounds reverse byte order in each 64-bit lane of input block */ \ -if (i < 8) W(i) = vreinterpretq_u64_u8(vrev64q_u8(msg[(i/4)%2].val[i%4])); \ -/* update message schedule for next rounds */ \ -if (i >= 8) W(i) = vsha512su1q_u64(vsha512su0q_u64(W(i), W(i-7)), W(i-1), vextq_u64(W(i-4), W(i-3), 1)); \ -/* add round constants */ \ -uint64x2_t tmp = vaddq_u64(W(i), rk.val[i%4]); \ -/* 2 round functions */ \ -uint64x2_t x0 = vaddq_u64(vextq_u64(tmp, tmp, 1), S(i+0)); \ -uint64x2_t x1 = vsha512hq_u64(x0, vextq_u64(S(i+1), S(i+0), 1), vextq_u64(S(i+2), S(i+1), 1)); \ -S(i+0) = vsha512h2q_u64(x1, S(i+2), S(i+3)); \ -S(i+2) = vaddq_u64(S(i+2), x1); \ -} while (0) - - // load initial state - uint64x2x4_t vstate = vld1q_u64_x4(state); - - do - { - // remember current state - uint64x2x4_t vlast = vstate; - - // load 128-byte block - uint8x16x4_t msg[2] = + #define DROUND(i, ab,cd,ef,gh, m0,m1,m2,m3,m4,m5,m6,m7) do { \ + /* update message schedule */ \ + if (i >= 8) m0 = vsha512su1q_u64(vsha512su0q_u64(m0, m1), m7, vextq_u64(m4, m5, 1)); \ + /* add round constants */ \ + uint64x2_t tmp = vaddq_u64(m0, rk.val[i%4]); \ + /* 2 round functions */ \ + uint64x2_t gf = vextq_u64(ef, gh, 1); \ + uint64x2_t ed = vextq_u64(cd, ef, 1); \ + uint64x2_t x0 = vaddq_u64(vextq_u64(tmp, tmp, 1), gh); \ + uint64x2_t x1 = vsha512hq_u64(x0, gf, ed); \ + gh = vsha512h2q_u64(x1, cd, ab); \ + cd = vaddq_u64(cd, x1); \ + } while (0) + + // load initial state + uint64x2x4_t s = vld1q_u64_x4(state); + uint64x2_t s0 = s.val[0]; // ab + uint64x2_t s1 = s.val[1]; // cd + uint64x2_t s2 = s.val[2]; // ef + uint64x2_t s3 = s.val[3]; // gh + + do { - vld1q_u8_x4(block + 0 * 16), - vld1q_u8_x4(block + 4 * 16), - }; - - uint64x2x4_t rk; - uint64x2_t w[8]; - - DROUND( 0); - DROUND( 1); - DROUND( 2); - DROUND( 3); - - DROUND( 4); - DROUND( 5); - DROUND( 6); - DROUND( 7); - - DROUND( 8); - DROUND( 9); - DROUND(10); - DROUND(11); - - DROUND(12); - DROUND(13); - DROUND(14); - DROUND(15); - - DROUND(16); - DROUND(17); - DROUND(18); - DROUND(19); - - DROUND(20); - DROUND(21); - DROUND(22); - DROUND(23); - - DROUND(24); - DROUND(25); - DROUND(26); - DROUND(27); - - DROUND(28); - DROUND(29); - DROUND(30); - DROUND(31); - - DROUND(32); - DROUND(33); - DROUND(34); - DROUND(35); - - DROUND(36); - DROUND(37); - DROUND(38); - DROUND(39); - - // update next state - vstate.val[0] = vaddq_u64(vstate.val[0], vlast.val[0]); - vstate.val[1] = vaddq_u64(vstate.val[1], vlast.val[1]); - vstate.val[2] = vaddq_u64(vstate.val[2], vlast.val[2]); - vstate.val[3] = vaddq_u64(vstate.val[3], vlast.val[3]); - - block += SHA512_BLOCK_SIZE; - } - while (--count); - - // save the new state - vst1q_u64_x4(state, vstate); - -#undef DROUND -#undef S -#undef W + // remember current state + uint64x2_t last0 = s0; + uint64x2_t last1 = s1; + uint64x2_t last2 = s2; + uint64x2_t last3 = s3; + + // load initial message schedule, 128-byte block + uint8x16x4_t msg0 = vld1q_u8_x4(block + 0 * 16); + uint8x16x4_t msg1 = vld1q_u8_x4(block + 4 * 16); + block += SHA512_BLOCK_SIZE; + + // reverse the byte order in each 64-bit lane + uint64x2_t w0 = vreinterpretq_u64_u8(vrev64q_u8(msg0.val[0])); + uint64x2_t w1 = vreinterpretq_u64_u8(vrev64q_u8(msg0.val[1])); + uint64x2_t w2 = vreinterpretq_u64_u8(vrev64q_u8(msg0.val[2])); + uint64x2_t w3 = vreinterpretq_u64_u8(vrev64q_u8(msg0.val[3])); + uint64x2_t w4 = vreinterpretq_u64_u8(vrev64q_u8(msg1.val[0])); + uint64x2_t w5 = vreinterpretq_u64_u8(vrev64q_u8(msg1.val[1])); + uint64x2_t w6 = vreinterpretq_u64_u8(vrev64q_u8(msg1.val[2])); + uint64x2_t w7 = vreinterpretq_u64_u8(vrev64q_u8(msg1.val[3])); + + uint64x2x4_t rk; + + rk = vld1q_u64_x4(&SHA512_K[0]); + DROUND( 0, s0,s1,s2,s3, w0,w1,w2,w3,w4,w5,w6,w7); + DROUND( 1, s3,s0,s1,s2, w1,w2,w3,w4,w5,w6,w7,w0); + DROUND( 2, s2,s3,s0,s1, w2,w3,w4,w5,w6,w7,w0,w1); + DROUND( 3, s1,s2,s3,s0, w3,w4,w5,w6,w7,w0,w1,w2); + + rk = vld1q_u64_x4(&SHA512_K[8]); + DROUND( 4, s0,s1,s2,s3, w4,w5,w6,w7,w0,w1,w2,w3); + DROUND( 5, s3,s0,s1,s2, w5,w6,w7,w0,w1,w2,w3,w4); + DROUND( 6, s2,s3,s0,s1, w6,w7,w0,w1,w2,w3,w4,w5); + DROUND( 7, s1,s2,s3,s0, w7,w0,w1,w2,w3,w4,w5,w6); + + rk = vld1q_u64_x4(&SHA512_K[16]); + DROUND( 8, s0,s1,s2,s3, w0,w1,w2,w3,w4,w5,w6,w7); + DROUND( 9, s3,s0,s1,s2, w1,w2,w3,w4,w5,w6,w7,w0); + DROUND(10, s2,s3,s0,s1, w2,w3,w4,w5,w6,w7,w0,w1); + DROUND(11, s1,s2,s3,s0, w3,w4,w5,w6,w7,w0,w1,w2); + + rk = vld1q_u64_x4(&SHA512_K[24]); + DROUND(12, s0,s1,s2,s3, w4,w5,w6,w7,w0,w1,w2,w3); + DROUND(13, s3,s0,s1,s2, w5,w6,w7,w0,w1,w2,w3,w4); + DROUND(14, s2,s3,s0,s1, w6,w7,w0,w1,w2,w3,w4,w5); + DROUND(15, s1,s2,s3,s0, w7,w0,w1,w2,w3,w4,w5,w6); + + rk = vld1q_u64_x4(&SHA512_K[32]); + DROUND(16, s0,s1,s2,s3, w0,w1,w2,w3,w4,w5,w6,w7); + DROUND(17, s3,s0,s1,s2, w1,w2,w3,w4,w5,w6,w7,w0); + DROUND(18, s2,s3,s0,s1, w2,w3,w4,w5,w6,w7,w0,w1); + DROUND(19, s1,s2,s3,s0, w3,w4,w5,w6,w7,w0,w1,w2); + + rk = vld1q_u64_x4(&SHA512_K[40]); + DROUND(20, s0,s1,s2,s3, w4,w5,w6,w7,w0,w1,w2,w3); + DROUND(21, s3,s0,s1,s2, w5,w6,w7,w0,w1,w2,w3,w4); + DROUND(22, s2,s3,s0,s1, w6,w7,w0,w1,w2,w3,w4,w5); + DROUND(23, s1,s2,s3,s0, w7,w0,w1,w2,w3,w4,w5,w6); + + rk = vld1q_u64_x4(&SHA512_K[48]); + DROUND(24, s0,s1,s2,s3, w0,w1,w2,w3,w4,w5,w6,w7); + DROUND(25, s3,s0,s1,s2, w1,w2,w3,w4,w5,w6,w7,w0); + DROUND(26, s2,s3,s0,s1, w2,w3,w4,w5,w6,w7,w0,w1); + DROUND(27, s1,s2,s3,s0, w3,w4,w5,w6,w7,w0,w1,w2); + + rk = vld1q_u64_x4(&SHA512_K[56]); + DROUND(28, s0,s1,s2,s3, w4,w5,w6,w7,w0,w1,w2,w3); + DROUND(29, s3,s0,s1,s2, w5,w6,w7,w0,w1,w2,w3,w4); + DROUND(30, s2,s3,s0,s1, w6,w7,w0,w1,w2,w3,w4,w5); + DROUND(31, s1,s2,s3,s0, w7,w0,w1,w2,w3,w4,w5,w6); + + rk = vld1q_u64_x4(&SHA512_K[64]); + DROUND(32, s0,s1,s2,s3, w0,w1,w2,w3,w4,w5,w6,w7); + DROUND(33, s3,s0,s1,s2, w1,w2,w3,w4,w5,w6,w7,w0); + DROUND(34, s2,s3,s0,s1, w2,w3,w4,w5,w6,w7,w0,w1); + DROUND(35, s1,s2,s3,s0, w3,w4,w5,w6,w7,w0,w1,w2); + + rk = vld1q_u64_x4(&SHA512_K[72]); + DROUND(36, s0,s1,s2,s3, w4,w5,w6,w7,w0,w1,w2,w3); + DROUND(37, s3,s0,s1,s2, w5,w6,w7,w0,w1,w2,w3,w4); + DROUND(38, s2,s3,s0,s1, w6,w7,w0,w1,w2,w3,w4,w5); + DROUND(39, s1,s2,s3,s0, w7,w0,w1,w2,w3,w4,w5,w6); + + // update next state + s0 = vaddq_u64(s0, last0); + s1 = vaddq_u64(s1, last1); + s2 = vaddq_u64(s2, last2); + s3 = vaddq_u64(s3, last3); + } + while (--count); + + // save the new state + s.val[0] = s0; + s.val[1] = s1; + s.val[2] = s2; + s.val[3] = s3; + vst1q_u64_x4(state, s); + + #undef DROUND } #endif // defined(__aarch64__) || defined(_M_ARM64) @@ -447,255 +468,264 @@ S(i+2) = vaddq_u64(S(i+2), x1); static void sha512_process(uint64_t* state, const uint8_t* block, size_t count) { #if defined(__x86_64__) || defined(_M_AMD64) - int cpuid = sha512_cpuid(); - if (cpuid & SHA512_CPUID_VSHA512) - { - sha512_process_vsha512(state, block, count); - return; - } + int cpuid = sha512_cpuid(); + if (cpuid & SHA512_CPUID_VSHA512) + { + sha512_process_vsha512(state, block, count); + return; + } #endif - + #if defined(__aarch64__) || defined(_M_ARM64) - int cpuid = sha512_cpuid(); - if (cpuid & SHA512_CPUID_ARM64) - { - sha512_process_arm64(state, block, count); - return; - } + int cpuid = sha512_cpuid(); + if (cpuid & SHA512_CPUID_ARM64) + { + sha512_process_arm64(state, block, count); + return; + } #endif - -#define Ch(x,y,z) ((x & (y ^ z)) ^ z) -#define Maj(x,y,z) ((x & y) | (z & (x | y))) - -#define BSig0(x) (SHA512_ROR64(x, 28) ^ SHA512_ROR64(x, 34) ^ SHA512_ROR64(x, 39)) -#define BSig1(x) (SHA512_ROR64(x, 14) ^ SHA512_ROR64(x, 18) ^ SHA512_ROR64(x, 41)) -#define SSig0(x) (SHA512_ROR64(x, 1) ^ SHA512_ROR64(x, 8) ^ (x >> 7)) -#define SSig1(x) (SHA512_ROR64(x, 19) ^ SHA512_ROR64(x, 61) ^ (x >> 6)) - -#define W(i) w[(i+16)%16] - -#define ROUND(i,a,b,c,d,e,f,g,h) do \ -{ \ -uint64_t w0; \ -if (i < 16) W(i) = w0 = SHA512_GET64BE(block + i*sizeof(uint64_t)); \ -if (i >= 16) W(i) = w0 = SSig1(W(i-2)) + W(i-7) + SSig0(W(i-15)) + W(i-16); \ - \ -uint64_t t1 = h + BSig1(e) + Ch(e,f,g) + SHA512_K[i] + w0; \ -uint64_t t2 = BSig0(a) + Maj(a,b,c); \ -d += t1; \ -h = t1 + t2; \ -} while (0) - - do - { - uint64_t a = state[0]; - uint64_t b = state[1]; - uint64_t c = state[2]; - uint64_t d = state[3]; - uint64_t e = state[4]; - uint64_t f = state[5]; - uint64_t g = state[6]; - uint64_t h = state[7]; - - uint64_t w[16]; - - ROUND( 0, a, b, c, d, e, f, g, h); - ROUND( 1, h, a, b, c, d, e, f, g); - ROUND( 2, g, h, a, b, c, d, e, f); - ROUND( 3, f, g, h, a, b, c, d, e); - ROUND( 4, e, f, g, h, a, b, c, d); - ROUND( 5, d, e, f, g, h, a, b, c); - ROUND( 6, c, d, e, f, g, h, a, b); - ROUND( 7, b, c, d, e, f, g, h, a); - ROUND( 8, a, b, c, d, e, f, g, h); - ROUND( 9, h, a, b, c, d, e, f, g); - ROUND(10, g, h, a, b, c, d, e, f); - ROUND(11, f, g, h, a, b, c, d, e); - ROUND(12, e, f, g, h, a, b, c, d); - ROUND(13, d, e, f, g, h, a, b, c); - ROUND(14, c, d, e, f, g, h, a, b); - ROUND(15, b, c, d, e, f, g, h, a); - ROUND(16, a, b, c, d, e, f, g, h); - ROUND(17, h, a, b, c, d, e, f, g); - ROUND(18, g, h, a, b, c, d, e, f); - ROUND(19, f, g, h, a, b, c, d, e); - ROUND(20, e, f, g, h, a, b, c, d); - ROUND(21, d, e, f, g, h, a, b, c); - ROUND(22, c, d, e, f, g, h, a, b); - ROUND(23, b, c, d, e, f, g, h, a); - ROUND(24, a, b, c, d, e, f, g, h); - ROUND(25, h, a, b, c, d, e, f, g); - ROUND(26, g, h, a, b, c, d, e, f); - ROUND(27, f, g, h, a, b, c, d, e); - ROUND(28, e, f, g, h, a, b, c, d); - ROUND(29, d, e, f, g, h, a, b, c); - ROUND(30, c, d, e, f, g, h, a, b); - ROUND(31, b, c, d, e, f, g, h, a); - ROUND(32, a, b, c, d, e, f, g, h); - ROUND(33, h, a, b, c, d, e, f, g); - ROUND(34, g, h, a, b, c, d, e, f); - ROUND(35, f, g, h, a, b, c, d, e); - ROUND(36, e, f, g, h, a, b, c, d); - ROUND(37, d, e, f, g, h, a, b, c); - ROUND(38, c, d, e, f, g, h, a, b); - ROUND(39, b, c, d, e, f, g, h, a); - ROUND(40, a, b, c, d, e, f, g, h); - ROUND(41, h, a, b, c, d, e, f, g); - ROUND(42, g, h, a, b, c, d, e, f); - ROUND(43, f, g, h, a, b, c, d, e); - ROUND(44, e, f, g, h, a, b, c, d); - ROUND(45, d, e, f, g, h, a, b, c); - ROUND(46, c, d, e, f, g, h, a, b); - ROUND(47, b, c, d, e, f, g, h, a); - ROUND(48, a, b, c, d, e, f, g, h); - ROUND(49, h, a, b, c, d, e, f, g); - ROUND(50, g, h, a, b, c, d, e, f); - ROUND(51, f, g, h, a, b, c, d, e); - ROUND(52, e, f, g, h, a, b, c, d); - ROUND(53, d, e, f, g, h, a, b, c); - ROUND(54, c, d, e, f, g, h, a, b); - ROUND(55, b, c, d, e, f, g, h, a); - ROUND(56, a, b, c, d, e, f, g, h); - ROUND(57, h, a, b, c, d, e, f, g); - ROUND(58, g, h, a, b, c, d, e, f); - ROUND(59, f, g, h, a, b, c, d, e); - ROUND(60, e, f, g, h, a, b, c, d); - ROUND(61, d, e, f, g, h, a, b, c); - ROUND(62, c, d, e, f, g, h, a, b); - ROUND(63, b, c, d, e, f, g, h, a); - ROUND(64, a, b, c, d, e, f, g, h); - ROUND(65, h, a, b, c, d, e, f, g); - ROUND(66, g, h, a, b, c, d, e, f); - ROUND(67, f, g, h, a, b, c, d, e); - ROUND(68, e, f, g, h, a, b, c, d); - ROUND(69, d, e, f, g, h, a, b, c); - ROUND(70, c, d, e, f, g, h, a, b); - ROUND(71, b, c, d, e, f, g, h, a); - ROUND(72, a, b, c, d, e, f, g, h); - ROUND(73, h, a, b, c, d, e, f, g); - ROUND(74, g, h, a, b, c, d, e, f); - ROUND(75, f, g, h, a, b, c, d, e); - ROUND(76, e, f, g, h, a, b, c, d); - ROUND(77, d, e, f, g, h, a, b, c); - ROUND(78, c, d, e, f, g, h, a, b); - ROUND(79, b, c, d, e, f, g, h, a); - - state[0] += a; - state[1] += b; - state[2] += c; - state[3] += d; - state[4] += e; - state[5] += f; - state[6] += g; - state[7] += h; - - block += SHA512_BLOCK_SIZE; - } - while (--count); - -#undef ROUND -#undef W -#undef Ch -#undef Maj -#undef BSig0 -#undef BSig1 -#undef SSig0 -#undef SSig1 + + #define Ch(x,y,z) ((x & (y ^ z)) ^ z) + #define Maj(x,y,z) ((x & y) | (z & (x | y))) + + #define BSig0(x) (SHA512_ROR64(x, 28) ^ SHA512_ROR64(x, 34) ^ SHA512_ROR64(x, 39)) + #define BSig1(x) (SHA512_ROR64(x, 14) ^ SHA512_ROR64(x, 18) ^ SHA512_ROR64(x, 41)) + #define SSig0(x) (SHA512_ROR64(x, 1) ^ SHA512_ROR64(x, 8) ^ (x >> 7)) + #define SSig1(x) (SHA512_ROR64(x, 19) ^ SHA512_ROR64(x, 61) ^ (x >> 6)) + + #define W(i) w[(i+16)%16] + + #define ROUND(i,a,b,c,d,e,f,g,h) do \ + { \ + uint64_t w0; \ + if (i < 16) W(i) = w0 = SHA512_GET64BE(block + i*sizeof(uint64_t)); \ + if (i >= 16) W(i) = w0 = SSig1(W(i-2)) + W(i-7) + SSig0(W(i-15)) + W(i-16); \ + \ + uint64_t t1 = h + BSig1(e) + Ch(e,f,g) + SHA512_K[i] + w0; \ + uint64_t t2 = BSig0(a) + Maj(a,b,c); \ + d += t1; \ + h = t1 + t2; \ + } while (0) + + do + { + uint64_t a = state[0]; + uint64_t b = state[1]; + uint64_t c = state[2]; + uint64_t d = state[3]; + uint64_t e = state[4]; + uint64_t f = state[5]; + uint64_t g = state[6]; + uint64_t h = state[7]; + + uint64_t w[16]; + + ROUND( 0, a, b, c, d, e, f, g, h); + ROUND( 1, h, a, b, c, d, e, f, g); + ROUND( 2, g, h, a, b, c, d, e, f); + ROUND( 3, f, g, h, a, b, c, d, e); + ROUND( 4, e, f, g, h, a, b, c, d); + ROUND( 5, d, e, f, g, h, a, b, c); + ROUND( 6, c, d, e, f, g, h, a, b); + ROUND( 7, b, c, d, e, f, g, h, a); + + ROUND( 8, a, b, c, d, e, f, g, h); + ROUND( 9, h, a, b, c, d, e, f, g); + ROUND(10, g, h, a, b, c, d, e, f); + ROUND(11, f, g, h, a, b, c, d, e); + ROUND(12, e, f, g, h, a, b, c, d); + ROUND(13, d, e, f, g, h, a, b, c); + ROUND(14, c, d, e, f, g, h, a, b); + ROUND(15, b, c, d, e, f, g, h, a); + + ROUND(16, a, b, c, d, e, f, g, h); + ROUND(17, h, a, b, c, d, e, f, g); + ROUND(18, g, h, a, b, c, d, e, f); + ROUND(19, f, g, h, a, b, c, d, e); + ROUND(20, e, f, g, h, a, b, c, d); + ROUND(21, d, e, f, g, h, a, b, c); + ROUND(22, c, d, e, f, g, h, a, b); + ROUND(23, b, c, d, e, f, g, h, a); + + ROUND(24, a, b, c, d, e, f, g, h); + ROUND(25, h, a, b, c, d, e, f, g); + ROUND(26, g, h, a, b, c, d, e, f); + ROUND(27, f, g, h, a, b, c, d, e); + ROUND(28, e, f, g, h, a, b, c, d); + ROUND(29, d, e, f, g, h, a, b, c); + ROUND(30, c, d, e, f, g, h, a, b); + ROUND(31, b, c, d, e, f, g, h, a); + + ROUND(32, a, b, c, d, e, f, g, h); + ROUND(33, h, a, b, c, d, e, f, g); + ROUND(34, g, h, a, b, c, d, e, f); + ROUND(35, f, g, h, a, b, c, d, e); + ROUND(36, e, f, g, h, a, b, c, d); + ROUND(37, d, e, f, g, h, a, b, c); + ROUND(38, c, d, e, f, g, h, a, b); + ROUND(39, b, c, d, e, f, g, h, a); + + ROUND(40, a, b, c, d, e, f, g, h); + ROUND(41, h, a, b, c, d, e, f, g); + ROUND(42, g, h, a, b, c, d, e, f); + ROUND(43, f, g, h, a, b, c, d, e); + ROUND(44, e, f, g, h, a, b, c, d); + ROUND(45, d, e, f, g, h, a, b, c); + ROUND(46, c, d, e, f, g, h, a, b); + ROUND(47, b, c, d, e, f, g, h, a); + + ROUND(48, a, b, c, d, e, f, g, h); + ROUND(49, h, a, b, c, d, e, f, g); + ROUND(50, g, h, a, b, c, d, e, f); + ROUND(51, f, g, h, a, b, c, d, e); + ROUND(52, e, f, g, h, a, b, c, d); + ROUND(53, d, e, f, g, h, a, b, c); + ROUND(54, c, d, e, f, g, h, a, b); + ROUND(55, b, c, d, e, f, g, h, a); + + ROUND(56, a, b, c, d, e, f, g, h); + ROUND(57, h, a, b, c, d, e, f, g); + ROUND(58, g, h, a, b, c, d, e, f); + ROUND(59, f, g, h, a, b, c, d, e); + ROUND(60, e, f, g, h, a, b, c, d); + ROUND(61, d, e, f, g, h, a, b, c); + ROUND(62, c, d, e, f, g, h, a, b); + ROUND(63, b, c, d, e, f, g, h, a); + + ROUND(64, a, b, c, d, e, f, g, h); + ROUND(65, h, a, b, c, d, e, f, g); + ROUND(66, g, h, a, b, c, d, e, f); + ROUND(67, f, g, h, a, b, c, d, e); + ROUND(68, e, f, g, h, a, b, c, d); + ROUND(69, d, e, f, g, h, a, b, c); + ROUND(70, c, d, e, f, g, h, a, b); + ROUND(71, b, c, d, e, f, g, h, a); + + ROUND(72, a, b, c, d, e, f, g, h); + ROUND(73, h, a, b, c, d, e, f, g); + ROUND(74, g, h, a, b, c, d, e, f); + ROUND(75, f, g, h, a, b, c, d, e); + ROUND(76, e, f, g, h, a, b, c, d); + ROUND(77, d, e, f, g, h, a, b, c); + ROUND(78, c, d, e, f, g, h, a, b); + ROUND(79, b, c, d, e, f, g, h, a); + + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + state[5] += f; + state[6] += g; + state[7] += h; + + block += SHA512_BLOCK_SIZE; + } + while (--count); + + #undef ROUND + #undef W + #undef Ch + #undef Maj + #undef BSig0 + #undef BSig1 + #undef SSig0 + #undef SSig1 } void sha512_init(sha512_ctx* ctx) { - ctx->count[0] = 0; - ctx->count[1] = 0; - ctx->state[0] = 0x6a09e667f3bcc908; - ctx->state[1] = 0xbb67ae8584caa73b; - ctx->state[2] = 0x3c6ef372fe94f82b; - ctx->state[3] = 0xa54ff53a5f1d36f1; - ctx->state[4] = 0x510e527fade682d1; - ctx->state[5] = 0x9b05688c2b3e6c1f; - ctx->state[6] = 0x1f83d9abfb41bd6b; - ctx->state[7] = 0x5be0cd19137e2179; + ctx->count[0] = 0; + ctx->count[1] = 0; + ctx->state[0] = 0x6a09e667f3bcc908; + ctx->state[1] = 0xbb67ae8584caa73b; + ctx->state[2] = 0x3c6ef372fe94f82b; + ctx->state[3] = 0xa54ff53a5f1d36f1; + ctx->state[4] = 0x510e527fade682d1; + ctx->state[5] = 0x9b05688c2b3e6c1f; + ctx->state[6] = 0x1f83d9abfb41bd6b; + ctx->state[7] = 0x5be0cd19137e2179; } void sha512_update(sha512_ctx* ctx, const void* data, size_t size) { - const uint8_t* buffer = (const uint8_t*)data; - - size_t pending = ctx->count[0] % SHA512_BLOCK_SIZE; - ctx->count[0] += size; - ctx->count[1] += size > ctx->count[0]; - - size_t available = SHA512_BLOCK_SIZE - pending; - if (pending && size >= available) - { - memcpy(ctx->buffer + pending, buffer, available); - sha512_process(ctx->state, ctx->buffer, 1); - buffer += available; - size -= available; - pending = 0; - } - - size_t count = size / SHA512_BLOCK_SIZE; - if (count) - { - sha512_process(ctx->state, buffer, count); - buffer += count * SHA512_BLOCK_SIZE; - size -= count * SHA512_BLOCK_SIZE; - } - - memcpy(ctx->buffer + pending, buffer, size); + const uint8_t* buffer = (const uint8_t*)data; + + size_t pending = ctx->count[0] % SHA512_BLOCK_SIZE; + ctx->count[0] += size; + ctx->count[1] += size > ctx->count[0]; + + size_t available = SHA512_BLOCK_SIZE - pending; + if (pending && size >= available) + { + memcpy(ctx->buffer + pending, buffer, available); + sha512_process(ctx->state, ctx->buffer, 1); + buffer += available; + size -= available; + pending = 0; + } + + size_t count = size / SHA512_BLOCK_SIZE; + if (count) + { + sha512_process(ctx->state, buffer, count); + buffer += count * SHA512_BLOCK_SIZE; + size -= count * SHA512_BLOCK_SIZE; + } + + memcpy(ctx->buffer + pending, buffer, size); } void sha512_finish(sha512_ctx* ctx, uint8_t digest[SHA512_DIGEST_SIZE]) { - uint64_t count0 = ctx->count[0]; - uint64_t count1 = ctx->count[1]; - uint64_t bitcount[2] = { (count0 << 3), (count1 << 3) | (count0 >> 61) }; - - size_t pending = count0 % SHA512_BLOCK_SIZE; - size_t blocks = pending < SHA512_BLOCK_SIZE - sizeof(bitcount) ? 1 : 2; - - ctx->buffer[pending++] = 0x80; - - uint8_t padding[2 * SHA512_BLOCK_SIZE]; - memcpy(padding, ctx->buffer, SHA512_BLOCK_SIZE); - memset(padding + pending, 0, SHA512_BLOCK_SIZE); - SHA512_SET64BE(padding + blocks * SHA512_BLOCK_SIZE - 2*sizeof(uint64_t), bitcount[1]); - SHA512_SET64BE(padding + blocks * SHA512_BLOCK_SIZE - 1*sizeof(uint64_t), bitcount[0]); - - sha512_process(ctx->state, padding, blocks); - - for (size_t i=0; i<8; i++) - { - SHA512_SET64BE(digest + i*sizeof(uint64_t), ctx->state[i]); - } + uint64_t count0 = ctx->count[0]; + uint64_t count1 = ctx->count[1]; + uint64_t bitcount[2] = { (count0 << 3), (count1 << 3) | (count0 >> 61) }; + + size_t pending = count0 % SHA512_BLOCK_SIZE; + size_t blocks = pending < SHA512_BLOCK_SIZE - sizeof(bitcount) ? 1 : 2; + + ctx->buffer[pending++] = 0x80; + + uint8_t padding[2 * SHA512_BLOCK_SIZE]; + memcpy(padding, ctx->buffer, SHA512_BLOCK_SIZE); + memset(padding + pending, 0, SHA512_BLOCK_SIZE); + SHA512_SET64BE(padding + blocks * SHA512_BLOCK_SIZE - 2*sizeof(uint64_t), bitcount[1]); + SHA512_SET64BE(padding + blocks * SHA512_BLOCK_SIZE - 1*sizeof(uint64_t), bitcount[0]); + + sha512_process(ctx->state, padding, blocks); + + for (size_t i=0; i<8; i++) + { + SHA512_SET64BE(digest + i*sizeof(uint64_t), ctx->state[i]); + } } void sha384_init(sha384_ctx* ctx) { - ctx->count[0] = 0; - ctx->count[1] = 0; - ctx->state[0] = 0xcbbb9d5dc1059ed8; - ctx->state[1] = 0x629a292a367cd507; - ctx->state[2] = 0x9159015a3070dd17; - ctx->state[3] = 0x152fecd8f70e5939; - ctx->state[4] = 0x67332667ffc00b31; - ctx->state[5] = 0x8eb44a8768581511; - ctx->state[6] = 0xdb0c2e0d64f98fa7; - ctx->state[7] = 0x47b5481dbefa4fa4; + ctx->count[0] = 0; + ctx->count[1] = 0; + ctx->state[0] = 0xcbbb9d5dc1059ed8; + ctx->state[1] = 0x629a292a367cd507; + ctx->state[2] = 0x9159015a3070dd17; + ctx->state[3] = 0x152fecd8f70e5939; + ctx->state[4] = 0x67332667ffc00b31; + ctx->state[5] = 0x8eb44a8768581511; + ctx->state[6] = 0xdb0c2e0d64f98fa7; + ctx->state[7] = 0x47b5481dbefa4fa4; } void sha384_update(sha512_ctx* ctx, const void* data, size_t size) { - sha512_update(ctx, data, size); + sha512_update(ctx, data, size); } void sha384_finish(sha384_ctx* ctx, uint8_t digest[SHA384_DIGEST_SIZE]) { - uint8_t temp[SHA512_DIGEST_SIZE]; - sha512_finish(ctx, temp); - - memcpy(digest, temp, SHA384_DIGEST_SIZE); + uint8_t temp[SHA512_DIGEST_SIZE]; + sha512_finish(ctx, temp); + + memcpy(digest, temp, SHA384_DIGEST_SIZE); } #if defined(__clang__)