From 8d869affca2529e1bc58bcbf54b38bff896958c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C4=81rti=C5=86=C5=A1=20Mo=C5=BEeiko?= Date: Wed, 2 Sep 2026 18:26:14 -0700 Subject: [PATCH] blake3: update to 1.8.7 version and use asm implementations Assembly code is >25% faster than intrinsic implementations on msvc. --- .github/workflows/builds.yml | 2 + build.bat | 16 + build.sh | 17 +- src/linker/base_ext/base_blake3.c | 99 +- src/linker/base_ext/base_blake3.h | 2 +- src/third_party/blake3/LICENSE_CC0 | 121 + src/third_party/blake3/asm/LICENSE | 330 --- src/third_party/blake3/asm/blake3.h | 82 - src/third_party/blake3/asm/blake3_dispatch.c | 278 -- src/third_party/blake3/{asm => }/blake3.c | 131 +- src/third_party/blake3/{c => }/blake3.h | 8 +- .../{asm => }/blake3_avx2_x86-64_unix.S | 0 .../blake3_avx2_x86-64_windows_msvc.asm | 0 .../{asm => }/blake3_avx512_x86-64_unix.S | 2239 +++++++++++++++++ .../blake3_avx512_x86-64_windows_msvc.asm | 0 .../blake3/{c => }/blake3_dispatch.c | 74 +- .../blake3/{asm => }/blake3_impl.h | 52 +- .../blake3/{asm => }/blake3_neon.c | 17 +- .../blake3/{asm => }/blake3_portable.c | 0 .../{asm => }/blake3_sse2_x86-64_unix.S | 0 .../blake3_sse2_x86-64_windows_msvc.asm | 0 .../{asm => }/blake3_sse41_x86-64_unix.S | 0 .../blake3_sse41_x86-64_windows_msvc.asm | 0 src/third_party/blake3/c/LICENSE | 330 --- src/third_party/blake3/c/blake3.c | 616 ----- src/third_party/blake3/c/blake3_avx2.c | 312 --- src/third_party/blake3/c/blake3_avx512.c | 1220 --------- src/third_party/blake3/c/blake3_impl.h | 285 --- src/third_party/blake3/c/blake3_neon.c | 368 --- src/third_party/blake3/c/blake3_portable.c | 160 -- src/third_party/blake3/c/blake3_sse2.c | 566 ----- src/third_party/blake3/c/blake3_sse41.c | 560 ----- 32 files changed, 2585 insertions(+), 5300 deletions(-) create mode 100644 src/third_party/blake3/LICENSE_CC0 delete mode 100644 src/third_party/blake3/asm/LICENSE delete mode 100644 src/third_party/blake3/asm/blake3.h delete mode 100644 src/third_party/blake3/asm/blake3_dispatch.c rename src/third_party/blake3/{asm => }/blake3.c (87%) rename src/third_party/blake3/{c => }/blake3.h (90%) rename src/third_party/blake3/{asm => }/blake3_avx2_x86-64_unix.S (100%) rename src/third_party/blake3/{asm => }/blake3_avx2_x86-64_windows_msvc.asm (100%) rename src/third_party/blake3/{asm => }/blake3_avx512_x86-64_unix.S (55%) rename src/third_party/blake3/{asm => }/blake3_avx512_x86-64_windows_msvc.asm (100%) rename src/third_party/blake3/{c => }/blake3_dispatch.c (85%) rename src/third_party/blake3/{asm => }/blake3_impl.h (84%) rename src/third_party/blake3/{asm => }/blake3_neon.c (96%) rename src/third_party/blake3/{asm => }/blake3_portable.c (100%) rename src/third_party/blake3/{asm => }/blake3_sse2_x86-64_unix.S (100%) rename src/third_party/blake3/{asm => }/blake3_sse2_x86-64_windows_msvc.asm (100%) rename src/third_party/blake3/{asm => }/blake3_sse41_x86-64_unix.S (100%) rename src/third_party/blake3/{asm => }/blake3_sse41_x86-64_windows_msvc.asm (100%) delete mode 100644 src/third_party/blake3/c/LICENSE delete mode 100644 src/third_party/blake3/c/blake3.c delete mode 100644 src/third_party/blake3/c/blake3_avx2.c delete mode 100644 src/third_party/blake3/c/blake3_avx512.c delete mode 100644 src/third_party/blake3/c/blake3_impl.h delete mode 100644 src/third_party/blake3/c/blake3_neon.c delete mode 100644 src/third_party/blake3/c/blake3_portable.c delete mode 100644 src/third_party/blake3/c/blake3_sse2.c delete mode 100644 src/third_party/blake3/c/blake3_sse41.c diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 58a20867..f666f2fc 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -45,6 +45,7 @@ jobs: - name: build shell: bash run: | + [[ "${{ matrix.compiler }}" == "clang-22" ]] && export AR=llvm-ar-22 CC="${{ matrix.compiler }}" ./build.sh "${{ matrix.target }}" "${{ matrix.mode }}" build-windows-2022: @@ -103,6 +104,7 @@ jobs: - name: Run shell: bash run: | + [[ "${{ matrix.compiler }}" == "clang-22" ]] && export AR=llvm-ar-22 CC="${{ matrix.compiler }}" ./run_tests.sh clang "${{ matrix.mode }}" -- \* $TORTURE_FLAGS - name: Upload Artifacts if: failure() diff --git a/build.bat b/build.bat index e51a529e..6a692d18 100644 --- a/build.bat +++ b/build.bat @@ -98,6 +98,10 @@ if "%msvc%"=="1" set rc=call rc if "%clang%"=="1" set rc=call llvm-rc if "%msvc%"=="1" set link_dll=/link /DLL if "%clang%"=="1" set link_dll=-Xlinker -DLL +if "%msvc%"=="1" set asm=call ml64 -nologo -c -Zi -Fo +if "%clang%"=="1" set asm=call ml64 -nologo -c -Zi -Fo +if "%msvc%"=="1" set mklib=call lib -nologo +if "%clang%"=="1" set mklib=call llvm-lib :: --- Choose Compile/Link Lines ---------------------------------------------- if "%msvc%"=="1" set compile_debug=%cl_debug% @@ -138,6 +142,18 @@ if "%no_meta%"=="" if exist metagen.exe ( ) popd +:: --- Assemble BLAKE3 -------------------------------------------------------- +if not exist build\blake3.lib if "%VSCMD_ARG_TGT_ARCH%" equ "x64" ( + echo [assembling blake3] + pushd build + %asm% blake3_sse2_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_sse2_x86-64_windows_msvc.asm || exit /b 1 + %asm% blake3_sse41_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_sse41_x86-64_windows_msvc.asm || exit /b 1 + %asm% blake3_avx2_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_avx2_x86-64_windows_msvc.asm || exit /b 1 + %asm% blake3_avx512_x86-64_windows_msvc.obj ..\src\third_party\blake3\blake3_avx512_x86-64_windows_msvc.asm || exit /b 1 + %mklib% -out:blake3.lib blake3_*_msvc.obj || exit /b 1 + popd +) + :: --- Build Everything (@build_targets) -------------------------------------- pushd build if "%raddbg%"=="1" set didbuild=1 && %compile% ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg.exe || exit /b 1 diff --git a/build.sh b/build.sh index 3724a40d..a3d11950 100755 --- a/build.sh +++ b/build.sh @@ -42,8 +42,8 @@ fi cc_icon="-DLNX_WM_ICON=1" # --- Choose Compile/Link Lines ----------------------------------------------- -if [[ "${gcc:-0}" == "1" ]]; then compiler="${CC:-gcc} $cc_cflags_gcc"; echo "[gcc compile]"; -elif [[ "${clang:-1}" == "1" ]]; then compiler="${CC:-clang} $cc_cflags_clang"; echo "[clang compile]"; +if [[ "${gcc:-0}" == "1" ]]; then compiler="${CC:-gcc} $cc_cflags_gcc"; ar="${AR:-ar}"; echo "[gcc compile]"; +elif [[ "${clang:-1}" == "1" ]]; then compiler="${CC:-clang} $cc_cflags_clang"; ar="${AR:-llvm-ar}"; echo "[clang compile]"; fi if [[ "${release:-0}" == "1" ]]; then echo "[release mode]"; compile="$compiler $cc_release"; elif [[ "${debug:-1}" == "1" ]]; then echo "[debug mode]"; compile="$compiler $cc_debug"; @@ -62,12 +62,23 @@ then cd .. fi +# --- Assemble BLAKE3 --------------------------------------------------------- +if [[ ! -f "build/blake3.a" ]] +then + echo "[assembling blake3]" + $compiler -c -g -o build/blake3_sse2_x86-64_unix.o src/third_party/blake3/blake3_sse2_x86-64_unix.S + $compiler -c -g -o build/blake3_sse41_x86-64_unix.o src/third_party/blake3/blake3_sse41_x86-64_unix.S + $compiler -c -g -o build/blake3_avx2_x86-64_unix.o src/third_party/blake3/blake3_avx2_x86-64_unix.S + $compiler -c -g -o build/blake3_avx512_x86-64_unix.o src/third_party/blake3/blake3_avx512_x86-64_unix.S + $ar rs build/blake3.a build/blake3_*_unix.o +fi + # --- Build Everything (@build_targets) --------------------------------------- cd build if [[ "${raddbg:-0}" == "1" ]]; then didbuild=1 && $compile ../src/raddbg/raddbg_main.c $cc_icon $cc_link $cc_os_gfx $cc_render $cc_font_provider -o raddbg; fi if [[ "${raddbg_non_graphical:-0}" == "1" ]]; then didbuild=1 && $compile ../src/raddbg/raddbg_main.c -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB $cc_link $cc_os_gfx $cc_render $cc_font_provider -o raddbg_non_graphical; fi if [[ "${radbin:-0}" == "1" ]]; then didbuild=1 && $compile ../src/radbin/radbin_main.c $cc_link -o radbin; fi -if [[ "${radlink:-0}" == "1" ]]; then didbuild=1 && $compile ../src/linker/lnk.c $cc_link -o radlink; fi +if [[ "${radlink:-0}" == "1" ]]; then didbuild=1 && $compile ../src/linker/lnk.c $cc_link -o radlink blake3.a; fi if [[ "${torture:-0}" == "1" ]]; then didbuild=1 && $compile ../src/torture/torture_main.c $cc_link $cc_os_gfx $cc_render $cc_font_provider -o torture; fi cd .. diff --git a/src/linker/base_ext/base_blake3.c b/src/linker/base_ext/base_blake3.c index 59f063c4..df932455 100644 --- a/src/linker/base_ext/base_blake3.c +++ b/src/linker/base_ext/base_blake3.c @@ -1,103 +1,16 @@ // Copyright (c) Epic Games Tools // Licensed under the MIT license (https://opensource.org/license/mit/) -#if defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wmacro-redefined" -#pragma clang diagnostic ignored "-Wsign-conversion" -#elif defined(_MSC_VER) -#pragma warning (push, 0) -#endif -#include "third_party/blake3/c/blake3_portable.c" - -#if defined(_M_AMD64) || defined(__x86_64__) - -#define round_fn sse2_round_fn -#define compress_pre sse2_compress_pre - -#include "third_party/blake3/c/blake3_sse2.c" - -#define loadu sse41_loadu -#define storeu sse41_storeu -#define addv sse41_addv -#define xorv sse41_xorv -#define set1 sse41_set1 -#define set4 sse41_set4 -#define rot16 sse41_rot16 -#define rot12 sse41_rot12 -#define rot8 sse41_rot8 -#define rot7 sse41_rot7 -#define g1 sse41_g1 -#define g2 sse41_g2 -#define diagonalize sse41_diagonalize -#define undiagonalize sse41_undiagonalize -#define compress_pre sse41_compress_pre -#define round_fn sse41_round_fn -#define transpose_vecs sse41_transpose_vecs -#define transpose_msg_vecs sse41_transpose_msg_vecs -#define load_counters sse41_load_counters - -#if defined(__clang__) -#pragma clang attribute push(__attribute__((target("sse4.1"))), apply_to=function) -#endif -#include "third_party/blake3/c/blake3_sse41.c" -#if defined(__clang__) -#pragma clang attribute pop -#endif - -#define loadu avx2_loadu -#define storeu avx2_storeu -#define addv avx2_addv -#define xorv avx2_xorv -#define set1 avx2_set1 -#define rot7 avx2_rot7 -#define rot8 avx2_rot8 -#define rot12 avx2_rot12 -#define rot16 avx2_rot16 -#define round_fn avx2_round_fn -#define transpose_vecs avx2_transpose_vecs -#define transpose_msg_vecs avx2_transpose_msg_vecs -#define load_counters avx2_load_counters - -#if defined(__clang__) -#pragma clang attribute push(__attribute__((target("avx2"))), apply_to=function) -#endif -#include "third_party/blake3/c/blake3_avx2.c" -#if defined(__clang__) -#pragma clang attribute pop -#endif - -#define set4 avx512_set4 -#define g1 avx512_g1 -#define g2 avx512_g2 -#define diagonalize avx512_diagonalize -#define undiagonalize avx512_undiagonalize -#define compress_pre avx512_compress_pre -#define transpose_vecs avx512_transpose_vecs -#define transpose_msg_vecs avx512_transpose_msg_vecs -#define load_counters avx512_load_counters - -#if defined(__clang__) -#pragma clang attribute push(__attribute__((target("avx512f,avx512vl"))), apply_to=function) -#endif -#include "third_party/blake3/c/blake3_avx512.c" -#if defined(__clang__) -#pragma clang attribute pop -#endif - -#endif +#include "third_party/blake3/blake3_portable.c" #if defined(__aarch64__) || defined(_M_ARM64) -#include "third_party/blake3/c/blake3_neon.c" +#include "third_party/blake3/blake3_neon.c" #endif -#include "third_party/blake3/c/blake3_dispatch.c" -#include "third_party/blake3/c/blake3.c" +#include "third_party/blake3/blake3_dispatch.c" +#include "third_party/blake3/blake3.c" -#if defined(__clang__) -#pragma clang diagnostic pop -#elif defined(_MSC_VER) -#pragma warning (pop, 0) +#if defined(_MSC_VER) && defined(_M_AMD64) +#pragma comment (lib, "blake3") #endif - diff --git a/src/linker/base_ext/base_blake3.h b/src/linker/base_ext/base_blake3.h index e80c1e35..90e7cdfd 100644 --- a/src/linker/base_ext/base_blake3.h +++ b/src/linker/base_ext/base_blake3.h @@ -6,7 +6,7 @@ #define BLAKE3_API static #define BLAKE3_PRIVATE static -#include "third_party/blake3/c/blake3.h" +#include "third_party/blake3/blake3.h" static void blake3(void* out, size_t outlen, void* in, size_t inlen) diff --git a/src/third_party/blake3/LICENSE_CC0 b/src/third_party/blake3/LICENSE_CC0 new file mode 100644 index 00000000..0e259d42 --- /dev/null +++ b/src/third_party/blake3/LICENSE_CC0 @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/src/third_party/blake3/asm/LICENSE b/src/third_party/blake3/asm/LICENSE deleted file mode 100644 index f5892efc..00000000 --- a/src/third_party/blake3/asm/LICENSE +++ /dev/null @@ -1,330 +0,0 @@ -This work is released into the public domain with CC0 1.0. Alternatively, it is -licensed under the Apache License 2.0. - -------------------------------------------------------------------------------- - -Creative Commons Legal Code - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not -limited to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and -vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention -of, applicable law, Affirmer hereby overtly, fully, permanently, -irrevocably and unconditionally waives, abandons, and surrenders all of -Affirmer's Copyright and Related Rights and associated claims and causes -of action, whether now known or unknown (including existing as well as -future claims and causes of action), in the Work (i) in all territories -worldwide, (ii) for the maximum duration provided by applicable law or -treaty (including future time extensions), (iii) in any current or future -medium and for any number of copies, and (iv) for any purpose whatsoever, -including without limitation commercial, advertising or promotional -purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each -member of the public at large and to the detriment of Affirmer's heirs and -successors, fully intending that such Waiver shall not be subject to -revocation, rescission, cancellation, termination, or any other legal or -equitable action to disrupt the quiet enjoyment of the Work by the public -as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason -be judged legally invalid or ineffective under applicable law, then the -Waiver shall be preserved to the maximum extent permitted taking into -account Affirmer's express Statement of Purpose. In addition, to the -extent the Waiver is so judged Affirmer hereby grants to each affected -person a royalty-free, non transferable, non sublicensable, non exclusive, -irrevocable and unconditional license to exercise Affirmer's Copyright and -Related Rights in the Work (i) in all territories worldwide, (ii) for the -maximum duration provided by applicable law or treaty (including future -time extensions), (iii) in any current or future medium and for any number -of copies, and (iv) for any purpose whatsoever, including without -limitation commercial, advertising or promotional purposes (the -"License"). The License shall be deemed effective as of the date CC0 was -applied by Affirmer to the Work. Should any part of the License for any -reason be judged legally invalid or ineffective under applicable law, such -partial invalidity or ineffectiveness shall not invalidate the remainder -of the License, and in such case Affirmer hereby affirms that he or she -will not (i) exercise any of his or her remaining Copyright and Related -Rights in the Work or (ii) assert any associated claims and causes of -action with respect to the Work, in either case contrary to Affirmer's -express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. - -------------------------------------------------------------------------------- - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2019 Jack O'Connor and Samuel Neves - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/third_party/blake3/asm/blake3.h b/src/third_party/blake3/asm/blake3.h deleted file mode 100644 index f694dcf2..00000000 --- a/src/third_party/blake3/asm/blake3.h +++ /dev/null @@ -1,82 +0,0 @@ -#ifndef BLAKE3_H -#define BLAKE3_H - -#include -#include - -#if !defined(BLAKE3_API) -# if defined(_WIN32) || defined(__CYGWIN__) -# if defined(BLAKE3_DLL) -# if defined(BLAKE3_DLL_EXPORTS) -# define BLAKE3_API __declspec(dllexport) -# else -# define BLAKE3_API __declspec(dllimport) -# endif -# define BLAKE3_PRIVATE -# else -# define BLAKE3_API -# define BLAKE3_PRIVATE -# endif -# elif __GNUC__ >= 4 -# define BLAKE3_API __attribute__((visibility("default"))) -# define BLAKE3_PRIVATE __attribute__((visibility("hidden"))) -# else -# define BLAKE3_API -# define BLAKE3_PRIVATE -# endif -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#define BLAKE3_VERSION_STRING "1.5.0" -#define BLAKE3_KEY_LEN 32 -#define BLAKE3_OUT_LEN 32 -#define BLAKE3_BLOCK_LEN 64 -#define BLAKE3_CHUNK_LEN 1024 -#define BLAKE3_MAX_DEPTH 54 - -// This struct is a private implementation detail. It has to be here because -// it's part of blake3_hasher below. -typedef struct { - uint32_t cv[8]; - uint64_t chunk_counter; - uint8_t buf[BLAKE3_BLOCK_LEN]; - uint8_t buf_len; - uint8_t blocks_compressed; - uint8_t flags; -} blake3_chunk_state; - -typedef struct { - uint32_t key[8]; - blake3_chunk_state chunk; - uint8_t cv_stack_len; - // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example, - // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk - // requires a 4th entry, rather than merging everything down to 1, because we - // don't know whether more input is coming. This is different from how the - // reference implementation does things. - uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN]; -} blake3_hasher; - -BLAKE3_API const char *blake3_version(void); -BLAKE3_API void blake3_hasher_init(blake3_hasher *self); -BLAKE3_API void blake3_hasher_init_keyed(blake3_hasher *self, - const uint8_t key[BLAKE3_KEY_LEN]); -BLAKE3_API void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context); -BLAKE3_API void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, - size_t context_len); -BLAKE3_API void blake3_hasher_update(blake3_hasher *self, const void *input, - size_t input_len); -BLAKE3_API void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, - size_t out_len); -BLAKE3_API void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, - uint8_t *out, size_t out_len); -BLAKE3_API void blake3_hasher_reset(blake3_hasher *self); - -#ifdef __cplusplus -} -#endif - -#endif /* BLAKE3_H */ diff --git a/src/third_party/blake3/asm/blake3_dispatch.c b/src/third_party/blake3/asm/blake3_dispatch.c deleted file mode 100644 index 6f814569..00000000 --- a/src/third_party/blake3/asm/blake3_dispatch.c +++ /dev/null @@ -1,278 +0,0 @@ -#include -#include -#include - -#include "blake3_impl.h" - -#if defined(IS_X86) -#if defined(_MSC_VER) -#include -#elif defined(__GNUC__) -#include -#else -#undef IS_X86 /* Unimplemented! */ -#endif -#endif - -#define MAYBE_UNUSED(x) (void)((x)) - -#if defined(IS_X86) -static uint64_t xgetbv(void) { -#if defined(_MSC_VER) - return _xgetbv(0); -#else - uint32_t eax = 0, edx = 0; - __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0)); - return ((uint64_t)edx << 32) | eax; -#endif -} - -static void cpuid(uint32_t out[4], uint32_t id) { -#if defined(_MSC_VER) - __cpuid((int *)out, id); -#elif defined(__i386__) || defined(_M_IX86) - __asm__ __volatile__("movl %%ebx, %1\n" - "cpuid\n" - "xchgl %1, %%ebx\n" - : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id)); -#else - __asm__ __volatile__("cpuid\n" - : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id)); -#endif -} - -static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) { -#if defined(_MSC_VER) - __cpuidex((int *)out, id, sid); -#elif defined(__i386__) || defined(_M_IX86) - __asm__ __volatile__("movl %%ebx, %1\n" - "cpuid\n" - "xchgl %1, %%ebx\n" - : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id), "c"(sid)); -#else - __asm__ __volatile__("cpuid\n" - : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) - : "a"(id), "c"(sid)); -#endif -} - -#endif - -enum cpu_feature { - SSE2 = 1 << 0, - SSSE3 = 1 << 1, - SSE41 = 1 << 2, - AVX = 1 << 3, - AVX2 = 1 << 4, - AVX512F = 1 << 5, - AVX512VL = 1 << 6, - /* ... */ - UNDEFINED = 1 << 30 -}; - -#if !defined(BLAKE3_TESTING) -static /* Allow the variable to be controlled manually for testing */ -#endif - volatile int g_cpu_features = UNDEFINED; - -#if !defined(BLAKE3_TESTING) -static -#endif - enum cpu_feature - get_cpu_features(void) { - - /* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */ - long features = g_cpu_features; - if (features != UNDEFINED) { - return (enum cpu_feature)features; - } else { -#if defined(IS_X86) - uint32_t regs[4] = {0}; - uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3]; - (void)edx; - features = 0; - cpuid(regs, 0); - const int max_id = *eax; - cpuid(regs, 1); -#if defined(__amd64__) || defined(_M_X64) - features |= SSE2; -#else - if (*edx & (1UL << 26)) - features |= SSE2; -#endif - if (*ecx & (1UL << 9)) - features |= SSSE3; - if (*ecx & (1UL << 19)) - features |= SSE41; - - if (*ecx & (1UL << 27)) { // OSXSAVE - const uint64_t mask = xgetbv(); - if ((mask & 6) == 6) { // SSE and AVX states - if (*ecx & (1UL << 28)) - features |= AVX; - if (max_id >= 7) { - cpuidex(regs, 7, 0); - if (*ebx & (1UL << 5)) - features |= AVX2; - if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm - if (*ebx & (1UL << 31)) - features |= AVX512VL; - if (*ebx & (1UL << 16)) - features |= AVX512F; - } - } - } - } - g_cpu_features = features; - return (enum cpu_feature)features; -#else - /* How to detect NEON? */ - return 0; -#endif - } -} - -void blake3_compress_in_place(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if (features & AVX512VL) { - blake3_compress_in_place_avx512(cv, block, block_len, counter, flags); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - blake3_compress_in_place_sse41(cv, block, block_len, counter, flags); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - blake3_compress_in_place_sse2(cv, block, block_len, counter, flags); - return; - } -#endif -#endif - blake3_compress_in_place_portable(cv, block, block_len, counter, flags); -} - -void blake3_compress_xof(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags, - uint8_t out[64]) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if (features & AVX512VL) { - blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - blake3_compress_xof_sse41(cv, block, block_len, counter, flags, out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - blake3_compress_xof_sse2(cv, block, block_len, counter, flags, out); - return; - } -#endif -#endif - blake3_compress_xof_portable(cv, block, block_len, counter, flags, out); -} - -void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { - blake3_hash_many_avx512(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#if !defined(BLAKE3_NO_AVX2) - if (features & AVX2) { - blake3_hash_many_avx2(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - blake3_hash_many_sse2(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); - return; - } -#endif -#endif - -#if BLAKE3_USE_NEON == 1 - blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, out); - return; -#endif - - blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); -} - -// The dynamically detected SIMD degree of the current platform. -size_t blake3_simd_degree(void) { -#if defined(IS_X86) - const enum cpu_feature features = get_cpu_features(); - MAYBE_UNUSED(features); -#if !defined(BLAKE3_NO_AVX512) - if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { - return 16; - } -#endif -#if !defined(BLAKE3_NO_AVX2) - if (features & AVX2) { - return 8; - } -#endif -#if !defined(BLAKE3_NO_SSE41) - if (features & SSE41) { - return 4; - } -#endif -#if !defined(BLAKE3_NO_SSE2) - if (features & SSE2) { - return 4; - } -#endif -#endif -#if BLAKE3_USE_NEON == 1 - return 4; -#endif - return 1; -} diff --git a/src/third_party/blake3/asm/blake3.c b/src/third_party/blake3/blake3.c similarity index 87% rename from src/third_party/blake3/asm/blake3.c rename to src/third_party/blake3/blake3.c index 692f4b02..00f91f44 100644 --- a/src/third_party/blake3/asm/blake3.c +++ b/src/third_party/blake3/blake3.c @@ -1,6 +1,7 @@ #include #include #include +#include #include "blake3.h" #include "blake3_impl.h" @@ -88,24 +89,30 @@ INLINE void output_chaining_value(const output_t *self, uint8_t cv[32]) { INLINE void output_root_bytes(const output_t *self, uint64_t seek, uint8_t *out, size_t out_len) { + if (out_len == 0) { + return; + } uint64_t output_block_counter = seek / 64; size_t offset_within_block = seek % 64; uint8_t wide_buf[64]; - while (out_len > 0) { - blake3_compress_xof(self->input_cv, self->block, self->block_len, - output_block_counter, self->flags | ROOT, wide_buf); - size_t available_bytes = 64 - offset_within_block; - size_t memcpy_len; - if (out_len > available_bytes) { - memcpy_len = available_bytes; - } else { - memcpy_len = out_len; - } - memcpy(out, wide_buf + offset_within_block, memcpy_len); - out += memcpy_len; - out_len -= memcpy_len; + if(offset_within_block) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + const size_t available_bytes = 64 - offset_within_block; + const size_t bytes = out_len > available_bytes ? available_bytes : out_len; + memcpy(out, wide_buf + offset_within_block, bytes); + out += bytes; + out_len -= bytes; output_block_counter += 1; - offset_within_block = 0; + } + if(out_len / 64) { + blake3_xof_many(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, out, out_len / 64); + } + output_block_counter += out_len / 64; + out += out_len & -64; + out_len -= out_len & -64; + if(out_len) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + memcpy(out, wide_buf, out_len); } } @@ -134,9 +141,7 @@ INLINE void chunk_state_update(blake3_chunk_state *self, const uint8_t *input, input_len -= BLAKE3_BLOCK_LEN; } - size_t take = chunk_state_fill_buf(self, input, input_len); - input += take; - input_len -= take; + chunk_state_fill_buf(self, input, input_len); } INLINE output_t chunk_state_output(const blake3_chunk_state *self) { @@ -154,10 +159,10 @@ INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], // Given some input larger than one chunk, return the number of bytes that // should go in the left subtree. This is the largest power-of-2 number of // chunks that leaves at least 1 byte for the right subtree. -INLINE size_t left_len(size_t content_len) { - // Subtract 1 to reserve at least one byte for the right side. content_len +INLINE size_t left_subtree_len(size_t input_len) { + // Subtract 1 to reserve at least one byte for the right side. input_len // should always be greater than BLAKE3_CHUNK_LEN. - size_t full_chunks = (content_len - 1) / BLAKE3_CHUNK_LEN; + size_t full_chunks = (input_len - 1) / BLAKE3_CHUNK_LEN; return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; } @@ -261,11 +266,10 @@ INLINE size_t compress_parents_parallel(const uint8_t *child_chaining_values, // Why not just have the caller split the input on the first update(), instead // of implementing this special rule? Because we don't want to limit SIMD or // multi-threading parallelism for that update(). -static size_t blake3_compress_subtree_wide(const uint8_t *input, - size_t input_len, - const uint32_t key[8], - uint64_t chunk_counter, - uint8_t flags, uint8_t *out) { +size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb) { // Note that the single chunk case does *not* bump the SIMD degree up to 2 // when it is 1. If this implementation adds multi-threading in the future, // this gives us the option of multi-threading even the 2-chunk case, which @@ -279,7 +283,7 @@ static size_t blake3_compress_subtree_wide(const uint8_t *input, // the input into left and right subtrees. (Note that this is only optimal // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree // of 3 or something, we'll need a more complicated strategy.) - size_t left_input_len = left_len(input_len); + size_t left_input_len = left_subtree_len(input_len); size_t right_input_len = input_len - left_input_len; const uint8_t *right_input = &input[left_input_len]; uint64_t right_chunk_counter = @@ -299,12 +303,24 @@ static size_t blake3_compress_subtree_wide(const uint8_t *input, } uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; - // Recurse! If this implementation adds multi-threading support in the - // future, this is where it will go. - size_t left_n = blake3_compress_subtree_wide(input, left_input_len, key, - chunk_counter, flags, cv_array); - size_t right_n = blake3_compress_subtree_wide( - right_input, right_input_len, key, right_chunk_counter, flags, right_cvs); + // Recurse! + size_t left_n = SIZE_MAX; + size_t right_n = SIZE_MAX; + +#if defined(BLAKE3_USE_TBB) + blake3_compress_subtree_wide_join_tbb( + key, flags, use_tbb, + // left-hand side + input, left_input_len, chunk_counter, cv_array, &left_n, + // right-hand side + right_input, right_input_len, right_chunk_counter, right_cvs, &right_n); +#else + left_n = blake3_compress_subtree_wide( + input, left_input_len, key, chunk_counter, flags, cv_array, use_tbb); + right_n = blake3_compress_subtree_wide(right_input, right_input_len, key, + right_chunk_counter, flags, right_cvs, + use_tbb); +#endif // BLAKE3_USE_TBB // The special case again. If simd_degree=1, then we'll have left_n=1 and // right_n=1. Rather than compressing them into a single output, return @@ -330,32 +346,37 @@ static size_t blake3_compress_subtree_wide(const uint8_t *input, // // As with compress_subtree_wide(), this function is not used on inputs of 1 // chunk or less. That's a different codepath. -INLINE void compress_subtree_to_parent_node( - const uint8_t *input, size_t input_len, const uint32_t key[8], - uint64_t chunk_counter, uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN]) { +INLINE void +compress_subtree_to_parent_node(const uint8_t *input, size_t input_len, + const uint32_t key[8], uint64_t chunk_counter, + uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN], + bool use_tbb) { #if defined(BLAKE3_TESTING) assert(input_len > BLAKE3_CHUNK_LEN); #endif uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, - chunk_counter, flags, cv_array); + chunk_counter, flags, cv_array, use_tbb); assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); - - // If MAX_SIMD_DEGREE is greater than 2 and there's enough input, + // The following loop never executes when MAX_SIMD_DEGREE_OR_2 is 2, because + // as we just asserted, num_cvs will always be <=2 in that case. But GCC + // (particularly GCC 8.5) can't tell that it never executes, and if NDEBUG is + // set then it emits incorrect warnings here. We tried a few different + // hacks to silence these, but in the end our hacks just produced different + // warnings (see https://github.com/BLAKE3-team/BLAKE3/pull/380). Out of + // desperation, we ifdef out this entire loop when we know it's not needed. +#if MAX_SIMD_DEGREE_OR_2 > 2 + // If MAX_SIMD_DEGREE_OR_2 is greater than 2 and there's enough input, // compress_subtree_wide() returns more than 2 chaining values. Condense // them into 2 by forming parent nodes repeatedly. uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; - // The second half of this loop condition is always true, and we just - // asserted it above. But GCC can't tell that it's always true, and if NDEBUG - // is set on platforms where MAX_SIMD_DEGREE_OR_2 == 2, GCC emits spurious - // warnings here. GCC 8.5 is particularly sensitive, so if you're changing - // this code, test it against that version. - while (num_cvs > 2 && num_cvs <= MAX_SIMD_DEGREE_OR_2) { + while (num_cvs > 2) { num_cvs = compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); } +#endif memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); } @@ -427,7 +448,7 @@ INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { // of the whole tree, and it would need to be ROOT finalized. We can't // compress it until we know. // 2) This 64 KiB input might complete a larger tree, whose root node is -// similarly going to be the the root of the whole tree. For example, maybe +// similarly going to be the root of the whole tree. For example, maybe // we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the // node at the root of the 256 KiB subtree until we know how to finalize it. // @@ -452,8 +473,8 @@ INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], self->cv_stack_len += 1; } -void blake3_hasher_update(blake3_hasher *self, const void *input, - size_t input_len) { +INLINE void blake3_hasher_update_base(blake3_hasher *self, const void *input, + size_t input_len, bool use_tbb) { // Explicitly checking for zero avoids causing UB by passing a null pointer // to memcpy. This comes up in practice with things like: // std::vector v; @@ -539,7 +560,7 @@ void blake3_hasher_update(blake3_hasher *self, const void *input, uint8_t cv_pair[2 * BLAKE3_OUT_LEN]; compress_subtree_to_parent_node(input_bytes, subtree_len, self->key, self->chunk.chunk_counter, - self->chunk.flags, cv_pair); + self->chunk.flags, cv_pair, use_tbb); hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], self->chunk.chunk_counter + (subtree_chunks / 2)); @@ -561,6 +582,20 @@ void blake3_hasher_update(blake3_hasher *self, const void *input, } } +void blake3_hasher_update(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = false; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} + +#if defined(BLAKE3_USE_TBB) +void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = true; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} +#endif // BLAKE3_USE_TBB + void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, size_t out_len) { blake3_hasher_finalize_seek(self, 0, out, out_len); diff --git a/src/third_party/blake3/c/blake3.h b/src/third_party/blake3/blake3.h similarity index 90% rename from src/third_party/blake3/c/blake3.h rename to src/third_party/blake3/blake3.h index f694dcf2..c5b79987 100644 --- a/src/third_party/blake3/c/blake3.h +++ b/src/third_party/blake3/blake3.h @@ -30,7 +30,7 @@ extern "C" { #endif -#define BLAKE3_VERSION_STRING "1.5.0" +#define BLAKE3_VERSION_STRING "1.8.7" #define BLAKE3_KEY_LEN 32 #define BLAKE3_OUT_LEN 32 #define BLAKE3_BLOCK_LEN 64 @@ -38,7 +38,7 @@ extern "C" { #define BLAKE3_MAX_DEPTH 54 // This struct is a private implementation detail. It has to be here because -// it's part of blake3_hasher below. +// it's part of the blake3_hasher structure defined below. typedef struct { uint32_t cv[8]; uint64_t chunk_counter; @@ -69,6 +69,10 @@ BLAKE3_API void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const voi size_t context_len); BLAKE3_API void blake3_hasher_update(blake3_hasher *self, const void *input, size_t input_len); +#if defined(BLAKE3_USE_TBB) +BLAKE3_API void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len); +#endif // BLAKE3_USE_TBB BLAKE3_API void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, size_t out_len); BLAKE3_API void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, diff --git a/src/third_party/blake3/asm/blake3_avx2_x86-64_unix.S b/src/third_party/blake3/blake3_avx2_x86-64_unix.S similarity index 100% rename from src/third_party/blake3/asm/blake3_avx2_x86-64_unix.S rename to src/third_party/blake3/blake3_avx2_x86-64_unix.S diff --git a/src/third_party/blake3/asm/blake3_avx2_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_avx2_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_avx2_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_avx2_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/asm/blake3_avx512_x86-64_unix.S b/src/third_party/blake3/blake3_avx512_x86-64_unix.S similarity index 55% rename from src/third_party/blake3/asm/blake3_avx512_x86-64_unix.S rename to src/third_party/blake3/blake3_avx512_x86-64_unix.S index a06aede0..9642e413 100644 --- a/src/third_party/blake3/asm/blake3_avx512_x86-64_unix.S +++ b/src/third_party/blake3/blake3_avx512_x86-64_unix.S @@ -19,6 +19,8 @@ .global _blake3_compress_in_place_avx512 .global blake3_compress_xof_avx512 .global _blake3_compress_xof_avx512 +.global blake3_xof_many_avx512 +.global _blake3_xof_many_avx512 #ifdef __APPLE__ .text @@ -2553,6 +2555,2243 @@ blake3_compress_xof_avx512: vmovdqu xmmword ptr [r9+0x30], xmm3 ret +.p2align 6 +blake3_xof_many_avx512: +_blake3_xof_many_avx512: + _CET_ENDBR + mov r10,QWORD PTR [rsp+0x8] + cmp r10,0x1 + ja 2f + vmovdqu xmm0,XMMWORD PTR [rdi] + vmovdqu xmm1,XMMWORD PTR [rdi+0x10] + movzx eax,r8b + movzx edx,dl + shl rax,0x20 + add rdx,rax + vmovq xmm3,rcx + vmovq xmm4,rdx + vpunpcklqdq xmm3,xmm3,xmm4 + vmovaps xmm2,XMMWORD PTR [BLAKE3_IV+rip] + vmovups xmm8,XMMWORD PTR [rsi] + vmovups xmm9,XMMWORD PTR [rsi+0x10] + vshufps xmm4,xmm8,xmm9,0x88 + vshufps xmm5,xmm8,xmm9,0xdd + vmovups xmm8,XMMWORD PTR [rsi+0x20] + vmovups xmm9,XMMWORD PTR [rsi+0x30] + vshufps xmm6,xmm8,xmm9,0x88 + vshufps xmm7,xmm8,xmm9,0xdd + vpshufd xmm6,xmm6,0x93 + vpshufd xmm7,xmm7,0x93 + mov al,0x7 +3: + vpaddd xmm0,xmm0,xmm4 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm5 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x93 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x39 + vpaddd xmm0,xmm0,xmm6 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm7 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x39 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x93 + dec al + je 3f + vshufps xmm8,xmm4,xmm5,0xd6 + vpshufd xmm9,xmm4,0xf + vpshufd xmm4,xmm8,0x39 + vshufps xmm8,xmm6,xmm7,0xfa + vpblendd xmm9,xmm9,xmm8,0xaa + vpunpcklqdq xmm8,xmm7,xmm5 + vpblendd xmm8,xmm8,xmm6,0x88 + vpshufd xmm8,xmm8,0x78 + vpunpckhdq xmm5,xmm5,xmm7 + vpunpckldq xmm6,xmm6,xmm5 + vpshufd xmm7,xmm6,0x1e + vmovdqa xmm5,xmm9 + vmovdqa xmm6,xmm8 + jmp 3b +3: + vpxor xmm0,xmm0,xmm2 + vpxor xmm1,xmm1,xmm3 + vpxor xmm2,xmm2,XMMWORD PTR [rdi] + vpxor xmm3,xmm3,XMMWORD PTR [rdi+0x10] + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + ret +.p2align 6 +2: + push rbp + mov rbp,rsp + sub rsp,0x90 + and rsp,0xffffffffffffffc0 + vpbroadcastd zmm0,ecx + shr rcx,0x20 + vpbroadcastd zmm1,ecx + vpaddd zmm2,zmm0,ZMMWORD PTR [ADD0+rip] + vpcmpltud k1,zmm2,zmm0 + vpaddd zmm1{k1},zmm1,DWORD PTR [ADD1+rip]{1to16} + vmovdqa32 ZMMWORD PTR [rsp],zmm2 + vmovdqa32 ZMMWORD PTR [rsp+0x40],zmm1 + cmp r10,0x10 + jb 2f +3: + vpbroadcastd zmm16,DWORD PTR [rsi] + vpbroadcastd zmm17,DWORD PTR [rsi+0x4] + vpbroadcastd zmm18,DWORD PTR [rsi+0x8] + vpbroadcastd zmm19,DWORD PTR [rsi+0xc] + vpbroadcastd zmm20,DWORD PTR [rsi+0x10] + vpbroadcastd zmm21,DWORD PTR [rsi+0x14] + vpbroadcastd zmm22,DWORD PTR [rsi+0x18] + vpbroadcastd zmm23,DWORD PTR [rsi+0x1c] + vpbroadcastd zmm24,DWORD PTR [rsi+0x20] + vpbroadcastd zmm25,DWORD PTR [rsi+0x24] + vpbroadcastd zmm26,DWORD PTR [rsi+0x28] + vpbroadcastd zmm27,DWORD PTR [rsi+0x2c] + vpbroadcastd zmm28,DWORD PTR [rsi+0x30] + vpbroadcastd zmm29,DWORD PTR [rsi+0x34] + vpbroadcastd zmm30,DWORD PTR [rsi+0x38] + vpbroadcastd zmm31,DWORD PTR [rsi+0x3c] + vpbroadcastd zmm0,DWORD PTR [rdi] + vpbroadcastd zmm1,DWORD PTR [rdi+0x4] + vpbroadcastd zmm2,DWORD PTR [rdi+0x8] + vpbroadcastd zmm3,DWORD PTR [rdi+0xc] + vpbroadcastd zmm4,DWORD PTR [rdi+0x10] + vpbroadcastd zmm5,DWORD PTR [rdi+0x14] + vpbroadcastd zmm6,DWORD PTR [rdi+0x18] + vpbroadcastd zmm7,DWORD PTR [rdi+0x1c] + vpbroadcastd zmm8,DWORD PTR [BLAKE3_IV_0+rip] + vpbroadcastd zmm9,DWORD PTR [BLAKE3_IV_1+rip] + vpbroadcastd zmm10,DWORD PTR [BLAKE3_IV_2+rip] + vpbroadcastd zmm11,DWORD PTR [BLAKE3_IV_3+rip] + vmovdqa32 zmm12,ZMMWORD PTR [rsp] + vmovdqa32 zmm13,ZMMWORD PTR [rsp+0x40] + vpbroadcastd zmm14,edx + vpbroadcastd zmm15,r8d + vpaddd zmm0,zmm0,zmm16 + vpaddd zmm1,zmm1,zmm18 + vpaddd zmm2,zmm2,zmm20 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm17 + vpaddd zmm1,zmm1,zmm19 + vpaddd zmm2,zmm2,zmm21 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm24 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm28 + vpaddd zmm3,zmm3,zmm30 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm25 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm29 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm18 + vpaddd zmm1,zmm1,zmm19 + vpaddd zmm2,zmm2,zmm23 + vpaddd zmm3,zmm3,zmm20 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm22 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm16 + vpaddd zmm3,zmm3,zmm29 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm17 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm25 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm27 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm30 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm19 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm29 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm20 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm18 + vpaddd zmm3,zmm3,zmm30 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm22 + vpaddd zmm1,zmm1,zmm25 + vpaddd zmm2,zmm2,zmm27 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm21 + vpaddd zmm1,zmm1,zmm16 + vpaddd zmm2,zmm2,zmm31 + vpaddd zmm3,zmm3,zmm17 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm26 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm30 + vpaddd zmm3,zmm3,zmm29 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm23 + vpaddd zmm1,zmm1,zmm25 + vpaddd zmm2,zmm2,zmm19 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm20 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm21 + vpaddd zmm3,zmm3,zmm17 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm16 + vpaddd zmm1,zmm1,zmm18 + vpaddd zmm2,zmm2,zmm24 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm28 + vpaddd zmm1,zmm1,zmm25 + vpaddd zmm2,zmm2,zmm31 + vpaddd zmm3,zmm3,zmm30 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm29 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm26 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm23 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm16 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm18 + vpaddd zmm1,zmm1,zmm19 + vpaddd zmm2,zmm2,zmm17 + vpaddd zmm3,zmm3,zmm20 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm25 + vpaddd zmm1,zmm1,zmm27 + vpaddd zmm2,zmm2,zmm24 + vpaddd zmm3,zmm3,zmm31 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm30 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm28 + vpaddd zmm3,zmm3,zmm17 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm29 + vpaddd zmm1,zmm1,zmm16 + vpaddd zmm2,zmm2,zmm18 + vpaddd zmm3,zmm3,zmm20 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm19 + vpaddd zmm1,zmm1,zmm26 + vpaddd zmm2,zmm2,zmm22 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpaddd zmm0,zmm0,zmm27 + vpaddd zmm1,zmm1,zmm21 + vpaddd zmm2,zmm2,zmm17 + vpaddd zmm3,zmm3,zmm24 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vprord zmm15,zmm15,0x10 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0xc + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vpaddd zmm0,zmm0,zmm31 + vpaddd zmm1,zmm1,zmm16 + vpaddd zmm2,zmm2,zmm25 + vpaddd zmm3,zmm3,zmm22 + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm1,zmm1,zmm5 + vpaddd zmm2,zmm2,zmm6 + vpaddd zmm3,zmm3,zmm7 + vpxord zmm12,zmm12,zmm0 + vpxord zmm13,zmm13,zmm1 + vpxord zmm14,zmm14,zmm2 + vpxord zmm15,zmm15,zmm3 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vprord zmm15,zmm15,0x8 + vpaddd zmm8,zmm8,zmm12 + vpaddd zmm9,zmm9,zmm13 + vpaddd zmm10,zmm10,zmm14 + vpaddd zmm11,zmm11,zmm15 + vpxord zmm4,zmm4,zmm8 + vpxord zmm5,zmm5,zmm9 + vpxord zmm6,zmm6,zmm10 + vpxord zmm7,zmm7,zmm11 + vprord zmm4,zmm4,0x7 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vpaddd zmm0,zmm0,zmm30 + vpaddd zmm1,zmm1,zmm18 + vpaddd zmm2,zmm2,zmm19 + vpaddd zmm3,zmm3,zmm23 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x10 + vprord zmm12,zmm12,0x10 + vprord zmm13,zmm13,0x10 + vprord zmm14,zmm14,0x10 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0xc + vprord zmm6,zmm6,0xc + vprord zmm7,zmm7,0xc + vprord zmm4,zmm4,0xc + vpaddd zmm0,zmm0,zmm26 + vpaddd zmm1,zmm1,zmm28 + vpaddd zmm2,zmm2,zmm20 + vpaddd zmm3,zmm3,zmm29 + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm1,zmm1,zmm6 + vpaddd zmm2,zmm2,zmm7 + vpaddd zmm3,zmm3,zmm4 + vpxord zmm15,zmm15,zmm0 + vpxord zmm12,zmm12,zmm1 + vpxord zmm13,zmm13,zmm2 + vpxord zmm14,zmm14,zmm3 + vprord zmm15,zmm15,0x8 + vprord zmm12,zmm12,0x8 + vprord zmm13,zmm13,0x8 + vprord zmm14,zmm14,0x8 + vpaddd zmm10,zmm10,zmm15 + vpaddd zmm11,zmm11,zmm12 + vpaddd zmm8,zmm8,zmm13 + vpaddd zmm9,zmm9,zmm14 + vpxord zmm5,zmm5,zmm10 + vpxord zmm6,zmm6,zmm11 + vpxord zmm7,zmm7,zmm8 + vpxord zmm4,zmm4,zmm9 + vprord zmm5,zmm5,0x7 + vprord zmm6,zmm6,0x7 + vprord zmm7,zmm7,0x7 + vprord zmm4,zmm4,0x7 + vpxord zmm0,zmm0,zmm8 + vpxord zmm1,zmm1,zmm9 + vpxord zmm2,zmm2,zmm10 + vpxord zmm3,zmm3,zmm11 + vpxord zmm4,zmm4,zmm12 + vpxord zmm5,zmm5,zmm13 + vpxord zmm6,zmm6,zmm14 + vpxord zmm7,zmm7,zmm15 + vpxord zmm8,zmm8,DWORD PTR [rdi]{1to16} + vpxord zmm9,zmm9,DWORD PTR [rdi+0x4]{1to16} + vpxord zmm10,zmm10,DWORD PTR [rdi+0x8]{1to16} + vpxord zmm11,zmm11,DWORD PTR [rdi+0xc]{1to16} + vpxord zmm12,zmm12,DWORD PTR [rdi+0x10]{1to16} + vpxord zmm13,zmm13,DWORD PTR [rdi+0x14]{1to16} + vpxord zmm14,zmm14,DWORD PTR [rdi+0x18]{1to16} + vpxord zmm15,zmm15,DWORD PTR [rdi+0x1c]{1to16} + vpunpckldq zmm16,zmm0,zmm1 + vpunpckhdq zmm17,zmm0,zmm1 + vpunpckldq zmm18,zmm2,zmm3 + vpunpckhdq zmm19,zmm2,zmm3 + vpunpckldq zmm20,zmm4,zmm5 + vpunpckhdq zmm21,zmm4,zmm5 + vpunpckldq zmm22,zmm6,zmm7 + vpunpckhdq zmm23,zmm6,zmm7 + vpunpckldq zmm24,zmm8,zmm9 + vpunpckhdq zmm25,zmm8,zmm9 + vpunpckldq zmm26,zmm10,zmm11 + vpunpckhdq zmm27,zmm10,zmm11 + vpunpckldq zmm28,zmm12,zmm13 + vpunpckhdq zmm29,zmm12,zmm13 + vpunpckldq zmm30,zmm14,zmm15 + vpunpckhdq zmm31,zmm14,zmm15 + vpunpcklqdq zmm0,zmm16,zmm18 + vpunpckhqdq zmm1,zmm16,zmm18 + vpunpcklqdq zmm2,zmm17,zmm19 + vpunpckhqdq zmm3,zmm17,zmm19 + vpunpcklqdq zmm4,zmm20,zmm22 + vpunpckhqdq zmm5,zmm20,zmm22 + vpunpcklqdq zmm6,zmm21,zmm23 + vpunpckhqdq zmm7,zmm21,zmm23 + vpunpcklqdq zmm8,zmm24,zmm26 + vpunpckhqdq zmm9,zmm24,zmm26 + vpunpcklqdq zmm10,zmm25,zmm27 + vpunpckhqdq zmm11,zmm25,zmm27 + vpunpcklqdq zmm12,zmm28,zmm30 + vpunpckhqdq zmm13,zmm28,zmm30 + vpunpcklqdq zmm14,zmm29,zmm31 + vpunpckhqdq zmm15,zmm29,zmm31 + vshufi32x4 zmm16,zmm0,zmm4,0x88 + vshufi32x4 zmm17,zmm1,zmm5,0x88 + vshufi32x4 zmm18,zmm2,zmm6,0x88 + vshufi32x4 zmm19,zmm3,zmm7,0x88 + vshufi32x4 zmm20,zmm0,zmm4,0xdd + vshufi32x4 zmm21,zmm1,zmm5,0xdd + vshufi32x4 zmm22,zmm2,zmm6,0xdd + vshufi32x4 zmm23,zmm3,zmm7,0xdd + vshufi32x4 zmm24,zmm8,zmm12,0x88 + vshufi32x4 zmm25,zmm9,zmm13,0x88 + vshufi32x4 zmm26,zmm10,zmm14,0x88 + vshufi32x4 zmm27,zmm11,zmm15,0x88 + vshufi32x4 zmm28,zmm8,zmm12,0xdd + vshufi32x4 zmm29,zmm9,zmm13,0xdd + vshufi32x4 zmm30,zmm10,zmm14,0xdd + vshufi32x4 zmm31,zmm11,zmm15,0xdd + vshufi32x4 zmm0,zmm16,zmm24,0x88 + vshufi32x4 zmm1,zmm17,zmm25,0x88 + vshufi32x4 zmm2,zmm18,zmm26,0x88 + vshufi32x4 zmm3,zmm19,zmm27,0x88 + vshufi32x4 zmm4,zmm20,zmm28,0x88 + vshufi32x4 zmm5,zmm21,zmm29,0x88 + vshufi32x4 zmm6,zmm22,zmm30,0x88 + vshufi32x4 zmm7,zmm23,zmm31,0x88 + vshufi32x4 zmm8,zmm16,zmm24,0xdd + vshufi32x4 zmm9,zmm17,zmm25,0xdd + vshufi32x4 zmm10,zmm18,zmm26,0xdd + vshufi32x4 zmm11,zmm19,zmm27,0xdd + vshufi32x4 zmm12,zmm20,zmm28,0xdd + vshufi32x4 zmm13,zmm21,zmm29,0xdd + vshufi32x4 zmm14,zmm22,zmm30,0xdd + vshufi32x4 zmm15,zmm23,zmm31,0xdd + vmovdqu32 ZMMWORD PTR [r9],zmm0 + vmovdqu32 ZMMWORD PTR [r9+0x40],zmm1 + vmovdqu32 ZMMWORD PTR [r9+0x80],zmm2 + vmovdqu32 ZMMWORD PTR [r9+0xc0],zmm3 + vmovdqu32 ZMMWORD PTR [r9+0x100],zmm4 + vmovdqu32 ZMMWORD PTR [r9+0x140],zmm5 + vmovdqu32 ZMMWORD PTR [r9+0x180],zmm6 + vmovdqu32 ZMMWORD PTR [r9+0x1c0],zmm7 + vmovdqu32 ZMMWORD PTR [r9+0x200],zmm8 + vmovdqu32 ZMMWORD PTR [r9+0x240],zmm9 + vmovdqu32 ZMMWORD PTR [r9+0x280],zmm10 + vmovdqu32 ZMMWORD PTR [r9+0x2c0],zmm11 + vmovdqu32 ZMMWORD PTR [r9+0x300],zmm12 + vmovdqu32 ZMMWORD PTR [r9+0x340],zmm13 + vmovdqu32 ZMMWORD PTR [r9+0x380],zmm14 + vmovdqu32 ZMMWORD PTR [r9+0x3c0],zmm15 + vmovdqa32 zmm0,ZMMWORD PTR [rsp] + vmovdqa32 zmm1,ZMMWORD PTR [rsp+0x40] + vpaddd zmm2,zmm0,DWORD PTR [ADD16+rip]{1to16} + vpcmpltud k1,zmm2,zmm0 + vpaddd zmm1{k1},zmm1,DWORD PTR [ADD1+rip]{1to16} + vmovdqa32 ZMMWORD PTR [rsp],zmm2 + vmovdqa32 ZMMWORD PTR [rsp+0x40],zmm1 + add r9,0x400 + sub r10,0x10 + cmp r10,0x10 + jae 3b + test r10,r10 + jne 2f +9: + vzeroupper + mov rsp,rbp + pop rbp + ret +2: + test r10,0x8 + je 2f + vpbroadcastd ymm16,DWORD PTR [rsi] + vpbroadcastd ymm17,DWORD PTR [rsi+0x4] + vpbroadcastd ymm18,DWORD PTR [rsi+0x8] + vpbroadcastd ymm19,DWORD PTR [rsi+0xc] + vpbroadcastd ymm20,DWORD PTR [rsi+0x10] + vpbroadcastd ymm21,DWORD PTR [rsi+0x14] + vpbroadcastd ymm22,DWORD PTR [rsi+0x18] + vpbroadcastd ymm23,DWORD PTR [rsi+0x1c] + vpbroadcastd ymm24,DWORD PTR [rsi+0x20] + vpbroadcastd ymm25,DWORD PTR [rsi+0x24] + vpbroadcastd ymm26,DWORD PTR [rsi+0x28] + vpbroadcastd ymm27,DWORD PTR [rsi+0x2c] + vpbroadcastd ymm28,DWORD PTR [rsi+0x30] + vpbroadcastd ymm29,DWORD PTR [rsi+0x34] + vpbroadcastd ymm30,DWORD PTR [rsi+0x38] + vpbroadcastd ymm31,DWORD PTR [rsi+0x3c] + vpbroadcastd ymm0,DWORD PTR [rdi] + vpbroadcastd ymm1,DWORD PTR [rdi+0x4] + vpbroadcastd ymm2,DWORD PTR [rdi+0x8] + vpbroadcastd ymm3,DWORD PTR [rdi+0xc] + vpbroadcastd ymm4,DWORD PTR [rdi+0x10] + vpbroadcastd ymm5,DWORD PTR [rdi+0x14] + vpbroadcastd ymm6,DWORD PTR [rdi+0x18] + vpbroadcastd ymm7,DWORD PTR [rdi+0x1c] + vpbroadcastd ymm8,DWORD PTR [BLAKE3_IV_0+rip] + vpbroadcastd ymm9,DWORD PTR [BLAKE3_IV_1+rip] + vpbroadcastd ymm10,DWORD PTR [BLAKE3_IV_2+rip] + vpbroadcastd ymm11,DWORD PTR [BLAKE3_IV_3+rip] + vmovdqa ymm12,YMMWORD PTR [rsp] + vmovdqa ymm13,YMMWORD PTR [rsp+0x40] + vpbroadcastd ymm14,edx + vpbroadcastd ymm15,r8d + vpaddd ymm0,ymm0,ymm16 + vpaddd ymm1,ymm1,ymm18 + vpaddd ymm2,ymm2,ymm20 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm17 + vpaddd ymm1,ymm1,ymm19 + vpaddd ymm2,ymm2,ymm21 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm24 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm28 + vpaddd ymm3,ymm3,ymm30 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm25 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm29 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm18 + vpaddd ymm1,ymm1,ymm19 + vpaddd ymm2,ymm2,ymm23 + vpaddd ymm3,ymm3,ymm20 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm22 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm16 + vpaddd ymm3,ymm3,ymm29 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm17 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm25 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm27 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm30 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm19 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm29 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm20 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm18 + vpaddd ymm3,ymm3,ymm30 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm22 + vpaddd ymm1,ymm1,ymm25 + vpaddd ymm2,ymm2,ymm27 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm21 + vpaddd ymm1,ymm1,ymm16 + vpaddd ymm2,ymm2,ymm31 + vpaddd ymm3,ymm3,ymm17 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm26 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm30 + vpaddd ymm3,ymm3,ymm29 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm23 + vpaddd ymm1,ymm1,ymm25 + vpaddd ymm2,ymm2,ymm19 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm20 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm21 + vpaddd ymm3,ymm3,ymm17 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm16 + vpaddd ymm1,ymm1,ymm18 + vpaddd ymm2,ymm2,ymm24 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm28 + vpaddd ymm1,ymm1,ymm25 + vpaddd ymm2,ymm2,ymm31 + vpaddd ymm3,ymm3,ymm30 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm29 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm26 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm23 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm16 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm18 + vpaddd ymm1,ymm1,ymm19 + vpaddd ymm2,ymm2,ymm17 + vpaddd ymm3,ymm3,ymm20 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm25 + vpaddd ymm1,ymm1,ymm27 + vpaddd ymm2,ymm2,ymm24 + vpaddd ymm3,ymm3,ymm31 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm30 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm28 + vpaddd ymm3,ymm3,ymm17 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm29 + vpaddd ymm1,ymm1,ymm16 + vpaddd ymm2,ymm2,ymm18 + vpaddd ymm3,ymm3,ymm20 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm19 + vpaddd ymm1,ymm1,ymm26 + vpaddd ymm2,ymm2,ymm22 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpaddd ymm0,ymm0,ymm27 + vpaddd ymm1,ymm1,ymm21 + vpaddd ymm2,ymm2,ymm17 + vpaddd ymm3,ymm3,ymm24 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vprord ymm15,ymm15,0x10 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0xc + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vpaddd ymm0,ymm0,ymm31 + vpaddd ymm1,ymm1,ymm16 + vpaddd ymm2,ymm2,ymm25 + vpaddd ymm3,ymm3,ymm22 + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm1,ymm1,ymm5 + vpaddd ymm2,ymm2,ymm6 + vpaddd ymm3,ymm3,ymm7 + vpxord ymm12,ymm12,ymm0 + vpxord ymm13,ymm13,ymm1 + vpxord ymm14,ymm14,ymm2 + vpxord ymm15,ymm15,ymm3 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vprord ymm15,ymm15,0x8 + vpaddd ymm8,ymm8,ymm12 + vpaddd ymm9,ymm9,ymm13 + vpaddd ymm10,ymm10,ymm14 + vpaddd ymm11,ymm11,ymm15 + vpxord ymm4,ymm4,ymm8 + vpxord ymm5,ymm5,ymm9 + vpxord ymm6,ymm6,ymm10 + vpxord ymm7,ymm7,ymm11 + vprord ymm4,ymm4,0x7 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vpaddd ymm0,ymm0,ymm30 + vpaddd ymm1,ymm1,ymm18 + vpaddd ymm2,ymm2,ymm19 + vpaddd ymm3,ymm3,ymm23 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x10 + vprord ymm12,ymm12,0x10 + vprord ymm13,ymm13,0x10 + vprord ymm14,ymm14,0x10 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0xc + vprord ymm6,ymm6,0xc + vprord ymm7,ymm7,0xc + vprord ymm4,ymm4,0xc + vpaddd ymm0,ymm0,ymm26 + vpaddd ymm1,ymm1,ymm28 + vpaddd ymm2,ymm2,ymm20 + vpaddd ymm3,ymm3,ymm29 + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm1,ymm1,ymm6 + vpaddd ymm2,ymm2,ymm7 + vpaddd ymm3,ymm3,ymm4 + vpxord ymm15,ymm15,ymm0 + vpxord ymm12,ymm12,ymm1 + vpxord ymm13,ymm13,ymm2 + vpxord ymm14,ymm14,ymm3 + vprord ymm15,ymm15,0x8 + vprord ymm12,ymm12,0x8 + vprord ymm13,ymm13,0x8 + vprord ymm14,ymm14,0x8 + vpaddd ymm10,ymm10,ymm15 + vpaddd ymm11,ymm11,ymm12 + vpaddd ymm8,ymm8,ymm13 + vpaddd ymm9,ymm9,ymm14 + vpxord ymm5,ymm5,ymm10 + vpxord ymm6,ymm6,ymm11 + vpxord ymm7,ymm7,ymm8 + vpxord ymm4,ymm4,ymm9 + vprord ymm5,ymm5,0x7 + vprord ymm6,ymm6,0x7 + vprord ymm7,ymm7,0x7 + vprord ymm4,ymm4,0x7 + vpxor ymm0,ymm0,ymm8 + vpxor ymm1,ymm1,ymm9 + vpxor ymm2,ymm2,ymm10 + vpxor ymm3,ymm3,ymm11 + vpxor ymm4,ymm4,ymm12 + vpxor ymm5,ymm5,ymm13 + vpxor ymm6,ymm6,ymm14 + vpxor ymm7,ymm7,ymm15 + vpxord ymm8,ymm8,DWORD PTR [rdi]{1to8} + vpxord ymm9,ymm9,DWORD PTR [rdi+0x4]{1to8} + vpxord ymm10,ymm10,DWORD PTR [rdi+0x8]{1to8} + vpxord ymm11,ymm11,DWORD PTR [rdi+0xc]{1to8} + vpxord ymm12,ymm12,DWORD PTR [rdi+0x10]{1to8} + vpxord ymm13,ymm13,DWORD PTR [rdi+0x14]{1to8} + vpxord ymm14,ymm14,DWORD PTR [rdi+0x18]{1to8} + vpxord ymm15,ymm15,DWORD PTR [rdi+0x1c]{1to8} + vpunpckldq ymm16,ymm0,ymm1 + vpunpckhdq ymm17,ymm0,ymm1 + vpunpckldq ymm18,ymm2,ymm3 + vpunpckhdq ymm19,ymm2,ymm3 + vpunpckldq ymm20,ymm4,ymm5 + vpunpckhdq ymm21,ymm4,ymm5 + vpunpckldq ymm22,ymm6,ymm7 + vpunpckhdq ymm23,ymm6,ymm7 + vpunpckldq ymm24,ymm8,ymm9 + vpunpckhdq ymm25,ymm8,ymm9 + vpunpckldq ymm26,ymm10,ymm11 + vpunpckhdq ymm27,ymm10,ymm11 + vpunpckldq ymm28,ymm12,ymm13 + vpunpckhdq ymm29,ymm12,ymm13 + vpunpckldq ymm30,ymm14,ymm15 + vpunpckhdq ymm31,ymm14,ymm15 + vpunpcklqdq ymm0,ymm16,ymm18 + vpunpckhqdq ymm1,ymm16,ymm18 + vpunpcklqdq ymm2,ymm17,ymm19 + vpunpckhqdq ymm3,ymm17,ymm19 + vpunpcklqdq ymm4,ymm20,ymm22 + vpunpckhqdq ymm5,ymm20,ymm22 + vpunpcklqdq ymm6,ymm21,ymm23 + vpunpckhqdq ymm7,ymm21,ymm23 + vpunpcklqdq ymm8,ymm24,ymm26 + vpunpckhqdq ymm9,ymm24,ymm26 + vpunpcklqdq ymm10,ymm25,ymm27 + vpunpckhqdq ymm11,ymm25,ymm27 + vpunpcklqdq ymm12,ymm28,ymm30 + vpunpckhqdq ymm13,ymm28,ymm30 + vpunpcklqdq ymm14,ymm29,ymm31 + vpunpckhqdq ymm15,ymm29,ymm31 + vshufi32x4 ymm16,ymm0,ymm4,0x0 + vshufi32x4 ymm17,ymm8,ymm12,0x0 + vshufi32x4 ymm18,ymm1,ymm5,0x0 + vshufi32x4 ymm19,ymm9,ymm13,0x0 + vshufi32x4 ymm20,ymm2,ymm6,0x0 + vshufi32x4 ymm21,ymm10,ymm14,0x0 + vshufi32x4 ymm22,ymm3,ymm7,0x0 + vshufi32x4 ymm23,ymm11,ymm15,0x0 + vshufi32x4 ymm24,ymm0,ymm4,0x3 + vshufi32x4 ymm25,ymm8,ymm12,0x3 + vshufi32x4 ymm26,ymm1,ymm5,0x3 + vshufi32x4 ymm27,ymm9,ymm13,0x3 + vshufi32x4 ymm28,ymm2,ymm6,0x3 + vshufi32x4 ymm29,ymm10,ymm14,0x3 + vshufi32x4 ymm30,ymm3,ymm7,0x3 + vshufi32x4 ymm31,ymm11,ymm15,0x3 + vmovdqu32 YMMWORD PTR [r9],ymm16 + vmovdqu32 YMMWORD PTR [r9+0x20],ymm17 + vmovdqu32 YMMWORD PTR [r9+0x40],ymm18 + vmovdqu32 YMMWORD PTR [r9+0x60],ymm19 + vmovdqu32 YMMWORD PTR [r9+0x80],ymm20 + vmovdqu32 YMMWORD PTR [r9+0xa0],ymm21 + vmovdqu32 YMMWORD PTR [r9+0xc0],ymm22 + vmovdqu32 YMMWORD PTR [r9+0xe0],ymm23 + vmovdqu32 YMMWORD PTR [r9+0x100],ymm24 + vmovdqu32 YMMWORD PTR [r9+0x120],ymm25 + vmovdqu32 YMMWORD PTR [r9+0x140],ymm26 + vmovdqu32 YMMWORD PTR [r9+0x160],ymm27 + vmovdqu32 YMMWORD PTR [r9+0x180],ymm28 + vmovdqu32 YMMWORD PTR [r9+0x1a0],ymm29 + vmovdqu32 YMMWORD PTR [r9+0x1c0],ymm30 + vmovdqu32 YMMWORD PTR [r9+0x1e0],ymm31 + vmovdqa ymm0,YMMWORD PTR [rsp+0x20] + vmovdqa ymm1,YMMWORD PTR [rsp+0x60] + vmovdqa YMMWORD PTR [rsp],ymm0 + vmovdqa YMMWORD PTR [rsp+0x40],ymm1 + add r9,0x200 + sub r10,0x8 +2: + test r10,0x4 + je 2f + vbroadcasti32x4 zmm0,XMMWORD PTR [rdi] + vbroadcasti32x4 zmm1,XMMWORD PTR [rdi+0x10] + vbroadcasti32x4 zmm2,XMMWORD PTR [BLAKE3_IV+rip] + vmovdqa xmm12,XMMWORD PTR [rsp] + vmovdqa xmm13,XMMWORD PTR [rsp+0x40] + vpunpckldq xmm14,xmm12,xmm13 + vpunpckhdq xmm15,xmm12,xmm13 + vpermq ymm14,ymm14,0xdc + vpermq ymm15,ymm15,0xdc + vpbroadcastd zmm12,edx + vinserti64x4 zmm13,zmm14,ymm15,0x1 + mov eax,0x4444 + kmovw k2,eax + vpblendmd zmm13{k2},zmm13,zmm12 + vpbroadcastd zmm15,r8d + mov eax,0x8888 + kmovw k4,eax + vpblendmd zmm3{k4},zmm13,zmm15 + mov eax,0xaaaa + kmovw k3,eax + vbroadcasti32x4 zmm8,XMMWORD PTR [rsi] + vbroadcasti32x4 zmm9,XMMWORD PTR [rsi+0x10] + vshufps zmm4,zmm8,zmm9,0x88 + vshufps zmm5,zmm8,zmm9,0xdd + vbroadcasti32x4 zmm8,XMMWORD PTR [rsi+0x20] + vbroadcasti32x4 zmm9,XMMWORD PTR [rsi+0x30] + vshufps zmm6,zmm8,zmm9,0x88 + vshufps zmm7,zmm8,zmm9,0xdd + vpshufd zmm6,zmm6,0x93 + vpshufd zmm7,zmm7,0x93 + mov al,0x7 +3: + vpaddd zmm0,zmm0,zmm4 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x10 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0xc + vpaddd zmm0,zmm0,zmm5 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x8 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0x7 + vpshufd zmm0,zmm0,0x93 + vpshufd zmm3,zmm3,0x4e + vpshufd zmm2,zmm2,0x39 + vpaddd zmm0,zmm0,zmm6 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x10 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0xc + vpaddd zmm0,zmm0,zmm7 + vpaddd zmm0,zmm0,zmm1 + vpxord zmm3,zmm3,zmm0 + vprord zmm3,zmm3,0x8 + vpaddd zmm2,zmm2,zmm3 + vpxord zmm1,zmm1,zmm2 + vprord zmm1,zmm1,0x7 + vpshufd zmm0,zmm0,0x39 + vpshufd zmm3,zmm3,0x4e + vpshufd zmm2,zmm2,0x93 + dec al + je 3f + vshufps zmm8,zmm4,zmm5,0xd6 + vpshufd zmm9,zmm4,0xf + vpshufd zmm4,zmm8,0x39 + vshufps zmm8,zmm6,zmm7,0xfa + vpblendmd zmm9{k3},zmm9,zmm8 + vpunpcklqdq zmm8,zmm7,zmm5 + vpblendmd zmm8{k4},zmm8,zmm6 + vpshufd zmm8,zmm8,0x78 + vpunpckhdq zmm5,zmm5,zmm7 + vpunpckldq zmm6,zmm6,zmm5 + vpshufd zmm7,zmm6,0x1e + vmovdqa32 zmm5,zmm9 + vmovdqa32 zmm6,zmm8 + jmp 3b +3: + vpxord zmm0,zmm0,zmm2 + vpxord zmm1,zmm1,zmm3 + vbroadcasti32x4 zmm8,XMMWORD PTR [rdi] + vbroadcasti32x4 zmm9,XMMWORD PTR [rdi+0x10] + vpxord zmm2,zmm2,zmm8 + vpxord zmm3,zmm3,zmm9 + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + vextracti128 XMMWORD PTR [r9+0x40],ymm0,0x1 + vextracti128 XMMWORD PTR [r9+0x50],ymm1,0x1 + vextracti128 XMMWORD PTR [r9+0x60],ymm2,0x1 + vextracti128 XMMWORD PTR [r9+0x70],ymm3,0x1 + vextracti32x4 XMMWORD PTR [r9+0x80],zmm0,0x2 + vextracti32x4 XMMWORD PTR [r9+0x90],zmm1,0x2 + vextracti32x4 XMMWORD PTR [r9+0xa0],zmm2,0x2 + vextracti32x4 XMMWORD PTR [r9+0xb0],zmm3,0x2 + vextracti32x4 XMMWORD PTR [r9+0xc0],zmm0,0x3 + vextracti32x4 XMMWORD PTR [r9+0xd0],zmm1,0x3 + vextracti32x4 XMMWORD PTR [r9+0xe0],zmm2,0x3 + vextracti32x4 XMMWORD PTR [r9+0xf0],zmm3,0x3 + vmovdqa xmm0,XMMWORD PTR [rsp+0x10] + vmovdqa xmm1,XMMWORD PTR [rsp+0x50] + vmovdqa XMMWORD PTR [rsp],xmm0 + vmovdqa XMMWORD PTR [rsp+0x40],xmm1 + add r9,0x100 + sub r10,0x4 +2: + test r10,0x2 + je 2f + vbroadcasti128 ymm0,XMMWORD PTR [rdi] + vbroadcasti128 ymm1,XMMWORD PTR [rdi+0x10] + vmovd xmm13,DWORD PTR [rsp] + vpinsrd xmm13,xmm13,DWORD PTR [rsp+0x40],0x1 + vpinsrd xmm13,xmm13,edx,0x2 + vmovd xmm14,DWORD PTR [rsp+0x4] + vpinsrd xmm14,xmm14,DWORD PTR [rsp+0x44],0x1 + vpinsrd xmm14,xmm14,edx,0x2 + vinserti128 ymm13,ymm13,xmm14,0x1 + vbroadcasti128 ymm2,XMMWORD PTR [BLAKE3_IV+rip] + vpbroadcastd ymm8,r8d + vpblendd ymm3,ymm13,ymm8,0x88 + vbroadcasti128 ymm8,XMMWORD PTR [rsi] + vbroadcasti128 ymm9,XMMWORD PTR [rsi+0x10] + vshufps ymm4,ymm8,ymm9,0x88 + vshufps ymm5,ymm8,ymm9,0xdd + vbroadcasti128 ymm8,XMMWORD PTR [rsi+0x20] + vbroadcasti128 ymm9,XMMWORD PTR [rsi+0x30] + vshufps ymm6,ymm8,ymm9,0x88 + vshufps ymm7,ymm8,ymm9,0xdd + vpshufd ymm6,ymm6,0x93 + vpshufd ymm7,ymm7,0x93 + mov al,0x7 +3: + vpaddd ymm0,ymm0,ymm4 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x10 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0xc + vpaddd ymm0,ymm0,ymm5 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x8 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0x7 + vpshufd ymm0,ymm0,0x93 + vpshufd ymm3,ymm3,0x4e + vpshufd ymm2,ymm2,0x39 + vpaddd ymm0,ymm0,ymm6 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x10 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0xc + vpaddd ymm0,ymm0,ymm7 + vpaddd ymm0,ymm0,ymm1 + vpxord ymm3,ymm3,ymm0 + vprord ymm3,ymm3,0x8 + vpaddd ymm2,ymm2,ymm3 + vpxord ymm1,ymm1,ymm2 + vprord ymm1,ymm1,0x7 + vpshufd ymm0,ymm0,0x39 + vpshufd ymm3,ymm3,0x4e + vpshufd ymm2,ymm2,0x93 + dec al + je 3f + vshufps ymm8,ymm4,ymm5,0xd6 + vpshufd ymm9,ymm4,0xf + vpshufd ymm4,ymm8,0x39 + vshufps ymm8,ymm6,ymm7,0xfa + vpblendd ymm9,ymm9,ymm8,0xaa + vpunpcklqdq ymm8,ymm7,ymm5 + vpblendd ymm8,ymm8,ymm6,0x88 + vpshufd ymm8,ymm8,0x78 + vpunpckhdq ymm5,ymm5,ymm7 + vpunpckldq ymm6,ymm6,ymm5 + vpshufd ymm7,ymm6,0x1e + vmovdqa ymm5,ymm9 + vmovdqa ymm6,ymm8 + jmp 3b +3: + vpxor ymm0,ymm0,ymm2 + vpxor ymm1,ymm1,ymm3 + vbroadcasti128 ymm8,XMMWORD PTR [rdi] + vbroadcasti128 ymm9,XMMWORD PTR [rdi+0x10] + vpxor ymm2,ymm2,ymm8 + vpxor ymm3,ymm3,ymm9 + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + vextracti128 XMMWORD PTR [r9+0x40],ymm0,0x1 + vextracti128 XMMWORD PTR [r9+0x50],ymm1,0x1 + vextracti128 XMMWORD PTR [r9+0x60],ymm2,0x1 + vextracti128 XMMWORD PTR [r9+0x70],ymm3,0x1 + vmovdqu xmm0,XMMWORD PTR [rsp+0x8] + vmovdqu xmm1,XMMWORD PTR [rsp+0x48] + vmovdqa XMMWORD PTR [rsp],xmm0 + vmovdqa XMMWORD PTR [rsp+0x40],xmm1 + add r9,0x80 + sub r10,0x2 +2: + test r10,0x1 + je 9b + vmovdqu xmm0,XMMWORD PTR [rdi] + vmovdqu xmm1,XMMWORD PTR [rdi+0x10] + vmovd xmm14,DWORD PTR [rsp] + vpinsrd xmm14,xmm14,DWORD PTR [rsp+0x40],0x1 + vpinsrd xmm14,xmm14,edx,0x2 + vmovdqa xmm2,XMMWORD PTR [BLAKE3_IV+rip] + vpinsrd xmm3,xmm14,r8d,0x3 + vmovups xmm8,XMMWORD PTR [rsi] + vmovups xmm9,XMMWORD PTR [rsi+0x10] + vshufps xmm4,xmm8,xmm9,0x88 + vshufps xmm5,xmm8,xmm9,0xdd + vmovups xmm8,XMMWORD PTR [rsi+0x20] + vmovups xmm9,XMMWORD PTR [rsi+0x30] + vshufps xmm6,xmm8,xmm9,0x88 + vshufps xmm7,xmm8,xmm9,0xdd + vpshufd xmm6,xmm6,0x93 + vpshufd xmm7,xmm7,0x93 + mov al,0x7 +3: + vpaddd xmm0,xmm0,xmm4 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm5 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x93 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x39 + vpaddd xmm0,xmm0,xmm6 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x10 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0xc + vpaddd xmm0,xmm0,xmm7 + vpaddd xmm0,xmm0,xmm1 + vpxord xmm3,xmm3,xmm0 + vprord xmm3,xmm3,0x8 + vpaddd xmm2,xmm2,xmm3 + vpxord xmm1,xmm1,xmm2 + vprord xmm1,xmm1,0x7 + vpshufd xmm0,xmm0,0x39 + vpshufd xmm3,xmm3,0x4e + vpshufd xmm2,xmm2,0x93 + dec al + je 3f + vshufps xmm8,xmm4,xmm5,0xd6 + vpshufd xmm9,xmm4,0xf + vpshufd xmm4,xmm8,0x39 + vshufps xmm8,xmm6,xmm7,0xfa + vpblendd xmm9,xmm9,xmm8,0xaa + vpunpcklqdq xmm8,xmm7,xmm5 + vpblendd xmm8,xmm8,xmm6,0x88 + vpshufd xmm8,xmm8,0x78 + vpunpckhdq xmm5,xmm5,xmm7 + vpunpckldq xmm6,xmm6,xmm5 + vpshufd xmm7,xmm6,0x1e + vmovdqa xmm5,xmm9 + vmovdqa xmm6,xmm8 + jmp 3b +3: + vpxor xmm0,xmm0,xmm2 + vpxor xmm1,xmm1,xmm3 + vpxor xmm2,xmm2,XMMWORD PTR [rdi] + vpxor xmm3,xmm3,XMMWORD PTR [rdi+0x10] + vmovdqu XMMWORD PTR [r9],xmm0 + vmovdqu XMMWORD PTR [r9+0x10],xmm1 + vmovdqu XMMWORD PTR [r9+0x20],xmm2 + vmovdqu XMMWORD PTR [r9+0x30],xmm3 + jmp 9b + + #ifdef __APPLE__ .static_data #else diff --git a/src/third_party/blake3/asm/blake3_avx512_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_avx512_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/c/blake3_dispatch.c b/src/third_party/blake3/blake3_dispatch.c similarity index 85% rename from src/third_party/blake3/c/blake3_dispatch.c rename to src/third_party/blake3/blake3_dispatch.c index 88f6bc88..0092a36c 100644 --- a/src/third_party/blake3/c/blake3_dispatch.c +++ b/src/third_party/blake3/blake3_dispatch.c @@ -4,43 +4,23 @@ #include "blake3_impl.h" + #if defined(IS_X86) -#if defined(_MSC_VER) + +#if defined(__GNUC__) || defined(__clang__) +#define ATOMIC_INT int +#define ATOMIC_LOAD(x) __atomic_load_n(&x, __ATOMIC_RELAXED) +#define ATOMIC_STORE(x, y) __atomic_store_n(&x, y, __ATOMIC_RELAXED) +#else #include -#elif defined(__GNUC__) -#include -#else -#undef IS_X86 /* Unimplemented! */ -#endif +#define ATOMIC_INT int +#define ATOMIC_LOAD(x) __iso_volatile_load32(&x) +#define ATOMIC_STORE(x, y) __iso_volatile_store32(&x, y) #endif - -#if !defined(BLAKE3_ATOMICS) -#if defined(__has_include) -#if __has_include() && !defined(_MSC_VER) -#define BLAKE3_ATOMICS 1 -#else -#define BLAKE3_ATOMICS 0 -#endif /* __has_include() && !defined(_MSC_VER) */ -#else -#define BLAKE3_ATOMICS 0 -#endif /* defined(__has_include) */ -#endif /* BLAKE3_ATOMICS */ - -#if BLAKE3_ATOMICS -#define ATOMIC_INT _Atomic int -#define ATOMIC_LOAD(x) x -#define ATOMIC_STORE(x, y) x = y -#elif defined(_MSC_VER) -#define ATOMIC_INT long -#define ATOMIC_LOAD(x) _InterlockedOr(&x, 0) -#define ATOMIC_STORE(x, y) _InterlockedExchange(&x, y) -#else -#define ATOMIC_INT int -#define ATOMIC_LOAD(x) x -#define ATOMIC_STORE(x, y) x = y #endif + #define MAYBE_UNUSED(x) (void)((x)) #if defined(IS_X86) @@ -86,7 +66,6 @@ static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) { #endif } -#endif enum cpu_feature { SSE2 = 1 << 0, @@ -112,9 +91,9 @@ static get_cpu_features(void) { /* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */ - long features = ATOMIC_LOAD(g_cpu_features); + enum cpu_feature features = ATOMIC_LOAD(g_cpu_features); if (features != UNDEFINED) { - return (enum cpu_feature)features; + return features; } else { #if defined(IS_X86) uint32_t regs[4] = {0}; @@ -154,13 +133,14 @@ static } } ATOMIC_STORE(g_cpu_features, features); - return (enum cpu_feature)features; + return features; #else /* How to detect NEON? */ return 0; #endif } } +#endif void blake3_compress_in_place(uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], @@ -220,6 +200,30 @@ void blake3_compress_xof(const uint32_t cv[8], blake3_compress_xof_portable(cv, block, block_len, counter, flags, out); } + +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks) { + if (outblocks == 0) { + // The current assembly implementation always outputs at least 1 block. + return; + } +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(_WIN32) && !defined(__CYGWIN__) && !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_xof_many_avx512(cv, block, block_len, counter, flags, out, outblocks); + return; + } +#endif +#endif + for(size_t i = 0; i < outblocks; ++i) { + blake3_compress_xof(cv, block, block_len, counter + i, flags, out + 64*i); + } +} + void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, diff --git a/src/third_party/blake3/asm/blake3_impl.h b/src/third_party/blake3/blake3_impl.h similarity index 84% rename from src/third_party/blake3/asm/blake3_impl.h rename to src/third_party/blake3/blake3_impl.h index beab5cf5..88e71e41 100644 --- a/src/third_party/blake3/asm/blake3_impl.h +++ b/src/third_party/blake3/blake3_impl.h @@ -9,6 +9,10 @@ #include "blake3.h" +#ifdef __cplusplus +extern "C" { +#endif + // internal flags enum blake3_flags { CHUNK_START = 1 << 0, @@ -28,7 +32,13 @@ enum blake3_flags { #define INLINE static inline __attribute__((always_inline)) #endif -#if defined(__x86_64__) || defined(_M_X64) +#ifdef __cplusplus +#define NOEXCEPT noexcept +#else +#define NOEXCEPT +#endif + +#if (defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC) #define IS_X86 #define IS_X86_64 #endif @@ -38,7 +48,7 @@ enum blake3_flags { #define IS_X86_32 #endif -#if defined(__aarch64__) || defined(_M_ARM64) +#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) #define IS_AARCH64 #endif @@ -162,6 +172,13 @@ INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], key_words[7] = load32(&key[7 * 4]); } +INLINE void load_block_words(const uint8_t block[BLAKE3_BLOCK_LEN], + uint32_t block_words[16]) { + for (size_t i = 0; i < 16; i++) { + block_words[i] = load32(&block[i * 4]); + } +} + INLINE void store32(void *dst, uint32_t w) { uint8_t *p = (uint8_t *)dst; p[0] = (uint8_t)(w >> 0); @@ -191,6 +208,11 @@ void blake3_compress_xof(const uint32_t cv[8], uint8_t block_len, uint64_t counter, uint8_t flags, uint8_t out[64]); +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks); + void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, @@ -198,6 +220,22 @@ void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, size_t blake3_simd_degree(void); +BLAKE3_PRIVATE size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb); + +#if defined(BLAKE3_USE_TBB) +BLAKE3_PRIVATE void blake3_compress_subtree_wide_join_tbb( + // shared params + const uint32_t key[8], uint8_t flags, bool use_tbb, + // left-hand side params + const uint8_t *l_input, size_t l_input_len, uint64_t l_chunk_counter, + uint8_t *l_cvs, size_t *l_n, + // right-hand side params + const uint8_t *r_input, size_t r_input_len, uint64_t r_chunk_counter, + uint8_t *r_cvs, size_t *r_n) NOEXCEPT; +#endif // Declarations for implementation-specific functions. void blake3_compress_in_place_portable(uint32_t cv[8], @@ -270,6 +308,13 @@ void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out); + +#if !defined(_WIN32) && !defined(__CYGWIN__) +void blake3_xof_many_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t* out, size_t outblocks); +#endif #endif #endif @@ -281,5 +326,8 @@ void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, uint8_t flags_end, uint8_t *out); #endif +#ifdef __cplusplus +} +#endif #endif /* BLAKE3_IMPL_H */ diff --git a/src/third_party/blake3/asm/blake3_neon.c b/src/third_party/blake3/blake3_neon.c similarity index 96% rename from src/third_party/blake3/asm/blake3_neon.c rename to src/third_party/blake3/blake3_neon.c index 8a818fc7..794ea80f 100644 --- a/src/third_party/blake3/asm/blake3_neon.c +++ b/src/third_party/blake3/blake3_neon.c @@ -10,14 +10,12 @@ INLINE uint32x4_t loadu_128(const uint8_t src[16]) { // vld1q_u32 has alignment requirements. Don't use it. - uint32x4_t x; - memcpy(&x, src, 16); - return x; + return vreinterpretq_u32_u8(vld1q_u8(src)); } INLINE void storeu_128(uint32x4_t src, uint8_t dest[16]) { // vst1q_u32 has alignment requirements. Don't use it. - memcpy(dest, &src, 16); + vst1q_u8(dest, vreinterpretq_u8_u32(src)); } INLINE uint32x4_t add_128(uint32x4_t a, uint32x4_t b) { @@ -36,7 +34,7 @@ INLINE uint32x4_t set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { } INLINE uint32x4_t rot16_128(uint32x4_t x) { - // The straightfoward implementation would be two shifts and an or, but that's + // The straightforward implementation would be two shifts and an or, but that's // slower on microarchitectures we've tested. See // https://github.com/BLAKE3-team/BLAKE3/pull/319. // return vorrq_u32(vshrq_n_u32(x, 16), vshlq_n_u32(x, 32 - 16)); @@ -245,10 +243,11 @@ INLINE void load_counters4(uint64_t counter, bool increment_counter, counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3))); } -void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { +static void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, + uint8_t *out) { uint32x4_t h_vecs[8] = { set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), diff --git a/src/third_party/blake3/asm/blake3_portable.c b/src/third_party/blake3/blake3_portable.c similarity index 100% rename from src/third_party/blake3/asm/blake3_portable.c rename to src/third_party/blake3/blake3_portable.c diff --git a/src/third_party/blake3/asm/blake3_sse2_x86-64_unix.S b/src/third_party/blake3/blake3_sse2_x86-64_unix.S similarity index 100% rename from src/third_party/blake3/asm/blake3_sse2_x86-64_unix.S rename to src/third_party/blake3/blake3_sse2_x86-64_unix.S diff --git a/src/third_party/blake3/asm/blake3_sse2_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_sse2_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/asm/blake3_sse41_x86-64_unix.S b/src/third_party/blake3/blake3_sse41_x86-64_unix.S similarity index 100% rename from src/third_party/blake3/asm/blake3_sse41_x86-64_unix.S rename to src/third_party/blake3/blake3_sse41_x86-64_unix.S diff --git a/src/third_party/blake3/asm/blake3_sse41_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_sse41_x86-64_windows_msvc.asm similarity index 100% rename from src/third_party/blake3/asm/blake3_sse41_x86-64_windows_msvc.asm rename to src/third_party/blake3/blake3_sse41_x86-64_windows_msvc.asm diff --git a/src/third_party/blake3/c/LICENSE b/src/third_party/blake3/c/LICENSE deleted file mode 100644 index f5892efc..00000000 --- a/src/third_party/blake3/c/LICENSE +++ /dev/null @@ -1,330 +0,0 @@ -This work is released into the public domain with CC0 1.0. Alternatively, it is -licensed under the Apache License 2.0. - -------------------------------------------------------------------------------- - -Creative Commons Legal Code - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not -limited to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and -vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention -of, applicable law, Affirmer hereby overtly, fully, permanently, -irrevocably and unconditionally waives, abandons, and surrenders all of -Affirmer's Copyright and Related Rights and associated claims and causes -of action, whether now known or unknown (including existing as well as -future claims and causes of action), in the Work (i) in all territories -worldwide, (ii) for the maximum duration provided by applicable law or -treaty (including future time extensions), (iii) in any current or future -medium and for any number of copies, and (iv) for any purpose whatsoever, -including without limitation commercial, advertising or promotional -purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each -member of the public at large and to the detriment of Affirmer's heirs and -successors, fully intending that such Waiver shall not be subject to -revocation, rescission, cancellation, termination, or any other legal or -equitable action to disrupt the quiet enjoyment of the Work by the public -as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason -be judged legally invalid or ineffective under applicable law, then the -Waiver shall be preserved to the maximum extent permitted taking into -account Affirmer's express Statement of Purpose. In addition, to the -extent the Waiver is so judged Affirmer hereby grants to each affected -person a royalty-free, non transferable, non sublicensable, non exclusive, -irrevocable and unconditional license to exercise Affirmer's Copyright and -Related Rights in the Work (i) in all territories worldwide, (ii) for the -maximum duration provided by applicable law or treaty (including future -time extensions), (iii) in any current or future medium and for any number -of copies, and (iv) for any purpose whatsoever, including without -limitation commercial, advertising or promotional purposes (the -"License"). The License shall be deemed effective as of the date CC0 was -applied by Affirmer to the Work. Should any part of the License for any -reason be judged legally invalid or ineffective under applicable law, such -partial invalidity or ineffectiveness shall not invalidate the remainder -of the License, and in such case Affirmer hereby affirms that he or she -will not (i) exercise any of his or her remaining Copyright and Related -Rights in the Work or (ii) assert any associated claims and causes of -action with respect to the Work, in either case contrary to Affirmer's -express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. - -------------------------------------------------------------------------------- - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2019 Jack O'Connor and Samuel Neves - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/third_party/blake3/c/blake3.c b/src/third_party/blake3/c/blake3.c deleted file mode 100644 index 692f4b02..00000000 --- a/src/third_party/blake3/c/blake3.c +++ /dev/null @@ -1,616 +0,0 @@ -#include -#include -#include - -#include "blake3.h" -#include "blake3_impl.h" - -const char *blake3_version(void) { return BLAKE3_VERSION_STRING; } - -INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8], - uint8_t flags) { - memcpy(self->cv, key, BLAKE3_KEY_LEN); - self->chunk_counter = 0; - memset(self->buf, 0, BLAKE3_BLOCK_LEN); - self->buf_len = 0; - self->blocks_compressed = 0; - self->flags = flags; -} - -INLINE void chunk_state_reset(blake3_chunk_state *self, const uint32_t key[8], - uint64_t chunk_counter) { - memcpy(self->cv, key, BLAKE3_KEY_LEN); - self->chunk_counter = chunk_counter; - self->blocks_compressed = 0; - memset(self->buf, 0, BLAKE3_BLOCK_LEN); - self->buf_len = 0; -} - -INLINE size_t chunk_state_len(const blake3_chunk_state *self) { - return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) + - ((size_t)self->buf_len); -} - -INLINE size_t chunk_state_fill_buf(blake3_chunk_state *self, - const uint8_t *input, size_t input_len) { - size_t take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len); - if (take > input_len) { - take = input_len; - } - uint8_t *dest = self->buf + ((size_t)self->buf_len); - memcpy(dest, input, take); - self->buf_len += (uint8_t)take; - return take; -} - -INLINE uint8_t chunk_state_maybe_start_flag(const blake3_chunk_state *self) { - if (self->blocks_compressed == 0) { - return CHUNK_START; - } else { - return 0; - } -} - -typedef struct { - uint32_t input_cv[8]; - uint64_t counter; - uint8_t block[BLAKE3_BLOCK_LEN]; - uint8_t block_len; - uint8_t flags; -} output_t; - -INLINE output_t make_output(const uint32_t input_cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - output_t ret; - memcpy(ret.input_cv, input_cv, 32); - memcpy(ret.block, block, BLAKE3_BLOCK_LEN); - ret.block_len = block_len; - ret.counter = counter; - ret.flags = flags; - return ret; -} - -// Chaining values within a given chunk (specifically the compress_in_place -// interface) are represented as words. This avoids unnecessary bytes<->words -// conversion overhead in the portable implementation. However, the hash_many -// interface handles both user input and parent node blocks, so it accepts -// bytes. For that reason, chaining values in the CV stack are represented as -// bytes. -INLINE void output_chaining_value(const output_t *self, uint8_t cv[32]) { - uint32_t cv_words[8]; - memcpy(cv_words, self->input_cv, 32); - blake3_compress_in_place(cv_words, self->block, self->block_len, - self->counter, self->flags); - store_cv_words(cv, cv_words); -} - -INLINE void output_root_bytes(const output_t *self, uint64_t seek, uint8_t *out, - size_t out_len) { - uint64_t output_block_counter = seek / 64; - size_t offset_within_block = seek % 64; - uint8_t wide_buf[64]; - while (out_len > 0) { - blake3_compress_xof(self->input_cv, self->block, self->block_len, - output_block_counter, self->flags | ROOT, wide_buf); - size_t available_bytes = 64 - offset_within_block; - size_t memcpy_len; - if (out_len > available_bytes) { - memcpy_len = available_bytes; - } else { - memcpy_len = out_len; - } - memcpy(out, wide_buf + offset_within_block, memcpy_len); - out += memcpy_len; - out_len -= memcpy_len; - output_block_counter += 1; - offset_within_block = 0; - } -} - -INLINE void chunk_state_update(blake3_chunk_state *self, const uint8_t *input, - size_t input_len) { - if (self->buf_len > 0) { - size_t take = chunk_state_fill_buf(self, input, input_len); - input += take; - input_len -= take; - if (input_len > 0) { - blake3_compress_in_place( - self->cv, self->buf, BLAKE3_BLOCK_LEN, self->chunk_counter, - self->flags | chunk_state_maybe_start_flag(self)); - self->blocks_compressed += 1; - self->buf_len = 0; - memset(self->buf, 0, BLAKE3_BLOCK_LEN); - } - } - - while (input_len > BLAKE3_BLOCK_LEN) { - blake3_compress_in_place(self->cv, input, BLAKE3_BLOCK_LEN, - self->chunk_counter, - self->flags | chunk_state_maybe_start_flag(self)); - self->blocks_compressed += 1; - input += BLAKE3_BLOCK_LEN; - input_len -= BLAKE3_BLOCK_LEN; - } - - size_t take = chunk_state_fill_buf(self, input, input_len); - input += take; - input_len -= take; -} - -INLINE output_t chunk_state_output(const blake3_chunk_state *self) { - uint8_t block_flags = - self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END; - return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter, - block_flags); -} - -INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], - const uint32_t key[8], uint8_t flags) { - return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT); -} - -// Given some input larger than one chunk, return the number of bytes that -// should go in the left subtree. This is the largest power-of-2 number of -// chunks that leaves at least 1 byte for the right subtree. -INLINE size_t left_len(size_t content_len) { - // Subtract 1 to reserve at least one byte for the right side. content_len - // should always be greater than BLAKE3_CHUNK_LEN. - size_t full_chunks = (content_len - 1) / BLAKE3_CHUNK_LEN; - return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; -} - -// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE chunks at the same time -// on a single thread. Write out the chunk chaining values and return the -// number of chunks hashed. These chunks are never the root and never empty; -// those cases use a different codepath. -INLINE size_t compress_chunks_parallel(const uint8_t *input, size_t input_len, - const uint32_t key[8], - uint64_t chunk_counter, uint8_t flags, - uint8_t *out) { -#if defined(BLAKE3_TESTING) - assert(0 < input_len); - assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN); -#endif - - const uint8_t *chunks_array[MAX_SIMD_DEGREE]; - size_t input_position = 0; - size_t chunks_array_len = 0; - while (input_len - input_position >= BLAKE3_CHUNK_LEN) { - chunks_array[chunks_array_len] = &input[input_position]; - input_position += BLAKE3_CHUNK_LEN; - chunks_array_len += 1; - } - - blake3_hash_many(chunks_array, chunks_array_len, - BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN, key, chunk_counter, - true, flags, CHUNK_START, CHUNK_END, out); - - // Hash the remaining partial chunk, if there is one. Note that the empty - // chunk (meaning the empty message) is a different codepath. - if (input_len > input_position) { - uint64_t counter = chunk_counter + (uint64_t)chunks_array_len; - blake3_chunk_state chunk_state; - chunk_state_init(&chunk_state, key, flags); - chunk_state.chunk_counter = counter; - chunk_state_update(&chunk_state, &input[input_position], - input_len - input_position); - output_t output = chunk_state_output(&chunk_state); - output_chaining_value(&output, &out[chunks_array_len * BLAKE3_OUT_LEN]); - return chunks_array_len + 1; - } else { - return chunks_array_len; - } -} - -// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE parents at the same time -// on a single thread. Write out the parent chaining values and return the -// number of parents hashed. (If there's an odd input chaining value left over, -// return it as an additional output.) These parents are never the root and -// never empty; those cases use a different codepath. -INLINE size_t compress_parents_parallel(const uint8_t *child_chaining_values, - size_t num_chaining_values, - const uint32_t key[8], uint8_t flags, - uint8_t *out) { -#if defined(BLAKE3_TESTING) - assert(2 <= num_chaining_values); - assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2); -#endif - - const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2]; - size_t parents_array_len = 0; - while (num_chaining_values - (2 * parents_array_len) >= 2) { - parents_array[parents_array_len] = - &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN]; - parents_array_len += 1; - } - - blake3_hash_many(parents_array, parents_array_len, 1, key, - 0, // Parents always use counter 0. - false, flags | PARENT, - 0, // Parents have no start flags. - 0, // Parents have no end flags. - out); - - // If there's an odd child left over, it becomes an output. - if (num_chaining_values > 2 * parents_array_len) { - memcpy(&out[parents_array_len * BLAKE3_OUT_LEN], - &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN], - BLAKE3_OUT_LEN); - return parents_array_len + 1; - } else { - return parents_array_len; - } -} - -// The wide helper function returns (writes out) an array of chaining values -// and returns the length of that array. The number of chaining values returned -// is the dynamically detected SIMD degree, at most MAX_SIMD_DEGREE. Or fewer, -// if the input is shorter than that many chunks. The reason for maintaining a -// wide array of chaining values going back up the tree, is to allow the -// implementation to hash as many parents in parallel as possible. -// -// As a special case when the SIMD degree is 1, this function will still return -// at least 2 outputs. This guarantees that this function doesn't perform the -// root compression. (If it did, it would use the wrong flags, and also we -// wouldn't be able to implement extendable output.) Note that this function is -// not used when the whole input is only 1 chunk long; that's a different -// codepath. -// -// Why not just have the caller split the input on the first update(), instead -// of implementing this special rule? Because we don't want to limit SIMD or -// multi-threading parallelism for that update(). -static size_t blake3_compress_subtree_wide(const uint8_t *input, - size_t input_len, - const uint32_t key[8], - uint64_t chunk_counter, - uint8_t flags, uint8_t *out) { - // Note that the single chunk case does *not* bump the SIMD degree up to 2 - // when it is 1. If this implementation adds multi-threading in the future, - // this gives us the option of multi-threading even the 2-chunk case, which - // can help performance on smaller platforms. - if (input_len <= blake3_simd_degree() * BLAKE3_CHUNK_LEN) { - return compress_chunks_parallel(input, input_len, key, chunk_counter, flags, - out); - } - - // With more than simd_degree chunks, we need to recurse. Start by dividing - // the input into left and right subtrees. (Note that this is only optimal - // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree - // of 3 or something, we'll need a more complicated strategy.) - size_t left_input_len = left_len(input_len); - size_t right_input_len = input_len - left_input_len; - const uint8_t *right_input = &input[left_input_len]; - uint64_t right_chunk_counter = - chunk_counter + (uint64_t)(left_input_len / BLAKE3_CHUNK_LEN); - - // Make space for the child outputs. Here we use MAX_SIMD_DEGREE_OR_2 to - // account for the special case of returning 2 outputs when the SIMD degree - // is 1. - uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; - size_t degree = blake3_simd_degree(); - if (left_input_len > BLAKE3_CHUNK_LEN && degree == 1) { - // The special case: We always use a degree of at least two, to make - // sure there are two outputs. Except, as noted above, at the chunk - // level, where we allow degree=1. (Note that the 1-chunk-input case is - // a different codepath.) - degree = 2; - } - uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; - - // Recurse! If this implementation adds multi-threading support in the - // future, this is where it will go. - size_t left_n = blake3_compress_subtree_wide(input, left_input_len, key, - chunk_counter, flags, cv_array); - size_t right_n = blake3_compress_subtree_wide( - right_input, right_input_len, key, right_chunk_counter, flags, right_cvs); - - // The special case again. If simd_degree=1, then we'll have left_n=1 and - // right_n=1. Rather than compressing them into a single output, return - // them directly, to make sure we always have at least two outputs. - if (left_n == 1) { - memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); - return 2; - } - - // Otherwise, do one layer of parent node compression. - size_t num_chaining_values = left_n + right_n; - return compress_parents_parallel(cv_array, num_chaining_values, key, flags, - out); -} - -// Hash a subtree with compress_subtree_wide(), and then condense the resulting -// list of chaining values down to a single parent node. Don't compress that -// last parent node, however. Instead, return its message bytes (the -// concatenated chaining values of its children). This is necessary when the -// first call to update() supplies a complete subtree, because the topmost -// parent node of that subtree could end up being the root. It's also necessary -// for extended output in the general case. -// -// As with compress_subtree_wide(), this function is not used on inputs of 1 -// chunk or less. That's a different codepath. -INLINE void compress_subtree_to_parent_node( - const uint8_t *input, size_t input_len, const uint32_t key[8], - uint64_t chunk_counter, uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN]) { -#if defined(BLAKE3_TESTING) - assert(input_len > BLAKE3_CHUNK_LEN); -#endif - - uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; - size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, - chunk_counter, flags, cv_array); - assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); - - // If MAX_SIMD_DEGREE is greater than 2 and there's enough input, - // compress_subtree_wide() returns more than 2 chaining values. Condense - // them into 2 by forming parent nodes repeatedly. - uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; - // The second half of this loop condition is always true, and we just - // asserted it above. But GCC can't tell that it's always true, and if NDEBUG - // is set on platforms where MAX_SIMD_DEGREE_OR_2 == 2, GCC emits spurious - // warnings here. GCC 8.5 is particularly sensitive, so if you're changing - // this code, test it against that version. - while (num_cvs > 2 && num_cvs <= MAX_SIMD_DEGREE_OR_2) { - num_cvs = - compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); - memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); - } - memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); -} - -INLINE void hasher_init_base(blake3_hasher *self, const uint32_t key[8], - uint8_t flags) { - memcpy(self->key, key, BLAKE3_KEY_LEN); - chunk_state_init(&self->chunk, key, flags); - self->cv_stack_len = 0; -} - -void blake3_hasher_init(blake3_hasher *self) { hasher_init_base(self, IV, 0); } - -void blake3_hasher_init_keyed(blake3_hasher *self, - const uint8_t key[BLAKE3_KEY_LEN]) { - uint32_t key_words[8]; - load_key_words(key, key_words); - hasher_init_base(self, key_words, KEYED_HASH); -} - -void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, - size_t context_len) { - blake3_hasher context_hasher; - hasher_init_base(&context_hasher, IV, DERIVE_KEY_CONTEXT); - blake3_hasher_update(&context_hasher, context, context_len); - uint8_t context_key[BLAKE3_KEY_LEN]; - blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN); - uint32_t context_key_words[8]; - load_key_words(context_key, context_key_words); - hasher_init_base(self, context_key_words, DERIVE_KEY_MATERIAL); -} - -void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context) { - blake3_hasher_init_derive_key_raw(self, context, strlen(context)); -} - -// As described in hasher_push_cv() below, we do "lazy merging", delaying -// merges until right before the next CV is about to be added. This is -// different from the reference implementation. Another difference is that we -// aren't always merging 1 chunk at a time. Instead, each CV might represent -// any power-of-two number of chunks, as long as the smaller-above-larger stack -// order is maintained. Instead of the "count the trailing 0-bits" algorithm -// described in the spec, we use a "count the total number of 1-bits" variant -// that doesn't require us to retain the subtree size of the CV on top of the -// stack. The principle is the same: each CV that should remain in the stack is -// represented by a 1-bit in the total number of chunks (or bytes) so far. -INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { - size_t post_merge_stack_len = (size_t)popcnt(total_len); - while (self->cv_stack_len > post_merge_stack_len) { - uint8_t *parent_node = - &self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN]; - output_t output = parent_output(parent_node, self->key, self->chunk.flags); - output_chaining_value(&output, parent_node); - self->cv_stack_len -= 1; - } -} - -// In reference_impl.rs, we merge the new CV with existing CVs from the stack -// before pushing it. We can do that because we know more input is coming, so -// we know none of the merges are root. -// -// This setting is different. We want to feed as much input as possible to -// compress_subtree_wide(), without setting aside anything for the chunk_state. -// If the user gives us 64 KiB, we want to parallelize over all 64 KiB at once -// as a single subtree, if at all possible. -// -// This leads to two problems: -// 1) This 64 KiB input might be the only call that ever gets made to update. -// In this case, the root node of the 64 KiB subtree would be the root node -// of the whole tree, and it would need to be ROOT finalized. We can't -// compress it until we know. -// 2) This 64 KiB input might complete a larger tree, whose root node is -// similarly going to be the the root of the whole tree. For example, maybe -// we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the -// node at the root of the 256 KiB subtree until we know how to finalize it. -// -// The second problem is solved with "lazy merging". That is, when we're about -// to add a CV to the stack, we don't merge it with anything first, as the -// reference impl does. Instead we do merges using the *previous* CV that was -// added, which is sitting on top of the stack, and we put the new CV -// (unmerged) on top of the stack afterwards. This guarantees that we never -// merge the root node until finalize(). -// -// Solving the first problem requires an additional tool, -// compress_subtree_to_parent_node(). That function always returns the top -// *two* chaining values of the subtree it's compressing. We then do lazy -// merging with each of them separately, so that the second CV will always -// remain unmerged. (That also helps us support extendable output when we're -// hashing an input all-at-once.) -INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], - uint64_t chunk_counter) { - hasher_merge_cv_stack(self, chunk_counter); - memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv, - BLAKE3_OUT_LEN); - self->cv_stack_len += 1; -} - -void blake3_hasher_update(blake3_hasher *self, const void *input, - size_t input_len) { - // Explicitly checking for zero avoids causing UB by passing a null pointer - // to memcpy. This comes up in practice with things like: - // std::vector v; - // blake3_hasher_update(&hasher, v.data(), v.size()); - if (input_len == 0) { - return; - } - - const uint8_t *input_bytes = (const uint8_t *)input; - - // If we have some partial chunk bytes in the internal chunk_state, we need - // to finish that chunk first. - if (chunk_state_len(&self->chunk) > 0) { - size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk); - if (take > input_len) { - take = input_len; - } - chunk_state_update(&self->chunk, input_bytes, take); - input_bytes += take; - input_len -= take; - // If we've filled the current chunk and there's more coming, finalize this - // chunk and proceed. In this case we know it's not the root. - if (input_len > 0) { - output_t output = chunk_state_output(&self->chunk); - uint8_t chunk_cv[32]; - output_chaining_value(&output, chunk_cv); - hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter); - chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1); - } else { - return; - } - } - - // Now the chunk_state is clear, and we have more input. If there's more than - // a single chunk (so, definitely not the root chunk), hash the largest whole - // subtree we can, with the full benefits of SIMD (and maybe in the future, - // multi-threading) parallelism. Two restrictions: - // - The subtree has to be a power-of-2 number of chunks. Only subtrees along - // the right edge can be incomplete, and we don't know where the right edge - // is going to be until we get to finalize(). - // - The subtree must evenly divide the total number of chunks up until this - // point (if total is not 0). If the current incomplete subtree is only - // waiting for 1 more chunk, we can't hash a subtree of 4 chunks. We have - // to complete the current subtree first. - // Because we might need to break up the input to form powers of 2, or to - // evenly divide what we already have, this part runs in a loop. - while (input_len > BLAKE3_CHUNK_LEN) { - size_t subtree_len = round_down_to_power_of_2(input_len); - uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN; - // Shrink the subtree_len until it evenly divides the count so far. We know - // that subtree_len itself is a power of 2, so we can use a bitmasking - // trick instead of an actual remainder operation. (Note that if the caller - // consistently passes power-of-2 inputs of the same size, as is hopefully - // typical, this loop condition will always fail, and subtree_len will - // always be the full length of the input.) - // - // An aside: We don't have to shrink subtree_len quite this much. For - // example, if count_so_far is 1, we could pass 2 chunks to - // compress_subtree_to_parent_node. Since we'll get 2 CVs back, we'll still - // get the right answer in the end, and we might get to use 2-way SIMD - // parallelism. The problem with this optimization, is that it gets us - // stuck always hashing 2 chunks. The total number of chunks will remain - // odd, and we'll never graduate to higher degrees of parallelism. See - // https://github.com/BLAKE3-team/BLAKE3/issues/69. - while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) { - subtree_len /= 2; - } - // The shrunken subtree_len might now be 1 chunk long. If so, hash that one - // chunk by itself. Otherwise, compress the subtree into a pair of CVs. - uint64_t subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN; - if (subtree_len <= BLAKE3_CHUNK_LEN) { - blake3_chunk_state chunk_state; - chunk_state_init(&chunk_state, self->key, self->chunk.flags); - chunk_state.chunk_counter = self->chunk.chunk_counter; - chunk_state_update(&chunk_state, input_bytes, subtree_len); - output_t output = chunk_state_output(&chunk_state); - uint8_t cv[BLAKE3_OUT_LEN]; - output_chaining_value(&output, cv); - hasher_push_cv(self, cv, chunk_state.chunk_counter); - } else { - // This is the high-performance happy path, though getting here depends - // on the caller giving us a long enough input. - uint8_t cv_pair[2 * BLAKE3_OUT_LEN]; - compress_subtree_to_parent_node(input_bytes, subtree_len, self->key, - self->chunk.chunk_counter, - self->chunk.flags, cv_pair); - hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); - hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], - self->chunk.chunk_counter + (subtree_chunks / 2)); - } - self->chunk.chunk_counter += subtree_chunks; - input_bytes += subtree_len; - input_len -= subtree_len; - } - - // If there's any remaining input less than a full chunk, add it to the chunk - // state. In that case, also do a final merge loop to make sure the subtree - // stack doesn't contain any unmerged pairs. The remaining input means we - // know these merges are non-root. This merge loop isn't strictly necessary - // here, because hasher_push_chunk_cv already does its own merge loop, but it - // simplifies blake3_hasher_finalize below. - if (input_len > 0) { - chunk_state_update(&self->chunk, input_bytes, input_len); - hasher_merge_cv_stack(self, self->chunk.chunk_counter); - } -} - -void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, - size_t out_len) { - blake3_hasher_finalize_seek(self, 0, out, out_len); -} - -void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, - uint8_t *out, size_t out_len) { - // Explicitly checking for zero avoids causing UB by passing a null pointer - // to memcpy. This comes up in practice with things like: - // std::vector v; - // blake3_hasher_finalize(&hasher, v.data(), v.size()); - if (out_len == 0) { - return; - } - - // If the subtree stack is empty, then the current chunk is the root. - if (self->cv_stack_len == 0) { - output_t output = chunk_state_output(&self->chunk); - output_root_bytes(&output, seek, out, out_len); - return; - } - // If there are any bytes in the chunk state, finalize that chunk and do a - // roll-up merge between that chunk hash and every subtree in the stack. In - // this case, the extra merge loop at the end of blake3_hasher_update - // guarantees that none of the subtrees in the stack need to be merged with - // each other first. Otherwise, if there are no bytes in the chunk state, - // then the top of the stack is a chunk hash, and we start the merge from - // that. - output_t output; - size_t cvs_remaining; - if (chunk_state_len(&self->chunk) > 0) { - cvs_remaining = self->cv_stack_len; - output = chunk_state_output(&self->chunk); - } else { - // There are always at least 2 CVs in the stack in this case. - cvs_remaining = self->cv_stack_len - 2; - output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key, - self->chunk.flags); - } - while (cvs_remaining > 0) { - cvs_remaining -= 1; - uint8_t parent_block[BLAKE3_BLOCK_LEN]; - memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32); - output_chaining_value(&output, &parent_block[32]); - output = parent_output(parent_block, self->key, self->chunk.flags); - } - output_root_bytes(&output, seek, out, out_len); -} - -void blake3_hasher_reset(blake3_hasher *self) { - chunk_state_reset(&self->chunk, self->key, 0); - self->cv_stack_len = 0; -} diff --git a/src/third_party/blake3/c/blake3_avx2.c b/src/third_party/blake3/c/blake3_avx2.c deleted file mode 100644 index 46b5cecf..00000000 --- a/src/third_party/blake3/c/blake3_avx2.c +++ /dev/null @@ -1,312 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define DEGREE 8 - -INLINE __m256i loadu(const uint8_t src[32]) { - return _mm256_loadu_si256((const __m256i *)src); -} - -INLINE void storeu(__m256i src, uint8_t dest[16]) { - _mm256_storeu_si256((__m256i *)dest, src); -} - -INLINE __m256i addv(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } - -// Note that clang-format doesn't like the name "xor" for some reason. -INLINE __m256i xorv(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } - -INLINE __m256i set1(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } - -INLINE __m256i rot16(__m256i x) { - return _mm256_shuffle_epi8( - x, _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, - 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); -} - -INLINE __m256i rot12(__m256i x) { - return _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 32 - 12)); -} - -INLINE __m256i rot8(__m256i x) { - return _mm256_shuffle_epi8( - x, _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1, - 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); -} - -INLINE __m256i rot7(__m256i x) { - return _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 32 - 7)); -} - -INLINE void round_fn(__m256i v[16], __m256i m[16], size_t r) { - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[15] = rot16(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot12(v[4]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[15] = rot8(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot7(v[4]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot16(v[15]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[4] = rot12(v[4]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot8(v[15]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - v[4] = rot7(v[4]); -} - -INLINE void transpose_vecs(__m256i vecs[DEGREE]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high - // is 22/33/66/77. - __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); - __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); - __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); - __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); - __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); - __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); - __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); - __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); - - // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is - // 11/33. - __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); - __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); - __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); - __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); - __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); - __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); - __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); - __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); - - // Interleave 128-bit lanes. - vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); - vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); - vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); - vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); - vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); - vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); - vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); - vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); -} - -INLINE void transpose_msg_vecs(const uint8_t *const *inputs, - size_t block_offset, __m256i out[16]) { - out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m256i)]); - out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m256i)]); - out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m256i)]); - out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m256i)]); - out[4] = loadu(&inputs[4][block_offset + 0 * sizeof(__m256i)]); - out[5] = loadu(&inputs[5][block_offset + 0 * sizeof(__m256i)]); - out[6] = loadu(&inputs[6][block_offset + 0 * sizeof(__m256i)]); - out[7] = loadu(&inputs[7][block_offset + 0 * sizeof(__m256i)]); - out[8] = loadu(&inputs[0][block_offset + 1 * sizeof(__m256i)]); - out[9] = loadu(&inputs[1][block_offset + 1 * sizeof(__m256i)]); - out[10] = loadu(&inputs[2][block_offset + 1 * sizeof(__m256i)]); - out[11] = loadu(&inputs[3][block_offset + 1 * sizeof(__m256i)]); - out[12] = loadu(&inputs[4][block_offset + 1 * sizeof(__m256i)]); - out[13] = loadu(&inputs[5][block_offset + 1 * sizeof(__m256i)]); - out[14] = loadu(&inputs[6][block_offset + 1 * sizeof(__m256i)]); - out[15] = loadu(&inputs[7][block_offset + 1 * sizeof(__m256i)]); - for (size_t i = 0; i < 8; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs(&out[0]); - transpose_vecs(&out[8]); -} - -INLINE void load_counters(uint64_t counter, bool increment_counter, - __m256i *out_lo, __m256i *out_hi) { - const __m256i mask = _mm256_set1_epi32(-(int32_t)increment_counter); - const __m256i add0 = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); - const __m256i add1 = _mm256_and_si256(mask, add0); - __m256i l = _mm256_add_epi32(_mm256_set1_epi32((int32_t)counter), add1); - __m256i carry = _mm256_cmpgt_epi32(_mm256_xor_si256(add1, _mm256_set1_epi32(0x80000000)), - _mm256_xor_si256( l, _mm256_set1_epi32(0x80000000))); - __m256i h = _mm256_sub_epi32(_mm256_set1_epi32((int32_t)(counter >> 32)), carry); - *out_lo = l; - *out_hi = h; -} - -static -void blake3_hash8_avx2(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m256i h_vecs[8] = { - set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), - set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), - }; - __m256i counter_low_vec, counter_high_vec; - load_counters(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m256i block_len_vec = set1(BLAKE3_BLOCK_LEN); - __m256i block_flags_vec = set1(block_flags); - __m256i msg_vecs[16]; - transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m256i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn(v, msg_vecs, 0); - round_fn(v, msg_vecs, 1); - round_fn(v, msg_vecs, 2); - round_fn(v, msg_vecs, 3); - round_fn(v, msg_vecs, 4); - round_fn(v, msg_vecs, 5); - round_fn(v, msg_vecs, 6); - h_vecs[0] = xorv(v[0], v[8]); - h_vecs[1] = xorv(v[1], v[9]); - h_vecs[2] = xorv(v[2], v[10]); - h_vecs[3] = xorv(v[3], v[11]); - h_vecs[4] = xorv(v[4], v[12]); - h_vecs[5] = xorv(v[5], v[13]); - h_vecs[6] = xorv(v[6], v[14]); - h_vecs[7] = xorv(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs(h_vecs); - storeu(h_vecs[0], &out[0 * sizeof(__m256i)]); - storeu(h_vecs[1], &out[1 * sizeof(__m256i)]); - storeu(h_vecs[2], &out[2 * sizeof(__m256i)]); - storeu(h_vecs[3], &out[3 * sizeof(__m256i)]); - storeu(h_vecs[4], &out[4 * sizeof(__m256i)]); - storeu(h_vecs[5], &out[5 * sizeof(__m256i)]); - storeu(h_vecs[6], &out[6 * sizeof(__m256i)]); - storeu(h_vecs[7], &out[7 * sizeof(__m256i)]); -} - -void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= DEGREE) { - blake3_hash8_avx2(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += DEGREE; - } - inputs += DEGREE; - num_inputs -= DEGREE; - out = &out[DEGREE * BLAKE3_OUT_LEN]; - } -#if !defined(BLAKE3_NO_SSE41) - blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, out); -#else - blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, - increment_counter, flags, flags_start, flags_end, - out); -#endif -} diff --git a/src/third_party/blake3/c/blake3_avx512.c b/src/third_party/blake3/c/blake3_avx512.c deleted file mode 100644 index f969bf37..00000000 --- a/src/third_party/blake3/c/blake3_avx512.c +++ /dev/null @@ -1,1220 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define _mm_shuffle_ps2(a, b, c) \ - (_mm_castps_si128( \ - _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) - -INLINE __m128i loadu_128(const uint8_t src[16]) { - return _mm_loadu_si128((const __m128i *)src); -} - -INLINE __m256i loadu_256(const uint8_t src[32]) { - return _mm256_loadu_si256((const __m256i *)src); -} - -INLINE __m512i loadu_512(const uint8_t src[64]) { - return _mm512_loadu_si512((const __m512i *)src); -} - -INLINE void storeu_128(__m128i src, uint8_t dest[16]) { - _mm_storeu_si128((__m128i *)dest, src); -} - -INLINE void storeu_256(__m256i src, uint8_t dest[16]) { - _mm256_storeu_si256((__m256i *)dest, src); -} - -INLINE __m128i add_128(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } - -INLINE __m256i add_256(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } - -INLINE __m512i add_512(__m512i a, __m512i b) { return _mm512_add_epi32(a, b); } - -INLINE __m128i xor_128(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } - -INLINE __m256i xor_256(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } - -INLINE __m512i xor_512(__m512i a, __m512i b) { return _mm512_xor_si512(a, b); } - -INLINE __m128i set1_128(uint32_t x) { return _mm_set1_epi32((int32_t)x); } - -INLINE __m256i set1_256(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } - -INLINE __m512i set1_512(uint32_t x) { return _mm512_set1_epi32((int32_t)x); } - -INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); -} - -INLINE __m128i rot16_128(__m128i x) { return _mm_ror_epi32(x, 16); } - -INLINE __m256i rot16_256(__m256i x) { return _mm256_ror_epi32(x, 16); } - -INLINE __m512i rot16_512(__m512i x) { return _mm512_ror_epi32(x, 16); } - -INLINE __m128i rot12_128(__m128i x) { return _mm_ror_epi32(x, 12); } - -INLINE __m256i rot12_256(__m256i x) { return _mm256_ror_epi32(x, 12); } - -INLINE __m512i rot12_512(__m512i x) { return _mm512_ror_epi32(x, 12); } - -INLINE __m128i rot8_128(__m128i x) { return _mm_ror_epi32(x, 8); } - -INLINE __m256i rot8_256(__m256i x) { return _mm256_ror_epi32(x, 8); } - -INLINE __m512i rot8_512(__m512i x) { return _mm512_ror_epi32(x, 8); } - -INLINE __m128i rot7_128(__m128i x) { return _mm_ror_epi32(x, 7); } - -INLINE __m256i rot7_256(__m256i x) { return _mm256_ror_epi32(x, 7); } - -INLINE __m512i rot7_512(__m512i x) { return _mm512_ror_epi32(x, 7); } - -/* - * ---------------------------------------------------------------------------- - * compress_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = add_128(add_128(*row0, m), *row1); - *row3 = xor_128(*row3, *row0); - *row3 = rot16_128(*row3); - *row2 = add_128(*row2, *row3); - *row1 = xor_128(*row1, *row2); - *row1 = rot12_128(*row1); -} - -INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = add_128(add_128(*row0, m), *row1); - *row3 = xor_128(*row3, *row0); - *row3 = rot8_128(*row3); - *row2 = add_128(*row2, *row3); - *row1 = xor_128(*row1, *row2); - *row1 = rot7_128(*row1); -} - -// Note the optimization here of leaving row1 as the unrotated row, rather than -// row0. All the message loads below are adjusted to compensate for this. See -// discussion at https://github.com/sneves/blake2-avx2/pull/4 -INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); -} - -INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); -} - -INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - rows[0] = loadu_128((uint8_t *)&cv[0]); - rows[1] = loadu_128((uint8_t *)&cv[4]); - rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); - rows[3] = set4(counter_low(counter), counter_high(counter), - (uint32_t)block_len, (uint32_t)flags); - - __m128i m0 = loadu_128(&block[sizeof(__m128i) * 0]); - __m128i m1 = loadu_128(&block[sizeof(__m128i) * 1]); - __m128i m2 = loadu_128(&block[sizeof(__m128i) * 2]); - __m128i m3 = loadu_128(&block[sizeof(__m128i) * 3]); - - __m128i t0, t1, t2, t3, tt; - - // Round 1. The first round permutes the message words from the original - // input order, into the groups that get mixed in parallel. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 - t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 - t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 2. This round and all following rounds apply a fixed permutation - // to the message words from the round before. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 3 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 4 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 5 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 6 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 7 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); -} - -void blake3_compress_xof_avx512(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu_128(xor_128(rows[0], rows[2]), &out[0]); - storeu_128(xor_128(rows[1], rows[3]), &out[16]); - storeu_128(xor_128(rows[2], loadu_128((uint8_t *)&cv[0])), &out[32]); - storeu_128(xor_128(rows[3], loadu_128((uint8_t *)&cv[4])), &out[48]); -} - -void blake3_compress_in_place_avx512(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu_128(xor_128(rows[0], rows[2]), (uint8_t *)&cv[0]); - storeu_128(xor_128(rows[1], rows[3]), (uint8_t *)&cv[4]); -} - -/* - * ---------------------------------------------------------------------------- - * hash4_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn4(__m128i v[16], __m128i m[16], size_t r) { - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[15] = rot16_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot12_128(v[4]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[15] = rot8_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot7_128(v[4]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot16_128(v[15]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[4] = rot12_128(v[4]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot8_128(v[15]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - v[4] = rot7_128(v[4]); -} - -INLINE void transpose_vecs_128(__m128i vecs[4]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is - // 22/33. Note that this doesn't split the vector into two lanes, as the - // AVX2 counterparts do. - __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); - __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); - __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); - __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); - - // Interleave 64-bit lanes. - __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); - __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); - __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); - __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); - - vecs[0] = abcd_0; - vecs[1] = abcd_1; - vecs[2] = abcd_2; - vecs[3] = abcd_3; -} - -INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, - size_t block_offset, __m128i out[16]) { - out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(__m128i)]); - out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(__m128i)]); - out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(__m128i)]); - out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(__m128i)]); - out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(__m128i)]); - out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(__m128i)]); - out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(__m128i)]); - out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(__m128i)]); - out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(__m128i)]); - out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(__m128i)]); - out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(__m128i)]); - out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(__m128i)]); - out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(__m128i)]); - out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(__m128i)]); - out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(__m128i)]); - out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(__m128i)]); - for (size_t i = 0; i < 4; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs_128(&out[0]); - transpose_vecs_128(&out[4]); - transpose_vecs_128(&out[8]); - transpose_vecs_128(&out[12]); -} - -INLINE void load_counters4(uint64_t counter, bool increment_counter, - __m128i *out_lo, __m128i *out_hi) { - uint64_t mask = (increment_counter ? ~0 : 0); - __m256i mask_vec = _mm256_set1_epi64x(mask); - __m256i deltas = _mm256_setr_epi64x(0, 1, 2, 3); - deltas = _mm256_and_si256(mask_vec, deltas); - __m256i counters = - _mm256_add_epi64(_mm256_set1_epi64x((int64_t)counter), deltas); - *out_lo = _mm256_cvtepi64_epi32(counters); - *out_hi = _mm256_cvtepi64_epi32(_mm256_srli_epi64(counters, 32)); -} - -static -void blake3_hash4_avx512(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m128i h_vecs[8] = { - set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), - set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), - }; - __m128i counter_low_vec, counter_high_vec; - load_counters4(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m128i block_len_vec = set1_128(BLAKE3_BLOCK_LEN); - __m128i block_flags_vec = set1_128(block_flags); - __m128i msg_vecs[16]; - transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m128i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_128(IV[0]), set1_128(IV[1]), set1_128(IV[2]), set1_128(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn4(v, msg_vecs, 0); - round_fn4(v, msg_vecs, 1); - round_fn4(v, msg_vecs, 2); - round_fn4(v, msg_vecs, 3); - round_fn4(v, msg_vecs, 4); - round_fn4(v, msg_vecs, 5); - round_fn4(v, msg_vecs, 6); - h_vecs[0] = xor_128(v[0], v[8]); - h_vecs[1] = xor_128(v[1], v[9]); - h_vecs[2] = xor_128(v[2], v[10]); - h_vecs[3] = xor_128(v[3], v[11]); - h_vecs[4] = xor_128(v[4], v[12]); - h_vecs[5] = xor_128(v[5], v[13]); - h_vecs[6] = xor_128(v[6], v[14]); - h_vecs[7] = xor_128(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs_128(&h_vecs[0]); - transpose_vecs_128(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu_128(h_vecs[0], &out[0 * sizeof(__m128i)]); - storeu_128(h_vecs[4], &out[1 * sizeof(__m128i)]); - storeu_128(h_vecs[1], &out[2 * sizeof(__m128i)]); - storeu_128(h_vecs[5], &out[3 * sizeof(__m128i)]); - storeu_128(h_vecs[2], &out[4 * sizeof(__m128i)]); - storeu_128(h_vecs[6], &out[5 * sizeof(__m128i)]); - storeu_128(h_vecs[3], &out[6 * sizeof(__m128i)]); - storeu_128(h_vecs[7], &out[7 * sizeof(__m128i)]); -} - -/* - * ---------------------------------------------------------------------------- - * hash8_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn8(__m256i v[16], __m256i m[16], size_t r) { - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_256(v[0], v[4]); - v[1] = add_256(v[1], v[5]); - v[2] = add_256(v[2], v[6]); - v[3] = add_256(v[3], v[7]); - v[12] = xor_256(v[12], v[0]); - v[13] = xor_256(v[13], v[1]); - v[14] = xor_256(v[14], v[2]); - v[15] = xor_256(v[15], v[3]); - v[12] = rot16_256(v[12]); - v[13] = rot16_256(v[13]); - v[14] = rot16_256(v[14]); - v[15] = rot16_256(v[15]); - v[8] = add_256(v[8], v[12]); - v[9] = add_256(v[9], v[13]); - v[10] = add_256(v[10], v[14]); - v[11] = add_256(v[11], v[15]); - v[4] = xor_256(v[4], v[8]); - v[5] = xor_256(v[5], v[9]); - v[6] = xor_256(v[6], v[10]); - v[7] = xor_256(v[7], v[11]); - v[4] = rot12_256(v[4]); - v[5] = rot12_256(v[5]); - v[6] = rot12_256(v[6]); - v[7] = rot12_256(v[7]); - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_256(v[0], v[4]); - v[1] = add_256(v[1], v[5]); - v[2] = add_256(v[2], v[6]); - v[3] = add_256(v[3], v[7]); - v[12] = xor_256(v[12], v[0]); - v[13] = xor_256(v[13], v[1]); - v[14] = xor_256(v[14], v[2]); - v[15] = xor_256(v[15], v[3]); - v[12] = rot8_256(v[12]); - v[13] = rot8_256(v[13]); - v[14] = rot8_256(v[14]); - v[15] = rot8_256(v[15]); - v[8] = add_256(v[8], v[12]); - v[9] = add_256(v[9], v[13]); - v[10] = add_256(v[10], v[14]); - v[11] = add_256(v[11], v[15]); - v[4] = xor_256(v[4], v[8]); - v[5] = xor_256(v[5], v[9]); - v[6] = xor_256(v[6], v[10]); - v[7] = xor_256(v[7], v[11]); - v[4] = rot7_256(v[4]); - v[5] = rot7_256(v[5]); - v[6] = rot7_256(v[6]); - v[7] = rot7_256(v[7]); - - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_256(v[0], v[5]); - v[1] = add_256(v[1], v[6]); - v[2] = add_256(v[2], v[7]); - v[3] = add_256(v[3], v[4]); - v[15] = xor_256(v[15], v[0]); - v[12] = xor_256(v[12], v[1]); - v[13] = xor_256(v[13], v[2]); - v[14] = xor_256(v[14], v[3]); - v[15] = rot16_256(v[15]); - v[12] = rot16_256(v[12]); - v[13] = rot16_256(v[13]); - v[14] = rot16_256(v[14]); - v[10] = add_256(v[10], v[15]); - v[11] = add_256(v[11], v[12]); - v[8] = add_256(v[8], v[13]); - v[9] = add_256(v[9], v[14]); - v[5] = xor_256(v[5], v[10]); - v[6] = xor_256(v[6], v[11]); - v[7] = xor_256(v[7], v[8]); - v[4] = xor_256(v[4], v[9]); - v[5] = rot12_256(v[5]); - v[6] = rot12_256(v[6]); - v[7] = rot12_256(v[7]); - v[4] = rot12_256(v[4]); - v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_256(v[0], v[5]); - v[1] = add_256(v[1], v[6]); - v[2] = add_256(v[2], v[7]); - v[3] = add_256(v[3], v[4]); - v[15] = xor_256(v[15], v[0]); - v[12] = xor_256(v[12], v[1]); - v[13] = xor_256(v[13], v[2]); - v[14] = xor_256(v[14], v[3]); - v[15] = rot8_256(v[15]); - v[12] = rot8_256(v[12]); - v[13] = rot8_256(v[13]); - v[14] = rot8_256(v[14]); - v[10] = add_256(v[10], v[15]); - v[11] = add_256(v[11], v[12]); - v[8] = add_256(v[8], v[13]); - v[9] = add_256(v[9], v[14]); - v[5] = xor_256(v[5], v[10]); - v[6] = xor_256(v[6], v[11]); - v[7] = xor_256(v[7], v[8]); - v[4] = xor_256(v[4], v[9]); - v[5] = rot7_256(v[5]); - v[6] = rot7_256(v[6]); - v[7] = rot7_256(v[7]); - v[4] = rot7_256(v[4]); -} - -INLINE void transpose_vecs_256(__m256i vecs[8]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high - // is 22/33/66/77. - __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); - __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); - __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); - __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); - __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); - __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); - __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); - __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); - - // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is - // 11/33. - __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); - __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); - __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); - __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); - __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); - __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); - __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); - __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); - - // Interleave 128-bit lanes. - vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); - vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); - vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); - vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); - vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); - vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); - vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); - vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); -} - -INLINE void transpose_msg_vecs8(const uint8_t *const *inputs, - size_t block_offset, __m256i out[16]) { - out[0] = loadu_256(&inputs[0][block_offset + 0 * sizeof(__m256i)]); - out[1] = loadu_256(&inputs[1][block_offset + 0 * sizeof(__m256i)]); - out[2] = loadu_256(&inputs[2][block_offset + 0 * sizeof(__m256i)]); - out[3] = loadu_256(&inputs[3][block_offset + 0 * sizeof(__m256i)]); - out[4] = loadu_256(&inputs[4][block_offset + 0 * sizeof(__m256i)]); - out[5] = loadu_256(&inputs[5][block_offset + 0 * sizeof(__m256i)]); - out[6] = loadu_256(&inputs[6][block_offset + 0 * sizeof(__m256i)]); - out[7] = loadu_256(&inputs[7][block_offset + 0 * sizeof(__m256i)]); - out[8] = loadu_256(&inputs[0][block_offset + 1 * sizeof(__m256i)]); - out[9] = loadu_256(&inputs[1][block_offset + 1 * sizeof(__m256i)]); - out[10] = loadu_256(&inputs[2][block_offset + 1 * sizeof(__m256i)]); - out[11] = loadu_256(&inputs[3][block_offset + 1 * sizeof(__m256i)]); - out[12] = loadu_256(&inputs[4][block_offset + 1 * sizeof(__m256i)]); - out[13] = loadu_256(&inputs[5][block_offset + 1 * sizeof(__m256i)]); - out[14] = loadu_256(&inputs[6][block_offset + 1 * sizeof(__m256i)]); - out[15] = loadu_256(&inputs[7][block_offset + 1 * sizeof(__m256i)]); - for (size_t i = 0; i < 8; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs_256(&out[0]); - transpose_vecs_256(&out[8]); -} - -INLINE void load_counters8(uint64_t counter, bool increment_counter, - __m256i *out_lo, __m256i *out_hi) { - uint64_t mask = (increment_counter ? ~0 : 0); - __m512i mask_vec = _mm512_set1_epi64(mask); - __m512i deltas = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); - deltas = _mm512_and_si512(mask_vec, deltas); - __m512i counters = - _mm512_add_epi64(_mm512_set1_epi64((int64_t)counter), deltas); - *out_lo = _mm512_cvtepi64_epi32(counters); - *out_hi = _mm512_cvtepi64_epi32(_mm512_srli_epi64(counters, 32)); -} - -static -void blake3_hash8_avx512(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m256i h_vecs[8] = { - set1_256(key[0]), set1_256(key[1]), set1_256(key[2]), set1_256(key[3]), - set1_256(key[4]), set1_256(key[5]), set1_256(key[6]), set1_256(key[7]), - }; - __m256i counter_low_vec, counter_high_vec; - load_counters8(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m256i block_len_vec = set1_256(BLAKE3_BLOCK_LEN); - __m256i block_flags_vec = set1_256(block_flags); - __m256i msg_vecs[16]; - transpose_msg_vecs8(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m256i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_256(IV[0]), set1_256(IV[1]), set1_256(IV[2]), set1_256(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn8(v, msg_vecs, 0); - round_fn8(v, msg_vecs, 1); - round_fn8(v, msg_vecs, 2); - round_fn8(v, msg_vecs, 3); - round_fn8(v, msg_vecs, 4); - round_fn8(v, msg_vecs, 5); - round_fn8(v, msg_vecs, 6); - h_vecs[0] = xor_256(v[0], v[8]); - h_vecs[1] = xor_256(v[1], v[9]); - h_vecs[2] = xor_256(v[2], v[10]); - h_vecs[3] = xor_256(v[3], v[11]); - h_vecs[4] = xor_256(v[4], v[12]); - h_vecs[5] = xor_256(v[5], v[13]); - h_vecs[6] = xor_256(v[6], v[14]); - h_vecs[7] = xor_256(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs_256(h_vecs); - storeu_256(h_vecs[0], &out[0 * sizeof(__m256i)]); - storeu_256(h_vecs[1], &out[1 * sizeof(__m256i)]); - storeu_256(h_vecs[2], &out[2 * sizeof(__m256i)]); - storeu_256(h_vecs[3], &out[3 * sizeof(__m256i)]); - storeu_256(h_vecs[4], &out[4 * sizeof(__m256i)]); - storeu_256(h_vecs[5], &out[5 * sizeof(__m256i)]); - storeu_256(h_vecs[6], &out[6 * sizeof(__m256i)]); - storeu_256(h_vecs[7], &out[7 * sizeof(__m256i)]); -} - -/* - * ---------------------------------------------------------------------------- - * hash16_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn16(__m512i v[16], __m512i m[16], size_t r) { - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_512(v[0], v[4]); - v[1] = add_512(v[1], v[5]); - v[2] = add_512(v[2], v[6]); - v[3] = add_512(v[3], v[7]); - v[12] = xor_512(v[12], v[0]); - v[13] = xor_512(v[13], v[1]); - v[14] = xor_512(v[14], v[2]); - v[15] = xor_512(v[15], v[3]); - v[12] = rot16_512(v[12]); - v[13] = rot16_512(v[13]); - v[14] = rot16_512(v[14]); - v[15] = rot16_512(v[15]); - v[8] = add_512(v[8], v[12]); - v[9] = add_512(v[9], v[13]); - v[10] = add_512(v[10], v[14]); - v[11] = add_512(v[11], v[15]); - v[4] = xor_512(v[4], v[8]); - v[5] = xor_512(v[5], v[9]); - v[6] = xor_512(v[6], v[10]); - v[7] = xor_512(v[7], v[11]); - v[4] = rot12_512(v[4]); - v[5] = rot12_512(v[5]); - v[6] = rot12_512(v[6]); - v[7] = rot12_512(v[7]); - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_512(v[0], v[4]); - v[1] = add_512(v[1], v[5]); - v[2] = add_512(v[2], v[6]); - v[3] = add_512(v[3], v[7]); - v[12] = xor_512(v[12], v[0]); - v[13] = xor_512(v[13], v[1]); - v[14] = xor_512(v[14], v[2]); - v[15] = xor_512(v[15], v[3]); - v[12] = rot8_512(v[12]); - v[13] = rot8_512(v[13]); - v[14] = rot8_512(v[14]); - v[15] = rot8_512(v[15]); - v[8] = add_512(v[8], v[12]); - v[9] = add_512(v[9], v[13]); - v[10] = add_512(v[10], v[14]); - v[11] = add_512(v[11], v[15]); - v[4] = xor_512(v[4], v[8]); - v[5] = xor_512(v[5], v[9]); - v[6] = xor_512(v[6], v[10]); - v[7] = xor_512(v[7], v[11]); - v[4] = rot7_512(v[4]); - v[5] = rot7_512(v[5]); - v[6] = rot7_512(v[6]); - v[7] = rot7_512(v[7]); - - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_512(v[0], v[5]); - v[1] = add_512(v[1], v[6]); - v[2] = add_512(v[2], v[7]); - v[3] = add_512(v[3], v[4]); - v[15] = xor_512(v[15], v[0]); - v[12] = xor_512(v[12], v[1]); - v[13] = xor_512(v[13], v[2]); - v[14] = xor_512(v[14], v[3]); - v[15] = rot16_512(v[15]); - v[12] = rot16_512(v[12]); - v[13] = rot16_512(v[13]); - v[14] = rot16_512(v[14]); - v[10] = add_512(v[10], v[15]); - v[11] = add_512(v[11], v[12]); - v[8] = add_512(v[8], v[13]); - v[9] = add_512(v[9], v[14]); - v[5] = xor_512(v[5], v[10]); - v[6] = xor_512(v[6], v[11]); - v[7] = xor_512(v[7], v[8]); - v[4] = xor_512(v[4], v[9]); - v[5] = rot12_512(v[5]); - v[6] = rot12_512(v[6]); - v[7] = rot12_512(v[7]); - v[4] = rot12_512(v[4]); - v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_512(v[0], v[5]); - v[1] = add_512(v[1], v[6]); - v[2] = add_512(v[2], v[7]); - v[3] = add_512(v[3], v[4]); - v[15] = xor_512(v[15], v[0]); - v[12] = xor_512(v[12], v[1]); - v[13] = xor_512(v[13], v[2]); - v[14] = xor_512(v[14], v[3]); - v[15] = rot8_512(v[15]); - v[12] = rot8_512(v[12]); - v[13] = rot8_512(v[13]); - v[14] = rot8_512(v[14]); - v[10] = add_512(v[10], v[15]); - v[11] = add_512(v[11], v[12]); - v[8] = add_512(v[8], v[13]); - v[9] = add_512(v[9], v[14]); - v[5] = xor_512(v[5], v[10]); - v[6] = xor_512(v[6], v[11]); - v[7] = xor_512(v[7], v[8]); - v[4] = xor_512(v[4], v[9]); - v[5] = rot7_512(v[5]); - v[6] = rot7_512(v[6]); - v[7] = rot7_512(v[7]); - v[4] = rot7_512(v[4]); -} - -// 0b10001000, or lanes a0/a2/b0/b2 in little-endian order -#define LO_IMM8 0x88 - -INLINE __m512i unpack_lo_128(__m512i a, __m512i b) { - return _mm512_shuffle_i32x4(a, b, LO_IMM8); -} - -// 0b11011101, or lanes a1/a3/b1/b3 in little-endian order -#define HI_IMM8 0xdd - -INLINE __m512i unpack_hi_128(__m512i a, __m512i b) { - return _mm512_shuffle_i32x4(a, b, HI_IMM8); -} - -INLINE void transpose_vecs_512(__m512i vecs[16]) { - // Interleave 32-bit lanes. The _0 unpack is lanes - // 0/0/1/1/4/4/5/5/8/8/9/9/12/12/13/13, and the _2 unpack is lanes - // 2/2/3/3/6/6/7/7/10/10/11/11/14/14/15/15. - __m512i ab_0 = _mm512_unpacklo_epi32(vecs[0], vecs[1]); - __m512i ab_2 = _mm512_unpackhi_epi32(vecs[0], vecs[1]); - __m512i cd_0 = _mm512_unpacklo_epi32(vecs[2], vecs[3]); - __m512i cd_2 = _mm512_unpackhi_epi32(vecs[2], vecs[3]); - __m512i ef_0 = _mm512_unpacklo_epi32(vecs[4], vecs[5]); - __m512i ef_2 = _mm512_unpackhi_epi32(vecs[4], vecs[5]); - __m512i gh_0 = _mm512_unpacklo_epi32(vecs[6], vecs[7]); - __m512i gh_2 = _mm512_unpackhi_epi32(vecs[6], vecs[7]); - __m512i ij_0 = _mm512_unpacklo_epi32(vecs[8], vecs[9]); - __m512i ij_2 = _mm512_unpackhi_epi32(vecs[8], vecs[9]); - __m512i kl_0 = _mm512_unpacklo_epi32(vecs[10], vecs[11]); - __m512i kl_2 = _mm512_unpackhi_epi32(vecs[10], vecs[11]); - __m512i mn_0 = _mm512_unpacklo_epi32(vecs[12], vecs[13]); - __m512i mn_2 = _mm512_unpackhi_epi32(vecs[12], vecs[13]); - __m512i op_0 = _mm512_unpacklo_epi32(vecs[14], vecs[15]); - __m512i op_2 = _mm512_unpackhi_epi32(vecs[14], vecs[15]); - - // Interleave 64-bit lanes. The _0 unpack is lanes - // 0/0/0/0/4/4/4/4/8/8/8/8/12/12/12/12, the _1 unpack is lanes - // 1/1/1/1/5/5/5/5/9/9/9/9/13/13/13/13, the _2 unpack is lanes - // 2/2/2/2/6/6/6/6/10/10/10/10/14/14/14/14, and the _3 unpack is lanes - // 3/3/3/3/7/7/7/7/11/11/11/11/15/15/15/15. - __m512i abcd_0 = _mm512_unpacklo_epi64(ab_0, cd_0); - __m512i abcd_1 = _mm512_unpackhi_epi64(ab_0, cd_0); - __m512i abcd_2 = _mm512_unpacklo_epi64(ab_2, cd_2); - __m512i abcd_3 = _mm512_unpackhi_epi64(ab_2, cd_2); - __m512i efgh_0 = _mm512_unpacklo_epi64(ef_0, gh_0); - __m512i efgh_1 = _mm512_unpackhi_epi64(ef_0, gh_0); - __m512i efgh_2 = _mm512_unpacklo_epi64(ef_2, gh_2); - __m512i efgh_3 = _mm512_unpackhi_epi64(ef_2, gh_2); - __m512i ijkl_0 = _mm512_unpacklo_epi64(ij_0, kl_0); - __m512i ijkl_1 = _mm512_unpackhi_epi64(ij_0, kl_0); - __m512i ijkl_2 = _mm512_unpacklo_epi64(ij_2, kl_2); - __m512i ijkl_3 = _mm512_unpackhi_epi64(ij_2, kl_2); - __m512i mnop_0 = _mm512_unpacklo_epi64(mn_0, op_0); - __m512i mnop_1 = _mm512_unpackhi_epi64(mn_0, op_0); - __m512i mnop_2 = _mm512_unpacklo_epi64(mn_2, op_2); - __m512i mnop_3 = _mm512_unpackhi_epi64(mn_2, op_2); - - // Interleave 128-bit lanes. The _0 unpack is - // 0/0/0/0/8/8/8/8/0/0/0/0/8/8/8/8, the _1 unpack is - // 1/1/1/1/9/9/9/9/1/1/1/1/9/9/9/9, and so on. - __m512i abcdefgh_0 = unpack_lo_128(abcd_0, efgh_0); - __m512i abcdefgh_1 = unpack_lo_128(abcd_1, efgh_1); - __m512i abcdefgh_2 = unpack_lo_128(abcd_2, efgh_2); - __m512i abcdefgh_3 = unpack_lo_128(abcd_3, efgh_3); - __m512i abcdefgh_4 = unpack_hi_128(abcd_0, efgh_0); - __m512i abcdefgh_5 = unpack_hi_128(abcd_1, efgh_1); - __m512i abcdefgh_6 = unpack_hi_128(abcd_2, efgh_2); - __m512i abcdefgh_7 = unpack_hi_128(abcd_3, efgh_3); - __m512i ijklmnop_0 = unpack_lo_128(ijkl_0, mnop_0); - __m512i ijklmnop_1 = unpack_lo_128(ijkl_1, mnop_1); - __m512i ijklmnop_2 = unpack_lo_128(ijkl_2, mnop_2); - __m512i ijklmnop_3 = unpack_lo_128(ijkl_3, mnop_3); - __m512i ijklmnop_4 = unpack_hi_128(ijkl_0, mnop_0); - __m512i ijklmnop_5 = unpack_hi_128(ijkl_1, mnop_1); - __m512i ijklmnop_6 = unpack_hi_128(ijkl_2, mnop_2); - __m512i ijklmnop_7 = unpack_hi_128(ijkl_3, mnop_3); - - // Interleave 128-bit lanes again for the final outputs. - vecs[0] = unpack_lo_128(abcdefgh_0, ijklmnop_0); - vecs[1] = unpack_lo_128(abcdefgh_1, ijklmnop_1); - vecs[2] = unpack_lo_128(abcdefgh_2, ijklmnop_2); - vecs[3] = unpack_lo_128(abcdefgh_3, ijklmnop_3); - vecs[4] = unpack_lo_128(abcdefgh_4, ijklmnop_4); - vecs[5] = unpack_lo_128(abcdefgh_5, ijklmnop_5); - vecs[6] = unpack_lo_128(abcdefgh_6, ijklmnop_6); - vecs[7] = unpack_lo_128(abcdefgh_7, ijklmnop_7); - vecs[8] = unpack_hi_128(abcdefgh_0, ijklmnop_0); - vecs[9] = unpack_hi_128(abcdefgh_1, ijklmnop_1); - vecs[10] = unpack_hi_128(abcdefgh_2, ijklmnop_2); - vecs[11] = unpack_hi_128(abcdefgh_3, ijklmnop_3); - vecs[12] = unpack_hi_128(abcdefgh_4, ijklmnop_4); - vecs[13] = unpack_hi_128(abcdefgh_5, ijklmnop_5); - vecs[14] = unpack_hi_128(abcdefgh_6, ijklmnop_6); - vecs[15] = unpack_hi_128(abcdefgh_7, ijklmnop_7); -} - -INLINE void transpose_msg_vecs16(const uint8_t *const *inputs, - size_t block_offset, __m512i out[16]) { - out[0] = loadu_512(&inputs[0][block_offset]); - out[1] = loadu_512(&inputs[1][block_offset]); - out[2] = loadu_512(&inputs[2][block_offset]); - out[3] = loadu_512(&inputs[3][block_offset]); - out[4] = loadu_512(&inputs[4][block_offset]); - out[5] = loadu_512(&inputs[5][block_offset]); - out[6] = loadu_512(&inputs[6][block_offset]); - out[7] = loadu_512(&inputs[7][block_offset]); - out[8] = loadu_512(&inputs[8][block_offset]); - out[9] = loadu_512(&inputs[9][block_offset]); - out[10] = loadu_512(&inputs[10][block_offset]); - out[11] = loadu_512(&inputs[11][block_offset]); - out[12] = loadu_512(&inputs[12][block_offset]); - out[13] = loadu_512(&inputs[13][block_offset]); - out[14] = loadu_512(&inputs[14][block_offset]); - out[15] = loadu_512(&inputs[15][block_offset]); - for (size_t i = 0; i < 16; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs_512(out); -} - -INLINE void load_counters16(uint64_t counter, bool increment_counter, - __m512i *out_lo, __m512i *out_hi) { - const __m512i mask = _mm512_set1_epi32(-(int32_t)increment_counter); - const __m512i deltas = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); - const __m512i masked_deltas = _mm512_and_si512(deltas, mask); - const __m512i low_words = _mm512_add_epi32( - _mm512_set1_epi32((int32_t)counter), - masked_deltas); - // The carry bit is 1 if the high bit of the word was 1 before addition and is - // 0 after. - // NOTE: It would be a bit more natural to use _mm512_cmp_epu32_mask to - // compute the carry bits here, and originally we did, but that intrinsic is - // broken under GCC 5.4. See https://github.com/BLAKE3-team/BLAKE3/issues/271. - const __m512i carries = _mm512_srli_epi32( - _mm512_andnot_si512( - low_words, // 0 after (gets inverted by andnot) - _mm512_set1_epi32((int32_t)counter)), // and 1 before - 31); - const __m512i high_words = _mm512_add_epi32( - _mm512_set1_epi32((int32_t)(counter >> 32)), - carries); - *out_lo = low_words; - *out_hi = high_words; -} - -static -void blake3_hash16_avx512(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, - uint8_t *out) { - __m512i h_vecs[8] = { - set1_512(key[0]), set1_512(key[1]), set1_512(key[2]), set1_512(key[3]), - set1_512(key[4]), set1_512(key[5]), set1_512(key[6]), set1_512(key[7]), - }; - __m512i counter_low_vec, counter_high_vec; - load_counters16(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m512i block_len_vec = set1_512(BLAKE3_BLOCK_LEN); - __m512i block_flags_vec = set1_512(block_flags); - __m512i msg_vecs[16]; - transpose_msg_vecs16(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m512i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_512(IV[0]), set1_512(IV[1]), set1_512(IV[2]), set1_512(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn16(v, msg_vecs, 0); - round_fn16(v, msg_vecs, 1); - round_fn16(v, msg_vecs, 2); - round_fn16(v, msg_vecs, 3); - round_fn16(v, msg_vecs, 4); - round_fn16(v, msg_vecs, 5); - round_fn16(v, msg_vecs, 6); - h_vecs[0] = xor_512(v[0], v[8]); - h_vecs[1] = xor_512(v[1], v[9]); - h_vecs[2] = xor_512(v[2], v[10]); - h_vecs[3] = xor_512(v[3], v[11]); - h_vecs[4] = xor_512(v[4], v[12]); - h_vecs[5] = xor_512(v[5], v[13]); - h_vecs[6] = xor_512(v[6], v[14]); - h_vecs[7] = xor_512(v[7], v[15]); - - block_flags = flags; - } - - // transpose_vecs_512 operates on a 16x16 matrix of words, but we only have 8 - // state vectors. Pad the matrix with zeros. After transposition, store the - // lower half of each vector. - __m512i padded[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_512(0), set1_512(0), set1_512(0), set1_512(0), - set1_512(0), set1_512(0), set1_512(0), set1_512(0), - }; - transpose_vecs_512(padded); - _mm256_mask_storeu_epi32(&out[0 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[0])); - _mm256_mask_storeu_epi32(&out[1 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[1])); - _mm256_mask_storeu_epi32(&out[2 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[2])); - _mm256_mask_storeu_epi32(&out[3 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[3])); - _mm256_mask_storeu_epi32(&out[4 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[4])); - _mm256_mask_storeu_epi32(&out[5 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[5])); - _mm256_mask_storeu_epi32(&out[6 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[6])); - _mm256_mask_storeu_epi32(&out[7 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[7])); - _mm256_mask_storeu_epi32(&out[8 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[8])); - _mm256_mask_storeu_epi32(&out[9 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[9])); - _mm256_mask_storeu_epi32(&out[10 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[10])); - _mm256_mask_storeu_epi32(&out[11 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[11])); - _mm256_mask_storeu_epi32(&out[12 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[12])); - _mm256_mask_storeu_epi32(&out[13 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[13])); - _mm256_mask_storeu_epi32(&out[14 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[14])); - _mm256_mask_storeu_epi32(&out[15 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[15])); -} - -/* - * ---------------------------------------------------------------------------- - * hash_many_avx512 - * ---------------------------------------------------------------------------- - */ - -INLINE void hash_one_avx512(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_avx512(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= 16) { - blake3_hash16_avx512(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 16; - } - inputs += 16; - num_inputs -= 16; - out = &out[16 * BLAKE3_OUT_LEN]; - } - while (num_inputs >= 8) { - blake3_hash8_avx512(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 8; - } - inputs += 8; - num_inputs -= 8; - out = &out[8 * BLAKE3_OUT_LEN]; - } - while (num_inputs >= 4) { - blake3_hash4_avx512(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 4; - } - inputs += 4; - num_inputs -= 4; - out = &out[4 * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_avx512(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_impl.h b/src/third_party/blake3/c/blake3_impl.h deleted file mode 100644 index beab5cf5..00000000 --- a/src/third_party/blake3/c/blake3_impl.h +++ /dev/null @@ -1,285 +0,0 @@ -#ifndef BLAKE3_IMPL_H -#define BLAKE3_IMPL_H - -#include -#include -#include -#include -#include - -#include "blake3.h" - -// internal flags -enum blake3_flags { - CHUNK_START = 1 << 0, - CHUNK_END = 1 << 1, - PARENT = 1 << 2, - ROOT = 1 << 3, - KEYED_HASH = 1 << 4, - DERIVE_KEY_CONTEXT = 1 << 5, - DERIVE_KEY_MATERIAL = 1 << 6, -}; - -// This C implementation tries to support recent versions of GCC, Clang, and -// MSVC. -#if defined(_MSC_VER) -#define INLINE static __forceinline -#else -#define INLINE static inline __attribute__((always_inline)) -#endif - -#if defined(__x86_64__) || defined(_M_X64) -#define IS_X86 -#define IS_X86_64 -#endif - -#if defined(__i386__) || defined(_M_IX86) -#define IS_X86 -#define IS_X86_32 -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#define IS_AARCH64 -#endif - -#if defined(IS_X86) -#if defined(_MSC_VER) -#include -#endif -#endif - -#if !defined(BLAKE3_USE_NEON) - // If BLAKE3_USE_NEON not manually set, autodetect based on AArch64ness - #if defined(IS_AARCH64) - #if defined(__ARM_BIG_ENDIAN) - #define BLAKE3_USE_NEON 0 - #else - #define BLAKE3_USE_NEON 1 - #endif - #else - #define BLAKE3_USE_NEON 0 - #endif -#endif - -#if defined(IS_X86) -#define MAX_SIMD_DEGREE 16 -#elif BLAKE3_USE_NEON == 1 -#define MAX_SIMD_DEGREE 4 -#else -#define MAX_SIMD_DEGREE 1 -#endif - -// There are some places where we want a static size that's equal to the -// MAX_SIMD_DEGREE, but also at least 2. -#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2) - -static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, - 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL, - 0x1F83D9ABUL, 0x5BE0CD19UL}; - -static const uint8_t MSG_SCHEDULE[7][16] = { - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, - {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, - {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, - {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, - {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, - {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}, -}; - -/* Find index of the highest set bit */ -/* x is assumed to be nonzero. */ -static unsigned int highest_one(uint64_t x) { -#if defined(__GNUC__) || defined(__clang__) - return 63 ^ (unsigned int)__builtin_clzll(x); -#elif defined(_MSC_VER) && defined(IS_X86_64) - unsigned long index; - _BitScanReverse64(&index, x); - return index; -#elif defined(_MSC_VER) && defined(IS_X86_32) - if(x >> 32) { - unsigned long index; - _BitScanReverse(&index, (unsigned long)(x >> 32)); - return 32 + index; - } else { - unsigned long index; - _BitScanReverse(&index, (unsigned long)x); - return index; - } -#else - unsigned int c = 0; - if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } - if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } - if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } - if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } - if(x & 0x000000000000000cULL) { x >>= 2; c += 2; } - if(x & 0x0000000000000002ULL) { c += 1; } - return c; -#endif -} - -// Count the number of 1 bits. -INLINE unsigned int popcnt(uint64_t x) { -#if defined(__GNUC__) || defined(__clang__) - return (unsigned int)__builtin_popcountll(x); -#else - unsigned int count = 0; - while (x != 0) { - count += 1; - x &= x - 1; - } - return count; -#endif -} - -// Largest power of two less than or equal to x. As a special case, returns 1 -// when x is 0. -INLINE uint64_t round_down_to_power_of_2(uint64_t x) { - return 1ULL << highest_one(x | 1); -} - -INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; } - -INLINE uint32_t counter_high(uint64_t counter) { - return (uint32_t)(counter >> 32); -} - -INLINE uint32_t load32(const void *src) { - const uint8_t *p = (const uint8_t *)src; - return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | - ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); -} - -INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], - uint32_t key_words[8]) { - key_words[0] = load32(&key[0 * 4]); - key_words[1] = load32(&key[1 * 4]); - key_words[2] = load32(&key[2 * 4]); - key_words[3] = load32(&key[3 * 4]); - key_words[4] = load32(&key[4 * 4]); - key_words[5] = load32(&key[5 * 4]); - key_words[6] = load32(&key[6 * 4]); - key_words[7] = load32(&key[7 * 4]); -} - -INLINE void store32(void *dst, uint32_t w) { - uint8_t *p = (uint8_t *)dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); -} - -INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) { - store32(&bytes_out[0 * 4], cv_words[0]); - store32(&bytes_out[1 * 4], cv_words[1]); - store32(&bytes_out[2 * 4], cv_words[2]); - store32(&bytes_out[3 * 4], cv_words[3]); - store32(&bytes_out[4 * 4], cv_words[4]); - store32(&bytes_out[5 * 4], cv_words[5]); - store32(&bytes_out[6 * 4], cv_words[6]); - store32(&bytes_out[7 * 4], cv_words[7]); -} - -void blake3_compress_in_place(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -void blake3_compress_xof(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags, - uint8_t out[64]); - -void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out); - -size_t blake3_simd_degree(void); - - -// Declarations for implementation-specific functions. -void blake3_compress_in_place_portable(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -void blake3_compress_xof_portable(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); - -void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); - -#if defined(IS_X86) -#if !defined(BLAKE3_NO_SSE2) -void blake3_compress_in_place_sse2(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); -void blake3_compress_xof_sse2(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); -void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#if !defined(BLAKE3_NO_SSE41) -void blake3_compress_in_place_sse41(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); -void blake3_compress_xof_sse41(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); -void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#if !defined(BLAKE3_NO_AVX2) -void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#if !defined(BLAKE3_NO_AVX512) -void blake3_compress_in_place_avx512(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -void blake3_compress_xof_avx512(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]); - -void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif -#endif - -#if BLAKE3_USE_NEON == 1 -void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out); -#endif - - -#endif /* BLAKE3_IMPL_H */ diff --git a/src/third_party/blake3/c/blake3_neon.c b/src/third_party/blake3/c/blake3_neon.c deleted file mode 100644 index 8a818fc7..00000000 --- a/src/third_party/blake3/c/blake3_neon.c +++ /dev/null @@ -1,368 +0,0 @@ -#include "blake3_impl.h" - -#include - -#ifdef __ARM_BIG_ENDIAN -#error "This implementation only supports little-endian ARM." -// It might be that all we need for big-endian support here is to get the loads -// and stores right, but step zero would be finding a way to test it in CI. -#endif - -INLINE uint32x4_t loadu_128(const uint8_t src[16]) { - // vld1q_u32 has alignment requirements. Don't use it. - uint32x4_t x; - memcpy(&x, src, 16); - return x; -} - -INLINE void storeu_128(uint32x4_t src, uint8_t dest[16]) { - // vst1q_u32 has alignment requirements. Don't use it. - memcpy(dest, &src, 16); -} - -INLINE uint32x4_t add_128(uint32x4_t a, uint32x4_t b) { - return vaddq_u32(a, b); -} - -INLINE uint32x4_t xor_128(uint32x4_t a, uint32x4_t b) { - return veorq_u32(a, b); -} - -INLINE uint32x4_t set1_128(uint32_t x) { return vld1q_dup_u32(&x); } - -INLINE uint32x4_t set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - uint32_t array[4] = {a, b, c, d}; - return vld1q_u32(array); -} - -INLINE uint32x4_t rot16_128(uint32x4_t x) { - // The straightfoward implementation would be two shifts and an or, but that's - // slower on microarchitectures we've tested. See - // https://github.com/BLAKE3-team/BLAKE3/pull/319. - // return vorrq_u32(vshrq_n_u32(x, 16), vshlq_n_u32(x, 32 - 16)); - return vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x))); -} - -INLINE uint32x4_t rot12_128(uint32x4_t x) { - // See comment in rot16_128. - // return vorrq_u32(vshrq_n_u32(x, 12), vshlq_n_u32(x, 32 - 12)); - return vsriq_n_u32(vshlq_n_u32(x, 32-12), x, 12); -} - -INLINE uint32x4_t rot8_128(uint32x4_t x) { - // See comment in rot16_128. - // return vorrq_u32(vshrq_n_u32(x, 8), vshlq_n_u32(x, 32 - 8)); -#if defined(__clang__) - return vreinterpretq_u32_u8(__builtin_shufflevector(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12)); -#elif __GNUC__ * 10000 + __GNUC_MINOR__ * 100 >=40700 - static const uint8x16_t r8 = {1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12}; - return vreinterpretq_u32_u8(__builtin_shuffle(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), r8)); -#else - return vsriq_n_u32(vshlq_n_u32(x, 32-8), x, 8); -#endif -} - -INLINE uint32x4_t rot7_128(uint32x4_t x) { - // See comment in rot16_128. - // return vorrq_u32(vshrq_n_u32(x, 7), vshlq_n_u32(x, 32 - 7)); - return vsriq_n_u32(vshlq_n_u32(x, 32-7), x, 7); -} - -// TODO: compress_neon - -// TODO: hash2_neon - -/* - * ---------------------------------------------------------------------------- - * hash4_neon - * ---------------------------------------------------------------------------- - */ - -INLINE void round_fn4(uint32x4_t v[16], uint32x4_t m[16], size_t r) { - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[15] = rot16_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot12_128(v[4]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = add_128(v[0], v[4]); - v[1] = add_128(v[1], v[5]); - v[2] = add_128(v[2], v[6]); - v[3] = add_128(v[3], v[7]); - v[12] = xor_128(v[12], v[0]); - v[13] = xor_128(v[13], v[1]); - v[14] = xor_128(v[14], v[2]); - v[15] = xor_128(v[15], v[3]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[15] = rot8_128(v[15]); - v[8] = add_128(v[8], v[12]); - v[9] = add_128(v[9], v[13]); - v[10] = add_128(v[10], v[14]); - v[11] = add_128(v[11], v[15]); - v[4] = xor_128(v[4], v[8]); - v[5] = xor_128(v[5], v[9]); - v[6] = xor_128(v[6], v[10]); - v[7] = xor_128(v[7], v[11]); - v[4] = rot7_128(v[4]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot16_128(v[15]); - v[12] = rot16_128(v[12]); - v[13] = rot16_128(v[13]); - v[14] = rot16_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot12_128(v[5]); - v[6] = rot12_128(v[6]); - v[7] = rot12_128(v[7]); - v[4] = rot12_128(v[4]); - v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = add_128(v[0], v[5]); - v[1] = add_128(v[1], v[6]); - v[2] = add_128(v[2], v[7]); - v[3] = add_128(v[3], v[4]); - v[15] = xor_128(v[15], v[0]); - v[12] = xor_128(v[12], v[1]); - v[13] = xor_128(v[13], v[2]); - v[14] = xor_128(v[14], v[3]); - v[15] = rot8_128(v[15]); - v[12] = rot8_128(v[12]); - v[13] = rot8_128(v[13]); - v[14] = rot8_128(v[14]); - v[10] = add_128(v[10], v[15]); - v[11] = add_128(v[11], v[12]); - v[8] = add_128(v[8], v[13]); - v[9] = add_128(v[9], v[14]); - v[5] = xor_128(v[5], v[10]); - v[6] = xor_128(v[6], v[11]); - v[7] = xor_128(v[7], v[8]); - v[4] = xor_128(v[4], v[9]); - v[5] = rot7_128(v[5]); - v[6] = rot7_128(v[6]); - v[7] = rot7_128(v[7]); - v[4] = rot7_128(v[4]); -} - -INLINE void transpose_vecs_128(uint32x4_t vecs[4]) { - // Individually transpose the four 2x2 sub-matrices in each corner. - uint32x4x2_t rows01 = vtrnq_u32(vecs[0], vecs[1]); - uint32x4x2_t rows23 = vtrnq_u32(vecs[2], vecs[3]); - - // Swap the top-right and bottom-left 2x2s (which just got transposed). - vecs[0] = - vcombine_u32(vget_low_u32(rows01.val[0]), vget_low_u32(rows23.val[0])); - vecs[1] = - vcombine_u32(vget_low_u32(rows01.val[1]), vget_low_u32(rows23.val[1])); - vecs[2] = - vcombine_u32(vget_high_u32(rows01.val[0]), vget_high_u32(rows23.val[0])); - vecs[3] = - vcombine_u32(vget_high_u32(rows01.val[1]), vget_high_u32(rows23.val[1])); -} - -INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, - size_t block_offset, uint32x4_t out[16]) { - out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(uint32x4_t)]); - out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(uint32x4_t)]); - out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(uint32x4_t)]); - out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(uint32x4_t)]); - out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(uint32x4_t)]); - out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(uint32x4_t)]); - out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(uint32x4_t)]); - out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(uint32x4_t)]); - out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(uint32x4_t)]); - out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(uint32x4_t)]); - out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(uint32x4_t)]); - out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(uint32x4_t)]); - out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(uint32x4_t)]); - out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(uint32x4_t)]); - out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(uint32x4_t)]); - out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(uint32x4_t)]); - transpose_vecs_128(&out[0]); - transpose_vecs_128(&out[4]); - transpose_vecs_128(&out[8]); - transpose_vecs_128(&out[12]); -} - -INLINE void load_counters4(uint64_t counter, bool increment_counter, - uint32x4_t *out_low, uint32x4_t *out_high) { - uint64_t mask = (increment_counter ? ~0 : 0); - *out_low = set4( - counter_low(counter + (mask & 0)), counter_low(counter + (mask & 1)), - counter_low(counter + (mask & 2)), counter_low(counter + (mask & 3))); - *out_high = set4( - counter_high(counter + (mask & 0)), counter_high(counter + (mask & 1)), - counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3))); -} - -void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - uint32x4_t h_vecs[8] = { - set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), - set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), - }; - uint32x4_t counter_low_vec, counter_high_vec; - load_counters4(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - uint32x4_t block_len_vec = set1_128(BLAKE3_BLOCK_LEN); - uint32x4_t block_flags_vec = set1_128(block_flags); - uint32x4_t msg_vecs[16]; - transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - uint32x4_t v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1_128(IV[0]), set1_128(IV[1]), set1_128(IV[2]), set1_128(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn4(v, msg_vecs, 0); - round_fn4(v, msg_vecs, 1); - round_fn4(v, msg_vecs, 2); - round_fn4(v, msg_vecs, 3); - round_fn4(v, msg_vecs, 4); - round_fn4(v, msg_vecs, 5); - round_fn4(v, msg_vecs, 6); - h_vecs[0] = xor_128(v[0], v[8]); - h_vecs[1] = xor_128(v[1], v[9]); - h_vecs[2] = xor_128(v[2], v[10]); - h_vecs[3] = xor_128(v[3], v[11]); - h_vecs[4] = xor_128(v[4], v[12]); - h_vecs[5] = xor_128(v[5], v[13]); - h_vecs[6] = xor_128(v[6], v[14]); - h_vecs[7] = xor_128(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs_128(&h_vecs[0]); - transpose_vecs_128(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu_128(h_vecs[0], &out[0 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[4], &out[1 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[1], &out[2 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[5], &out[3 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[2], &out[4 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[6], &out[5 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[3], &out[6 * sizeof(uint32x4_t)]); - storeu_128(h_vecs[7], &out[7 * sizeof(uint32x4_t)]); -} - -/* - * ---------------------------------------------------------------------------- - * hash_many_neon - * ---------------------------------------------------------------------------- - */ - -void blake3_compress_in_place_portable(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags); - -INLINE void hash_one_neon(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, uint8_t flags_end, - uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - // TODO: Implement compress_neon. However note that according to - // https://github.com/BLAKE2/BLAKE2/commit/7965d3e6e1b4193438b8d3a656787587d2579227, - // compress_neon might not be any faster than compress_portable. - blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= 4) { - blake3_hash4_neon(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += 4; - } - inputs += 4; - num_inputs -= 4; - out = &out[4 * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_neon(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_portable.c b/src/third_party/blake3/c/blake3_portable.c deleted file mode 100644 index 062dd1b4..00000000 --- a/src/third_party/blake3/c/blake3_portable.c +++ /dev/null @@ -1,160 +0,0 @@ -#include "blake3_impl.h" -#include - -INLINE uint32_t rotr32(uint32_t w, uint32_t c) { - return (w >> c) | (w << (32 - c)); -} - -INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, - uint32_t x, uint32_t y) { - state[a] = state[a] + state[b] + x; - state[d] = rotr32(state[d] ^ state[a], 16); - state[c] = state[c] + state[d]; - state[b] = rotr32(state[b] ^ state[c], 12); - state[a] = state[a] + state[b] + y; - state[d] = rotr32(state[d] ^ state[a], 8); - state[c] = state[c] + state[d]; - state[b] = rotr32(state[b] ^ state[c], 7); -} - -INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { - // Select the message schedule based on the round. - const uint8_t *schedule = MSG_SCHEDULE[round]; - - // Mix the columns. - g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); - g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); - g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); - g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); - - // Mix the rows. - g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); - g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); - g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); - g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); -} - -INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - uint32_t block_words[16]; - block_words[0] = load32(block + 4 * 0); - block_words[1] = load32(block + 4 * 1); - block_words[2] = load32(block + 4 * 2); - block_words[3] = load32(block + 4 * 3); - block_words[4] = load32(block + 4 * 4); - block_words[5] = load32(block + 4 * 5); - block_words[6] = load32(block + 4 * 6); - block_words[7] = load32(block + 4 * 7); - block_words[8] = load32(block + 4 * 8); - block_words[9] = load32(block + 4 * 9); - block_words[10] = load32(block + 4 * 10); - block_words[11] = load32(block + 4 * 11); - block_words[12] = load32(block + 4 * 12); - block_words[13] = load32(block + 4 * 13); - block_words[14] = load32(block + 4 * 14); - block_words[15] = load32(block + 4 * 15); - - state[0] = cv[0]; - state[1] = cv[1]; - state[2] = cv[2]; - state[3] = cv[3]; - state[4] = cv[4]; - state[5] = cv[5]; - state[6] = cv[6]; - state[7] = cv[7]; - state[8] = IV[0]; - state[9] = IV[1]; - state[10] = IV[2]; - state[11] = IV[3]; - state[12] = counter_low(counter); - state[13] = counter_high(counter); - state[14] = (uint32_t)block_len; - state[15] = (uint32_t)flags; - - round_fn(state, &block_words[0], 0); - round_fn(state, &block_words[0], 1); - round_fn(state, &block_words[0], 2); - round_fn(state, &block_words[0], 3); - round_fn(state, &block_words[0], 4); - round_fn(state, &block_words[0], 5); - round_fn(state, &block_words[0], 6); -} - -void blake3_compress_in_place_portable(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - uint32_t state[16]; - compress_pre(state, cv, block, block_len, counter, flags); - cv[0] = state[0] ^ state[8]; - cv[1] = state[1] ^ state[9]; - cv[2] = state[2] ^ state[10]; - cv[3] = state[3] ^ state[11]; - cv[4] = state[4] ^ state[12]; - cv[5] = state[5] ^ state[13]; - cv[6] = state[6] ^ state[14]; - cv[7] = state[7] ^ state[15]; -} - -void blake3_compress_xof_portable(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - uint32_t state[16]; - compress_pre(state, cv, block, block_len, counter, flags); - - store32(&out[0 * 4], state[0] ^ state[8]); - store32(&out[1 * 4], state[1] ^ state[9]); - store32(&out[2 * 4], state[2] ^ state[10]); - store32(&out[3 * 4], state[3] ^ state[11]); - store32(&out[4 * 4], state[4] ^ state[12]); - store32(&out[5 * 4], state[5] ^ state[13]); - store32(&out[6 * 4], state[6] ^ state[14]); - store32(&out[7 * 4], state[7] ^ state[15]); - store32(&out[8 * 4], state[8] ^ cv[0]); - store32(&out[9 * 4], state[9] ^ cv[1]); - store32(&out[10 * 4], state[10] ^ cv[2]); - store32(&out[11 * 4], state[11] ^ cv[3]); - store32(&out[12 * 4], state[12] ^ cv[4]); - store32(&out[13 * 4], state[13] ^ cv[5]); - store32(&out[14 * 4], state[14] ^ cv[6]); - store32(&out[15 * 4], state[15] ^ cv[7]); -} - -INLINE void hash_one_portable(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - store_cv_words(out, cv); -} - -void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs > 0) { - hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_sse2.c b/src/third_party/blake3/c/blake3_sse2.c deleted file mode 100644 index 86bb17ab..00000000 --- a/src/third_party/blake3/c/blake3_sse2.c +++ /dev/null @@ -1,566 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define DEGREE 4 - -#define _mm_shuffle_ps2(a, b, c) \ - (_mm_castps_si128( \ - _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) - -INLINE __m128i loadu(const uint8_t src[16]) { - return _mm_loadu_si128((const __m128i *)src); -} - -INLINE void storeu(__m128i src, uint8_t dest[16]) { - _mm_storeu_si128((__m128i *)dest, src); -} - -INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } - -// Note that clang-format doesn't like the name "xor" for some reason. -INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } - -INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } - -INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); -} - -INLINE __m128i rot16(__m128i x) { - return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, 0xB1), 0xB1); -} - -INLINE __m128i rot12(__m128i x) { - return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); -} - -INLINE __m128i rot8(__m128i x) { - return xorv(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 32 - 8)); -} - -INLINE __m128i rot7(__m128i x) { - return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); -} - -INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot16(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot12(*row1); -} - -INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot8(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot7(*row1); -} - -// Note the optimization here of leaving row1 as the unrotated row, rather than -// row0. All the message loads below are adjusted to compensate for this. See -// discussion at https://github.com/sneves/blake2-avx2/pull/4 -INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); -} - -INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); -} - -INLINE __m128i blend_epi16(__m128i a, __m128i b, const int16_t imm8) { - const __m128i bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01); - __m128i mask = _mm_set1_epi16(imm8); - mask = _mm_and_si128(mask, bits); - mask = _mm_cmpeq_epi16(mask, bits); - return _mm_or_si128(_mm_and_si128(mask, b), _mm_andnot_si128(mask, a)); -} - -INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - rows[0] = loadu((uint8_t *)&cv[0]); - rows[1] = loadu((uint8_t *)&cv[4]); - rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); - rows[3] = set4(counter_low(counter), counter_high(counter), - (uint32_t)block_len, (uint32_t)flags); - - __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); - __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); - __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); - __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); - - __m128i t0, t1, t2, t3, tt; - - // Round 1. The first round permutes the message words from the original - // input order, into the groups that get mixed in parallel. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 - t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 - t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 2. This round and all following rounds apply a fixed permutation - // to the message words from the round before. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 3 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 4 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 5 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 6 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 7 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); -} - -void blake3_compress_in_place_sse2(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); - storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); -} - -void blake3_compress_xof_sse2(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), &out[0]); - storeu(xorv(rows[1], rows[3]), &out[16]); - storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); - storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); -} - -INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[15] = rot16(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot12(v[4]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[15] = rot8(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot7(v[4]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot16(v[15]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[4] = rot12(v[4]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot8(v[15]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - v[4] = rot7(v[4]); -} - -INLINE void transpose_vecs(__m128i vecs[DEGREE]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is - // 22/33. Note that this doesn't split the vector into two lanes, as the - // AVX2 counterparts do. - __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); - __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); - __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); - __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); - - // Interleave 64-bit lanes. - __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); - __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); - __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); - __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); - - vecs[0] = abcd_0; - vecs[1] = abcd_1; - vecs[2] = abcd_2; - vecs[3] = abcd_3; -} - -INLINE void transpose_msg_vecs(const uint8_t *const *inputs, - size_t block_offset, __m128i out[16]) { - out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); - out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); - out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); - out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); - out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); - out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); - out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); - out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); - out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); - out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); - out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); - out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); - out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); - out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); - out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); - out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); - for (size_t i = 0; i < 4; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs(&out[0]); - transpose_vecs(&out[4]); - transpose_vecs(&out[8]); - transpose_vecs(&out[12]); -} - -INLINE void load_counters(uint64_t counter, bool increment_counter, - __m128i *out_lo, __m128i *out_hi) { - const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); - const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); - const __m128i add1 = _mm_and_si128(mask, add0); - __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); - __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), - _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); - __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); - *out_lo = l; - *out_hi = h; -} - -static -void blake3_hash4_sse2(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m128i h_vecs[8] = { - set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), - set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), - }; - __m128i counter_low_vec, counter_high_vec; - load_counters(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); - __m128i block_flags_vec = set1(block_flags); - __m128i msg_vecs[16]; - transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m128i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn(v, msg_vecs, 0); - round_fn(v, msg_vecs, 1); - round_fn(v, msg_vecs, 2); - round_fn(v, msg_vecs, 3); - round_fn(v, msg_vecs, 4); - round_fn(v, msg_vecs, 5); - round_fn(v, msg_vecs, 6); - h_vecs[0] = xorv(v[0], v[8]); - h_vecs[1] = xorv(v[1], v[9]); - h_vecs[2] = xorv(v[2], v[10]); - h_vecs[3] = xorv(v[3], v[11]); - h_vecs[4] = xorv(v[4], v[12]); - h_vecs[5] = xorv(v[5], v[13]); - h_vecs[6] = xorv(v[6], v[14]); - h_vecs[7] = xorv(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs(&h_vecs[0]); - transpose_vecs(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); - storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); - storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); - storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); - storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); - storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); - storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); - storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); -} - -INLINE void hash_one_sse2(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_sse2(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= DEGREE) { - blake3_hash4_sse2(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += DEGREE; - } - inputs += DEGREE; - num_inputs -= DEGREE; - out = &out[DEGREE * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_sse2(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -} diff --git a/src/third_party/blake3/c/blake3_sse41.c b/src/third_party/blake3/c/blake3_sse41.c deleted file mode 100644 index b28ae13b..00000000 --- a/src/third_party/blake3/c/blake3_sse41.c +++ /dev/null @@ -1,560 +0,0 @@ -#include "blake3_impl.h" - -#include - -#define DEGREE 4 - -#define _mm_shuffle_ps2(a, b, c) \ - (_mm_castps_si128( \ - _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) - -INLINE __m128i loadu(const uint8_t src[16]) { - return _mm_loadu_si128((const __m128i *)src); -} - -INLINE void storeu(__m128i src, uint8_t dest[16]) { - _mm_storeu_si128((__m128i *)dest, src); -} - -INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } - -// Note that clang-format doesn't like the name "xor" for some reason. -INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } - -INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } - -INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); -} - -INLINE __m128i rot16(__m128i x) { - return _mm_shuffle_epi8( - x, _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); -} - -INLINE __m128i rot12(__m128i x) { - return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); -} - -INLINE __m128i rot8(__m128i x) { - return _mm_shuffle_epi8( - x, _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); -} - -INLINE __m128i rot7(__m128i x) { - return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); -} - -INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot16(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot12(*row1); -} - -INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, - __m128i m) { - *row0 = addv(addv(*row0, m), *row1); - *row3 = xorv(*row3, *row0); - *row3 = rot8(*row3); - *row2 = addv(*row2, *row3); - *row1 = xorv(*row1, *row2); - *row1 = rot7(*row1); -} - -// Note the optimization here of leaving row1 as the unrotated row, rather than -// row0. All the message loads below are adjusted to compensate for this. See -// discussion at https://github.com/sneves/blake2-avx2/pull/4 -INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); -} - -INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { - *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); - *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); - *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); -} - -INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, uint8_t flags) { - rows[0] = loadu((uint8_t *)&cv[0]); - rows[1] = loadu((uint8_t *)&cv[4]); - rows[2] = set4(IV[0], IV[1], IV[2], IV[3]); - rows[3] = set4(counter_low(counter), counter_high(counter), - (uint32_t)block_len, (uint32_t)flags); - - __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); - __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); - __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); - __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); - - __m128i t0, t1, t2, t3, tt; - - // Round 1. The first round permutes the message words from the original - // input order, into the groups that get mixed in parallel. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 - t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 - t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 2. This round and all following rounds apply a fixed permutation - // to the message words from the round before. - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 3 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 4 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 5 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 6 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); - m0 = t0; - m1 = t1; - m2 = t2; - m3 = t3; - - // Round 7 - t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); - t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); - t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); - tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); - t1 = _mm_blend_epi16(tt, t1, 0xCC); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); - diagonalize(&rows[0], &rows[2], &rows[3]); - t2 = _mm_unpacklo_epi64(m3, m1); - tt = _mm_blend_epi16(t2, m2, 0xC0); - t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); - g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); - t3 = _mm_unpackhi_epi32(m1, m3); - tt = _mm_unpacklo_epi32(m2, t3); - t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); - g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); - undiagonalize(&rows[0], &rows[2], &rows[3]); -} - -void blake3_compress_in_place_sse41(uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); - storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); -} - -void blake3_compress_xof_sse41(const uint32_t cv[8], - const uint8_t block[BLAKE3_BLOCK_LEN], - uint8_t block_len, uint64_t counter, - uint8_t flags, uint8_t out[64]) { - __m128i rows[4]; - compress_pre(rows, cv, block, block_len, counter, flags); - storeu(xorv(rows[0], rows[2]), &out[0]); - storeu(xorv(rows[1], rows[3]), &out[16]); - storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); - storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); -} - -INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[15] = rot16(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot12(v[4]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); - v[0] = addv(v[0], v[4]); - v[1] = addv(v[1], v[5]); - v[2] = addv(v[2], v[6]); - v[3] = addv(v[3], v[7]); - v[12] = xorv(v[12], v[0]); - v[13] = xorv(v[13], v[1]); - v[14] = xorv(v[14], v[2]); - v[15] = xorv(v[15], v[3]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[15] = rot8(v[15]); - v[8] = addv(v[8], v[12]); - v[9] = addv(v[9], v[13]); - v[10] = addv(v[10], v[14]); - v[11] = addv(v[11], v[15]); - v[4] = xorv(v[4], v[8]); - v[5] = xorv(v[5], v[9]); - v[6] = xorv(v[6], v[10]); - v[7] = xorv(v[7], v[11]); - v[4] = rot7(v[4]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot16(v[15]); - v[12] = rot16(v[12]); - v[13] = rot16(v[13]); - v[14] = rot16(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot12(v[5]); - v[6] = rot12(v[6]); - v[7] = rot12(v[7]); - v[4] = rot12(v[4]); - v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); - v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); - v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); - v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); - v[0] = addv(v[0], v[5]); - v[1] = addv(v[1], v[6]); - v[2] = addv(v[2], v[7]); - v[3] = addv(v[3], v[4]); - v[15] = xorv(v[15], v[0]); - v[12] = xorv(v[12], v[1]); - v[13] = xorv(v[13], v[2]); - v[14] = xorv(v[14], v[3]); - v[15] = rot8(v[15]); - v[12] = rot8(v[12]); - v[13] = rot8(v[13]); - v[14] = rot8(v[14]); - v[10] = addv(v[10], v[15]); - v[11] = addv(v[11], v[12]); - v[8] = addv(v[8], v[13]); - v[9] = addv(v[9], v[14]); - v[5] = xorv(v[5], v[10]); - v[6] = xorv(v[6], v[11]); - v[7] = xorv(v[7], v[8]); - v[4] = xorv(v[4], v[9]); - v[5] = rot7(v[5]); - v[6] = rot7(v[6]); - v[7] = rot7(v[7]); - v[4] = rot7(v[4]); -} - -INLINE void transpose_vecs(__m128i vecs[DEGREE]) { - // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is - // 22/33. Note that this doesn't split the vector into two lanes, as the - // AVX2 counterparts do. - __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); - __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); - __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); - __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); - - // Interleave 64-bit lanes. - __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); - __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); - __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); - __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); - - vecs[0] = abcd_0; - vecs[1] = abcd_1; - vecs[2] = abcd_2; - vecs[3] = abcd_3; -} - -INLINE void transpose_msg_vecs(const uint8_t *const *inputs, - size_t block_offset, __m128i out[16]) { - out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); - out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); - out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); - out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); - out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); - out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); - out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); - out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); - out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); - out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); - out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); - out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); - out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); - out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); - out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); - out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); - for (size_t i = 0; i < 4; ++i) { - _mm_prefetch((const char *)&inputs[i][block_offset + 256], _MM_HINT_T0); - } - transpose_vecs(&out[0]); - transpose_vecs(&out[4]); - transpose_vecs(&out[8]); - transpose_vecs(&out[12]); -} - -INLINE void load_counters(uint64_t counter, bool increment_counter, - __m128i *out_lo, __m128i *out_hi) { - const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); - const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); - const __m128i add1 = _mm_and_si128(mask, add0); - __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); - __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), - _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); - __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); - *out_lo = l; - *out_hi = h; -} - -static -void blake3_hash4_sse41(const uint8_t *const *inputs, size_t blocks, - const uint32_t key[8], uint64_t counter, - bool increment_counter, uint8_t flags, - uint8_t flags_start, uint8_t flags_end, uint8_t *out) { - __m128i h_vecs[8] = { - set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), - set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), - }; - __m128i counter_low_vec, counter_high_vec; - load_counters(counter, increment_counter, &counter_low_vec, - &counter_high_vec); - uint8_t block_flags = flags | flags_start; - - for (size_t block = 0; block < blocks; block++) { - if (block + 1 == blocks) { - block_flags |= flags_end; - } - __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); - __m128i block_flags_vec = set1(block_flags); - __m128i msg_vecs[16]; - transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); - - __m128i v[16] = { - h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], - h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], - set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), - counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, - }; - round_fn(v, msg_vecs, 0); - round_fn(v, msg_vecs, 1); - round_fn(v, msg_vecs, 2); - round_fn(v, msg_vecs, 3); - round_fn(v, msg_vecs, 4); - round_fn(v, msg_vecs, 5); - round_fn(v, msg_vecs, 6); - h_vecs[0] = xorv(v[0], v[8]); - h_vecs[1] = xorv(v[1], v[9]); - h_vecs[2] = xorv(v[2], v[10]); - h_vecs[3] = xorv(v[3], v[11]); - h_vecs[4] = xorv(v[4], v[12]); - h_vecs[5] = xorv(v[5], v[13]); - h_vecs[6] = xorv(v[6], v[14]); - h_vecs[7] = xorv(v[7], v[15]); - - block_flags = flags; - } - - transpose_vecs(&h_vecs[0]); - transpose_vecs(&h_vecs[4]); - // The first four vecs now contain the first half of each output, and the - // second four vecs contain the second half of each output. - storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); - storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); - storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); - storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); - storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); - storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); - storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); - storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); -} - -INLINE void hash_one_sse41(const uint8_t *input, size_t blocks, - const uint32_t key[8], uint64_t counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { - uint32_t cv[8]; - memcpy(cv, key, BLAKE3_KEY_LEN); - uint8_t block_flags = flags | flags_start; - while (blocks > 0) { - if (blocks == 1) { - block_flags |= flags_end; - } - blake3_compress_in_place_sse41(cv, input, BLAKE3_BLOCK_LEN, counter, - block_flags); - input = &input[BLAKE3_BLOCK_LEN]; - blocks -= 1; - block_flags = flags; - } - memcpy(out, cv, BLAKE3_OUT_LEN); -} - -void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, - size_t blocks, const uint32_t key[8], - uint64_t counter, bool increment_counter, - uint8_t flags, uint8_t flags_start, - uint8_t flags_end, uint8_t *out) { - while (num_inputs >= DEGREE) { - blake3_hash4_sse41(inputs, blocks, key, counter, increment_counter, flags, - flags_start, flags_end, out); - if (increment_counter) { - counter += DEGREE; - } - inputs += DEGREE; - num_inputs -= DEGREE; - out = &out[DEGREE * BLAKE3_OUT_LEN]; - } - while (num_inputs > 0) { - hash_one_sse41(inputs[0], blocks, key, counter, flags, flags_start, - flags_end, out); - if (increment_counter) { - counter += 1; - } - inputs += 1; - num_inputs -= 1; - out = &out[BLAKE3_OUT_LEN]; - } -}