Add SHA256 fast implementations

cybojanek · Sep 6, 2021 · 4b950c2 · 4b950c2
1 parent 3e0f58f
commit 4b950c2
Show file tree

Hide file tree

Showing 7 changed files with 2,289 additions and 0 deletions.
diff --git a/lib/libicp/Makefile.am b/lib/libicp/Makefile.am
@@ -19,6 +19,10 @@ ASM_SOURCES_AS = \
 	asm-x86_64/modes/ghash-x86_64.S \
 	asm-x86_64/sha1/sha1-x86_64.S \
 	asm-x86_64/sha2/sha256_impl.S \
+	asm-x86_64/sha2/sha256_avx.S \
+	asm-x86_64/sha2/sha256_avx2.S \
+	asm-x86_64/sha2/sha256_ssse3.S \
+	asm-x86_64/sha2/sha256_ni.S \
 	asm-x86_64/sha2/sha512_impl.S
 else
 ASM_SOURCES_C =

diff --git a/module/icp/Makefile.in b/module/icp/Makefile.in
@@ -58,6 +58,10 @@ $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/modes/aesni-gcm-x86_64.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/modes/ghash-x86_64.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha1/sha1-x86_64.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha256_impl.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha256_avx.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha256_avx2.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha256_ssse3.o
+$(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha256_ni.o
 $(MODULE)-$(CONFIG_X86_64) += asm-x86_64/sha2/sha512_impl.o
 
 $(MODULE)-$(CONFIG_X86) += algs/modes/gcm_pclmulqdq.o

diff --git a/module/icp/algs/sha2/sha2.c b/module/icp/algs/sha2/sha2.c
@@ -58,6 +58,8 @@ static void Encode64(uint8_t *, uint64_t *, size_t);
 /* userspace only supports the generic version */
 #if	defined(__amd64) && defined(_KERNEL)
 
+#include <sys/simd.h>
+
 typedef void (*sha2_block_f)(SHA2_CTX *ctx, const void *in, size_t num);
 
 void SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);
@@ -82,13 +84,74 @@ static alg_impl_ops_t sha256_impl_generic = {
 #if defined(__x86_64)
 static alg_impl_ops_t sha256_x86_64_generic = {
 	SHA256TransformBlocks, alg_impl_will_always_work, 1, "x86_64"};
+
+#if defined(HAVE_AVX) && defined(_KERNEL)
+static boolean_t
+sha256_avx_will_work(void)
+{
+	return (kfpu_allowed() && zfs_avx_available());
+}
+
+extern void sha256_avx_transform(SHA2_CTX *ctx, const void *in, size_t num);
+static alg_impl_ops_t sha256_x86_64_avx = {
+	sha256_avx_transform, sha256_avx_will_work, 10, "sha-avx"};
+#endif
+
+#if defined(HAVE_AVX2) && defined(_KERNEL)
+static boolean_t
+sha256_avx2_will_work(void)
+{
+	return (kfpu_allowed() && zfs_avx2_available());
+}
+
+extern void sha256_avx2_transform(SHA2_CTX *ctx, const void *in, size_t num);
+static alg_impl_ops_t sha256_x86_64_avx2 = {
+	sha256_avx2_transform, sha256_avx2_will_work, 20, "sha-avx2"};
+#endif
+
+#if defined(HAVE_SSSE3) && defined(_KERNEL)
+static boolean_t
+sha256_ssse3_will_work(void)
+{
+	return (kfpu_allowed() && zfs_ssse3_available());
+}
+
+extern void sha256_ssse3_transform(SHA2_CTX *ctx, const void *in, size_t num);
+static alg_impl_ops_t sha256_x86_64_ssse3 = {
+	sha256_ssse3_transform, sha256_ssse3_will_work, 30, "sha-ssse3"};
+#endif
+
+#if defined(HAVE_SHA) && defined(_KERNEL)
+static boolean_t
+sha256_ni_will_work(void)
+{
+	return (kfpu_allowed() && zfs_sha_available());
+}
+
+extern void sha256_ni_transform(SHA2_CTX *ctx, const void *in, size_t num);
+static alg_impl_ops_t sha256_x86_64_ni = {
+	sha256_ni_transform, sha256_ni_will_work, 40, "sha-ni"};
+#endif
+
 #endif
 
 /* All compiled in implementations */
 static const alg_impl_ops_t *sha256_all_impl[] = {
 	&sha256_impl_generic,
 #if defined(__x86_64)
 	&sha256_x86_64_generic,
+#if defined(HAVE_AVX) && defined(_KERNEL)
+	&sha256_x86_64_avx,
+#endif
+#if defined(HAVE_AVX2) && defined(_KERNEL)
+	&sha256_x86_64_avx2,
+#endif
+#if defined(HAVE_SSSE3) && defined(_KERNEL)
+	&sha256_x86_64_ssse3,
+#endif
+#if defined(HAVE_SHA) && defined(_KERNEL)
+	&sha256_x86_64_ni,
+#endif
 #endif
 };