Initialize sha2 module with existing implementations

lib/libspl/include/sys/sha2.h

@@ -114,6 +114,8 @@ extern void SHA2Update(SHA2_CTX *, const void *, size_t);
 
 extern void SHA2Final(void *, SHA2_CTX *);
 
+extern void sha2_impl_init(void);
+
 #ifdef _SHA2_IMPL
 /*
  * The following types/functions are all private to the implementation

module/icp/algs/sha2/sha2.c

@@ -39,7 +39,9 @@
 #include <sys/zfs_context.h>
 #define	_SHA2_IMPL
 #include <sys/sha2.h>
+#include <impl/impl.h>
 #include <sha2/sha2_consts.h>
+#include <sha2/sha2_impl.h>
 
 #define	_RESTRICT_KYWD
 
@@ -52,19 +54,75 @@
 static void Encode(uint8_t *, uint32_t *, size_t);
 static void Encode64(uint8_t *, uint64_t *, size_t);
 
+
 /* userspace only supports the generic version */
 #if	defined(__amd64) && defined(_KERNEL)
-#define	SHA512Transform(ctx, in) SHA512TransformBlocks((ctx), (in), 1)
-#define	SHA256Transform(ctx, in) SHA256TransformBlocks((ctx), (in), 1)
+
+typedef void (*sha2_block_f)(SHA2_CTX *ctx, const void *in, size_t num);
 
 void SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);
 void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);
 
 #else
 static void SHA256Transform(SHA2_CTX *, const uint8_t *);
 static void SHA512Transform(SHA2_CTX *, const uint8_t *);
+
+static void SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);
+static void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num);
+
 #endif	/* __amd64 && _KERNEL */
 
+/*
+ * For compatibility with existing code, generic and x86_64 may point to the
+ * same implementation.
+ */
+static alg_impl_ops_t sha256_impl_generic = {
+	SHA256TransformBlocks, alg_impl_will_always_work, 0, "generic"};
+
+#if defined(__x86_64)
+static alg_impl_ops_t sha256_x86_64_generic = {
+	SHA256TransformBlocks, alg_impl_will_always_work, 1, "x86_64"};
+#endif
+
+/* All compiled in implementations */
+static const alg_impl_ops_t *sha256_all_impl[] = {
+	&sha256_impl_generic,
+#if defined(__x86_64)
+	&sha256_x86_64_generic,
+#endif
+};
+
+static alg_impl_ops_t *sha256_supp_impl[ARRAY_SIZE(sha256_all_impl)];
+
+static alg_impl_conf_t sha256_conf_impl = ALG_IMPL_CONF_DECL(
+	sha256_all_impl, sha256_supp_impl, sha256_impl_generic);
+
+static alg_impl_ops_t sha512_impl_generic = {
+	SHA512TransformBlocks, alg_impl_will_always_work, 0, "generic"};
+
+#if defined(__x86_64)
+static alg_impl_ops_t sha512_x86_64_generic = {
+	SHA512TransformBlocks, alg_impl_will_always_work, 1, "x86_64"};
+#endif
+
+/* All compiled in implementations */
+static const alg_impl_ops_t *sha512_all_impl[] = {
+	&sha512_impl_generic,
+#if defined(__x86_64)
+	&sha512_x86_64_generic,
+#endif
+};
+
+static alg_impl_ops_t *sha512_supp_impl[ARRAY_SIZE(sha512_all_impl)];
+
+static alg_impl_conf_t sha512_conf_impl = ALG_IMPL_CONF_DECL(
+	sha512_all_impl, sha512_supp_impl, sha512_impl_generic);
+
+void sha2_impl_init(void) {
+	alg_impl_init(&sha256_conf_impl);
+	alg_impl_init(&sha512_conf_impl);
+}
+
 static uint8_t PADDING[128] = { 0x80, /* all zeros */ };
 
 /*
@@ -146,6 +204,15 @@ static uint8_t PADDING[128] = { 0x80, /* all zeros */ };
 #if	!defined(__amd64) || !defined(_KERNEL)
 /* SHA256 Transform */
 
+static void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num)
+{
+	const uint8_t *ptr = in;
+	for (size_t i = 0; i < num; i++) {
+		SHA256Transform(ctx, ptr);
+		ptr += 64;
+	}
+}
+
 static void
 SHA256Transform(SHA2_CTX *ctx, const uint8_t *blk)
 {
@@ -355,6 +422,15 @@ SHA256Transform(SHA2_CTX *ctx, const uint8_t *blk)
 
 /* SHA384 and SHA512 Transform */
 
+static void SHA512TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num)
+{
+	const uint8_t *ptr = in;
+	for (size_t i = 0; i < num; i++) {
+		SHA512Transform(ctx, ptr);
+		ptr += 128;
+	}
+}
+
 static void
 SHA512Transform(SHA2_CTX *ctx, const uint8_t *blk)
 {
@@ -785,6 +861,10 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len)
 	const uint8_t	*input = inptr;
 	uint32_t	algotype = ctx->algotype;
 
+#if	defined(__amd64) && defined(_KERNEL)
+	sha2_block_f sha2_impl = NULL;
+#endif
+
 	/* check for noop */
 	if (input_len == 0)
 		return;
@@ -801,6 +881,10 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len)
 
 		ctx->count.c32[0] += (input_len >> 29);
 
+#if	defined(__amd64) && defined(_KERNEL)
+		const alg_impl_ops_t *ops = alg_impl_get_ops(&sha256_conf_impl);
+		sha2_impl = (sha2_block_f)(ops->ctx);
+#endif
 	} else {
 		buf_limit = 128;
 
@@ -812,6 +896,11 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len)
 			ctx->count.c64[0]++;
 
 		ctx->count.c64[0] += (input_len >> 29);
+
+#if	defined(__amd64) && defined(_KERNEL)
+		const alg_impl_ops_t *ops = alg_impl_get_ops(&sha512_conf_impl);
+		sha2_impl = (sha2_block_f)(ops->ctx);
+#endif
 	}
 
 	buf_len = buf_limit - buf_index;
@@ -831,11 +920,14 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len)
 		 */
 		if (buf_index) {
 			bcopy(input, &ctx->buf_un.buf8[buf_index], buf_len);
+#if !defined(__amd64) || !defined(_KERNEL)
 			if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE)
 				SHA256Transform(ctx, ctx->buf_un.buf8);
 			else
 				SHA512Transform(ctx, ctx->buf_un.buf8);
-
+#else
+			sha2_impl(ctx, ctx->buf_un.buf8, 1);
+#endif
 			i = buf_len;
 		}
 
@@ -855,15 +947,13 @@ SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len)
 		if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) {
 			block_count = (input_len - i) >> 6;
 			if (block_count > 0) {
-				SHA256TransformBlocks(ctx, &input[i],
-				    block_count);
+				sha2_impl(ctx, &input[i], block_count);
 				i += block_count << 6;
 			}
 		} else {
 			block_count = (input_len - i) >> 7;
 			if (block_count > 0) {
-				SHA512TransformBlocks(ctx, &input[i],
-				    block_count);
+				sha2_impl(ctx, &input[i], block_count);
 				i += block_count << 7;
 			}
 		}
@@ -954,3 +1044,22 @@ EXPORT_SYMBOL(SHA2Init);
 EXPORT_SYMBOL(SHA2Update);
 EXPORT_SYMBOL(SHA2Final);
 #endif
+
+#if defined(_KERNEL) && defined(__linux__)
+
+static int
+icp_sha256_impl_set(const char *val, zfs_kernel_param_t *kp)
+{
+	return (alg_impl_set(&sha256_conf_impl, val));
+}
+
+static int
+icp_sha256_impl_get(char *buffer, zfs_kernel_param_t *kp)
+{
+	return (alg_impl_get(&sha256_conf_impl, buffer));
+}
+
+module_param_call(icp_sha256_impl, icp_sha256_impl_set, icp_sha256_impl_get,
+    NULL, 0644);
+MODULE_PARM_DESC(icp_sha256_impl, "Select sha256 implementation.");
+#endif

module/icp/include/sha2/sha2_impl.h

@@ -57,6 +57,11 @@ typedef struct sha2_hmac_ctx {
 	SHA2_CTX		hc_ocontext;	/* outer SHA2 context */
 } sha2_hmac_ctx_t;
 
+/*
+ * Initialize sha2 implementations.
+ */
+void sha2_impl_init(void);
+
 #ifdef	__cplusplus
 }
 #endif

module/icp/io/sha2_mod.c

@@ -216,6 +216,9 @@ sha2_mod_init(void)
 {
 	int ret;
 
+	/* Determine the fastest available implementation. */
+	sha2_impl_init();
+
 	if ((ret = mod_install(&modlinkage)) != 0)
 		return (ret);
 

tests/zfs-tests/tests/functional/checksum/Makefile.am

@@ -2,6 +2,7 @@ include $(top_srcdir)/config/Rules.am
 
 LDADD = \
 	$(abs_top_builddir)/lib/libicp/libicp.la \
+	$(abs_top_builddir)/lib/libspl/libspl.la \
 	$(abs_top_builddir)/lib/libspl/libspl_assert.la
 
 pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/checksum