Add icp algorithm implementation selector

lib/libicp/Makefile.am

@@ -38,6 +38,7 @@ KERNEL_C = \
 	algs/aes/aes_impl.c \
 	algs/aes/aes_modes.c \
 	algs/edonr/edonr.c \
+	algs/impl/impl.c \
 	algs/modes/modes.c \
 	algs/modes/cbc.c \
 	algs/modes/gcm_generic.c \

lib/libzfs/Makefile.am

@@ -48,6 +48,7 @@ USER_C += \
 endif
 
 KERNEL_C = \
+	algs/impl/impl.c \
 	algs/sha2/sha2.c \
 	cityhash.c \
 	zfeature_common.c \

module/icp/Makefile.in

@@ -43,6 +43,7 @@ $(MODULE)-objs += algs/aes/aes_impl_generic.o
 $(MODULE)-objs += algs/aes/aes_impl.o
 $(MODULE)-objs += algs/aes/aes_modes.o
 $(MODULE)-objs += algs/edonr/edonr.o
+$(MODULE)-objs += algs/impl/impl.o
 $(MODULE)-objs += algs/sha1/sha1.o
 $(MODULE)-objs += algs/sha2/sha2.o
 $(MODULE)-objs += algs/skein/skein.o
@@ -86,6 +87,7 @@ ICP_DIRS = \
 	algs/aes \
 	algs/edonr \
 	algs/modes \
+	algs/impl \
 	algs/sha1 \
 	algs/sha2 \
 	algs/skein \

module/icp/algs/impl/impl.c

@@ -0,0 +1,175 @@
+
+#include <sys/simd.h>
+#include <impl/impl.h>
+
+#define	ALG_IMPL_READ(i) (*(volatile uint32_t *) &(i))
+
+static const struct {
+	char *name;
+	uint32_t sel;
+} alg_impl_opts[] = {
+	{ "cycle", ALG_IMPL_CYCLE },
+	{ "fastest", ALG_IMPL_FASTEST },
+};
+
+void
+alg_impl_init(alg_impl_conf_t *conf)
+{
+	const alg_impl_ops_t *curr_impl, *fast_impl = NULL;
+	size_t i, c;
+	uint32_t max_priority = 0;
+
+	/* Copy from available to supported */
+	for (i = 0, c = 0; i < conf->available_n; i++) {
+		curr_impl = conf->available[i];
+
+		if (curr_impl->is_supported()) {
+			conf->supported[c++] = curr_impl;
+
+			/* Keep track of fastest */
+			if (curr_impl->priority > max_priority) {
+				max_priority = curr_impl->priority;
+				fast_impl = curr_impl;
+			}
+		}
+	}
+	conf->supported_n = c;
+
+	/* Copy the fastest implementation. */
+	ASSERT3P(fast_impl, !=, NULL);
+	memcpy(&conf->fastest, fast_impl, sizeof (*fast_impl));
+	strlcpy(conf->fastest.name, "fastest", ALG_IMPL_NAME_MAX);
+
+	/*
+	 * Finish initialization. At this, user_sel_impl can only be one of
+	 * alg_impl_opts, because initialized is not yet set, so
+	 * alg_impl_set would have returned EINVAL for any other value.
+	 */
+	conf->cycle_impl_idx = 0;
+	atomic_swap_32(&conf->icp_alg_impl, conf->user_sel_impl);
+	conf->initialized = B_TRUE;
+}
+
+int
+alg_impl_get(alg_impl_conf_t *conf, char *buffer)
+{
+	int cnt = 0;
+	size_t i = 0;
+	char *fmt = NULL;
+	const uint32_t impl = ALG_IMPL_READ(conf->icp_alg_impl);
+
+	ASSERT(conf->initialized);
+
+	/* First check mandatory options. */
+	for (i = 0; i < ARRAY_SIZE(alg_impl_opts); i++) {
+		fmt = (impl == alg_impl_opts[i].sel) ? "[%s] " : "%s ";
+		cnt += sprintf(buffer + cnt, fmt, alg_impl_opts[i].name);
+	}
+
+	/* Then check all supported. */
+	for (i = 0; i < conf->supported_n; i++) {
+		fmt = (i == impl) ? "[%s] " : "%s ";
+		cnt += sprintf(buffer + cnt, fmt, conf->supported[i]->name);
+	}
+
+	return (cnt);
+}
+
+int
+alg_impl_set(alg_impl_conf_t *conf, const char *val)
+{
+	int err = -EINVAL;
+	char req_name[ALG_IMPL_NAME_MAX];
+	uint32_t impl = ALG_IMPL_READ(conf->user_sel_impl);
+	size_t i;
+
+	/* Sanitize input. */
+	i = strnlen(val, ALG_IMPL_NAME_MAX);
+	if (i == 0 || i >= ALG_IMPL_NAME_MAX) {
+		return (err);
+	}
+
+	/* Copy and NULL terminate. */
+	strlcpy(req_name, val, ALG_IMPL_NAME_MAX);
+	while (i > 0 && isspace(req_name[i-1])) {
+		i--;
+	}
+	req_name[i] = '\0';
+
+	/* First check mandatory options. */
+	for (i = 0; i < ARRAY_SIZE(alg_impl_opts); i++) {
+		if (strcmp(req_name, alg_impl_opts[i].name) == 0) {
+			impl = alg_impl_opts[i].sel;
+			err = 0;
+			break;
+		}
+	}
+
+	/* Then check all supported impl if init() was already called. */
+	if (err != 0 && conf->initialized) {
+		for (i = 0; i < conf->supported_n; i++) {
+			if (strcmp(req_name, conf->supported[i]->name) == 0) {
+				impl = i;
+				err = 0;
+				break;
+			}
+		}
+	}
+
+	/* If success, then set depending on configuration state. */
+	if (err == 0) {
+		if (conf->initialized) {
+			atomic_swap_32(&conf->icp_alg_impl, impl);
+		} else {
+			atomic_swap_32(&conf->user_sel_impl, impl);
+		}
+	}
+
+	return (err);
+}
+
+const alg_impl_ops_t *
+alg_impl_get_ops(alg_impl_conf_t *conf)
+{
+	const alg_impl_ops_t *ops = conf->generic;
+
+	/* Check if we are not allowed to use faster versions. */
+	if (!kfpu_allowed()) {
+		return (ops);
+	}
+
+	const uint32_t impl = ALG_IMPL_READ(conf->icp_alg_impl);
+
+	switch (impl) {
+	case ALG_IMPL_FASTEST:
+		ASSERT(conf->initialized);
+		ops = &conf->fastest;
+		break;
+	case ALG_IMPL_CYCLE:
+		/* Cycle through supported implementations */
+		ASSERT(conf->initialized);
+		ASSERT3U(conf->supported_n, >, 0);
+
+		size_t idx = (++conf->cycle_impl_idx) % conf->supported_n;
+		ops = conf->supported[idx];
+		break;
+	default:
+		/* Use a specific implementation. */
+		ASSERT3U(impl, <, conf->supported_n);
+		ASSERT3U(conf->supported_n, >, 0);
+		if (impl < conf->supported_n) {
+			ops = conf->supported[impl];
+		}
+		break;
+	}
+
+	ASSERT3P(ops, !=, NULL);
+
+	return (ops);
+}
+
+boolean_t
+alg_impl_will_always_work(void)
+{
+	return (B_TRUE);
+}

module/icp/include/impl/impl.h

@@ -0,0 +1,145 @@
+#ifndef	_ALG_IMPL_H
+#define	_ALG_IMPL_H
+
+/*
+ * Common code for managing hash implementations.
+ */
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#include <sys/zfs_context.h>
+
+/* A function that tests whether method will work. */
+typedef boolean_t (*alg_impl_will_work_f)(void);
+
+/* Maximum length of hash implementation name (including NULL). */
+#define	ALG_IMPL_NAME_MAX (16)
+
+/* Hash implementation operations and name. */
+typedef struct alg_impl_ops {
+	void *ctx; /* Algorithm implementation context. */
+
+	alg_impl_will_work_f is_supported;
+
+	/* Algorithm priorty. Higher is faster. */
+	uint32_t priority;
+
+	char name[ALG_IMPL_NAME_MAX];
+} alg_impl_ops_t;
+
+/* Fastest hash implementation. */
+#define	ALG_IMPL_FASTEST (UINT32_MAX)
+
+/* Cycle to next implementation. */
+#define	ALG_IMPL_CYCLE (UINT32_MAX-1)
+
+/*
+ * Algorithm implementation configuration.
+ */
+typedef struct alg_impl_conf {
+	const alg_impl_ops_t **available; /* Available implementations. */
+	size_t available_n; /* Length of available. */
+
+	const alg_impl_ops_t **supported; /* Supported implementations. */
+	size_t supported_n; /* Length of supported. */
+
+    size_t cycle_impl_idx; /* Index in case of "cycle" implementation. */
+
+	/*
+	 * Index into supported or one of ALG_IMPL_CYCLE or ALG_IMPL_FASTEST.
+	 */
+	uint32_t icp_alg_impl;
+
+	/*
+	 * User desired value of icp_alg_impl, before initialization finishes.
+	 * Must be one of ALG_IMPL_FASTEST or ALG_IMPL_CYCLE.
+	 */
+	uint32_t user_sel_impl;
+
+
+	alg_impl_ops_t fastest; /* Fastest implementation. */
+	alg_impl_ops_t *generic; /* Fallback generic implementation. */
+
+	boolean_t initialized; /* Is configuration initilized. */
+} alg_impl_conf_t;
+
+/*
+ * Delcare a alg_impl_ops_t struct.
+ */
+#define	ALG_IMPL_CONF_DECL(avail, supp, gen) \
+	{ \
+		.available = avail, \
+		.available_n = ARRAY_SIZE(avail), \
+		.supported = (const alg_impl_ops_t **)(supp), \
+		.supported_n = ARRAY_SIZE(supp), \
+		.icp_alg_impl = ALG_IMPL_FASTEST, \
+		.user_sel_impl = ALG_IMPL_FASTEST, \
+		.generic = &(gen), \
+	}
+
+/*
+ * Initialize conf.
+ *
+ * The following must be already set:
+ * - available
+ * - available_n
+ * - generic
+ * - supported - points to an empty array of size available_n
+ * - user_sel_impl
+ *
+ * @param[in,out] conf
+ */
+void alg_impl_init(alg_impl_conf_t *conf);
+
+/*
+ * Get the selected and supported implementations.
+ *
+ * Uses sprintf and writes at most: N * (ALG_IMPL_NAME_MAX + 2) bytes to
+ * buffer, where N is ARRAY_SIZE(alg_impl_opts) + conf->supported_n
+ *
+ * @param[in]     conf
+ * @param[in,out] buffer
+ *
+ * @return number of bytes written
+ */
+int alg_impl_get(alg_impl_conf_t *conf, char *buffer);
+
+/*
+ * Set the desired implementation to use.
+ *
+ * @param[in,out] conf
+ * @param[in]     val one of supported[i].name
+ *
+ * @retval 0 success
+ * @retval -EINVAL unknown or unsupported implementation
+ */
+int alg_impl_set(alg_impl_conf_t *conf, const char *val);
+
+/*
+ * Get the implementation ops to use.
+ *
+ * - If !kfpu_allowed, then returns conf->generic
+ * - If alg_impl_set was set to "cycle", then returns the next implementation.
+ * - If alg_impl_set was set to any of supported[i].name, then returns that
+ *   implementation.
+ *
+ * @param[in,out] conf
+ *
+ * @return one of conf->generic, conf->fastest, or conf->supported
+ */
+const alg_impl_ops_t *alg_impl_get_ops(alg_impl_conf_t *conf);
+
+/*
+ * Helper function that always returns true.
+ *
+ * @retval B_TRUE
+ */
+boolean_t alg_impl_will_always_work(void);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _ALG_IMPL_H */