diff --git a/lib/libicp/Makefile.am b/lib/libicp/Makefile.am index 0b87a988c07e..0b44fedf1a47 100644 --- a/lib/libicp/Makefile.am +++ b/lib/libicp/Makefile.am @@ -38,6 +38,7 @@ KERNEL_C = \ algs/aes/aes_impl.c \ algs/aes/aes_modes.c \ algs/edonr/edonr.c \ + algs/impl/impl.c \ algs/modes/modes.c \ algs/modes/cbc.c \ algs/modes/gcm_generic.c \ diff --git a/lib/libzfs/Makefile.am b/lib/libzfs/Makefile.am index e3527ffe7058..ec1183550d66 100644 --- a/lib/libzfs/Makefile.am +++ b/lib/libzfs/Makefile.am @@ -48,6 +48,7 @@ USER_C += \ endif KERNEL_C = \ + algs/impl/impl.c \ algs/sha2/sha2.c \ cityhash.c \ zfeature_common.c \ diff --git a/module/icp/Makefile.in b/module/icp/Makefile.in index 858c5a610c26..11ca301b1ced 100644 --- a/module/icp/Makefile.in +++ b/module/icp/Makefile.in @@ -43,6 +43,7 @@ $(MODULE)-objs += algs/aes/aes_impl_generic.o $(MODULE)-objs += algs/aes/aes_impl.o $(MODULE)-objs += algs/aes/aes_modes.o $(MODULE)-objs += algs/edonr/edonr.o +$(MODULE)-objs += algs/impl/impl.o $(MODULE)-objs += algs/sha1/sha1.o $(MODULE)-objs += algs/sha2/sha2.o $(MODULE)-objs += algs/skein/skein.o @@ -86,6 +87,7 @@ ICP_DIRS = \ algs/aes \ algs/edonr \ algs/modes \ + algs/impl \ algs/sha1 \ algs/sha2 \ algs/skein \ diff --git a/module/icp/algs/impl/impl.c b/module/icp/algs/impl/impl.c new file mode 100644 index 000000000000..f26dee801766 --- /dev/null +++ b/module/icp/algs/impl/impl.c @@ -0,0 +1,175 @@ + +#include +#include + +#define ALG_IMPL_READ(i) (*(volatile uint32_t *) &(i)) + +static const struct { + char *name; + uint32_t sel; +} alg_impl_opts[] = { + { "cycle", ALG_IMPL_CYCLE }, + { "fastest", ALG_IMPL_FASTEST }, +}; + +void +alg_impl_init(alg_impl_conf_t *conf) +{ + const alg_impl_ops_t *curr_impl, *fast_impl = NULL; + size_t i, c; + uint32_t max_priority = 0; + + /* Copy from available to supported */ + for (i = 0, c = 0; i < conf->available_n; i++) { + curr_impl = conf->available[i]; + + if (curr_impl->is_supported()) { + conf->supported[c++] = curr_impl; + + /* Keep track of fastest */ + if (curr_impl->priority > max_priority) { + max_priority = curr_impl->priority; + fast_impl = curr_impl; + } + } + } + conf->supported_n = c; + + /* Copy the fastest implementation. */ + ASSERT3P(fast_impl, !=, NULL); + memcpy(&conf->fastest, fast_impl, sizeof (*fast_impl)); + strlcpy(conf->fastest.name, "fastest", ALG_IMPL_NAME_MAX); + + /* + * Finish initialization. At this, user_sel_impl can only be one of + * alg_impl_opts, because initialized is not yet set, so + * alg_impl_set would have returned EINVAL for any other value. + */ + conf->cycle_impl_idx = 0; + atomic_swap_32(&conf->icp_alg_impl, conf->user_sel_impl); + conf->initialized = B_TRUE; +} + +int +alg_impl_get(alg_impl_conf_t *conf, char *buffer) +{ + int cnt = 0; + size_t i = 0; + char *fmt = NULL; + const uint32_t impl = ALG_IMPL_READ(conf->icp_alg_impl); + + ASSERT(conf->initialized); + + /* First check mandatory options. */ + for (i = 0; i < ARRAY_SIZE(alg_impl_opts); i++) { + fmt = (impl == alg_impl_opts[i].sel) ? "[%s] " : "%s "; + cnt += sprintf(buffer + cnt, fmt, alg_impl_opts[i].name); + } + + /* Then check all supported. */ + for (i = 0; i < conf->supported_n; i++) { + fmt = (i == impl) ? "[%s] " : "%s "; + cnt += sprintf(buffer + cnt, fmt, conf->supported[i]->name); + } + + return (cnt); +} + +int +alg_impl_set(alg_impl_conf_t *conf, const char *val) +{ + int err = -EINVAL; + char req_name[ALG_IMPL_NAME_MAX]; + uint32_t impl = ALG_IMPL_READ(conf->user_sel_impl); + size_t i; + + /* Sanitize input. */ + i = strnlen(val, ALG_IMPL_NAME_MAX); + if (i == 0 || i >= ALG_IMPL_NAME_MAX) { + return (err); + } + + /* Copy and NULL terminate. */ + strlcpy(req_name, val, ALG_IMPL_NAME_MAX); + while (i > 0 && isspace(req_name[i-1])) { + i--; + } + req_name[i] = '\0'; + + /* First check mandatory options. */ + for (i = 0; i < ARRAY_SIZE(alg_impl_opts); i++) { + if (strcmp(req_name, alg_impl_opts[i].name) == 0) { + impl = alg_impl_opts[i].sel; + err = 0; + break; + } + } + + /* Then check all supported impl if init() was already called. */ + if (err != 0 && conf->initialized) { + for (i = 0; i < conf->supported_n; i++) { + if (strcmp(req_name, conf->supported[i]->name) == 0) { + impl = i; + err = 0; + break; + } + } + } + + /* If success, then set depending on configuration state. */ + if (err == 0) { + if (conf->initialized) { + atomic_swap_32(&conf->icp_alg_impl, impl); + } else { + atomic_swap_32(&conf->user_sel_impl, impl); + } + } + + return (err); +} + +const alg_impl_ops_t * +alg_impl_get_ops(alg_impl_conf_t *conf) +{ + const alg_impl_ops_t *ops = conf->generic; + + /* Check if we are not allowed to use faster versions. */ + if (!kfpu_allowed()) { + return (ops); + } + + const uint32_t impl = ALG_IMPL_READ(conf->icp_alg_impl); + + switch (impl) { + case ALG_IMPL_FASTEST: + ASSERT(conf->initialized); + ops = &conf->fastest; + break; + case ALG_IMPL_CYCLE: + /* Cycle through supported implementations */ + ASSERT(conf->initialized); + ASSERT3U(conf->supported_n, >, 0); + + size_t idx = (++conf->cycle_impl_idx) % conf->supported_n; + ops = conf->supported[idx]; + break; + default: + /* Use a specific implementation. */ + ASSERT3U(impl, <, conf->supported_n); + ASSERT3U(conf->supported_n, >, 0); + if (impl < conf->supported_n) { + ops = conf->supported[impl]; + } + break; + } + + ASSERT3P(ops, !=, NULL); + + return (ops); +} + +boolean_t +alg_impl_will_always_work(void) +{ + return (B_TRUE); +} diff --git a/module/icp/include/impl/impl.h b/module/icp/include/impl/impl.h new file mode 100644 index 000000000000..8223332a70d6 --- /dev/null +++ b/module/icp/include/impl/impl.h @@ -0,0 +1,145 @@ +#ifndef _ALG_IMPL_H +#define _ALG_IMPL_H + +/* + * Common code for managing hash implementations. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* A function that tests whether method will work. */ +typedef boolean_t (*alg_impl_will_work_f)(void); + +/* Maximum length of hash implementation name (including NULL). */ +#define ALG_IMPL_NAME_MAX (16) + +/* Hash implementation operations and name. */ +typedef struct alg_impl_ops { + void *ctx; /* Algorithm implementation context. */ + + alg_impl_will_work_f is_supported; + + /* Algorithm priorty. Higher is faster. */ + uint32_t priority; + + char name[ALG_IMPL_NAME_MAX]; +} alg_impl_ops_t; + +/* Fastest hash implementation. */ +#define ALG_IMPL_FASTEST (UINT32_MAX) + +/* Cycle to next implementation. */ +#define ALG_IMPL_CYCLE (UINT32_MAX-1) + +/* + * Algorithm implementation configuration. + */ +typedef struct alg_impl_conf { + const alg_impl_ops_t **available; /* Available implementations. */ + size_t available_n; /* Length of available. */ + + const alg_impl_ops_t **supported; /* Supported implementations. */ + size_t supported_n; /* Length of supported. */ + + size_t cycle_impl_idx; /* Index in case of "cycle" implementation. */ + + /* + * Index into supported or one of ALG_IMPL_CYCLE or ALG_IMPL_FASTEST. + */ + uint32_t icp_alg_impl; + + /* + * User desired value of icp_alg_impl, before initialization finishes. + * Must be one of ALG_IMPL_FASTEST or ALG_IMPL_CYCLE. + */ + uint32_t user_sel_impl; + + + alg_impl_ops_t fastest; /* Fastest implementation. */ + alg_impl_ops_t *generic; /* Fallback generic implementation. */ + + boolean_t initialized; /* Is configuration initilized. */ +} alg_impl_conf_t; + +/* + * Delcare a alg_impl_ops_t struct. + */ +#define ALG_IMPL_CONF_DECL(avail, supp, gen) \ + { \ + .available = avail, \ + .available_n = ARRAY_SIZE(avail), \ + .supported = (const alg_impl_ops_t **)(supp), \ + .supported_n = ARRAY_SIZE(supp), \ + .icp_alg_impl = ALG_IMPL_FASTEST, \ + .user_sel_impl = ALG_IMPL_FASTEST, \ + .generic = &(gen), \ + } + +/* + * Initialize conf. + * + * The following must be already set: + * - available + * - available_n + * - generic + * - supported - points to an empty array of size available_n + * - user_sel_impl + * + * @param[in,out] conf + */ +void alg_impl_init(alg_impl_conf_t *conf); + +/* + * Get the selected and supported implementations. + * + * Uses sprintf and writes at most: N * (ALG_IMPL_NAME_MAX + 2) bytes to + * buffer, where N is ARRAY_SIZE(alg_impl_opts) + conf->supported_n + * + * @param[in] conf + * @param[in,out] buffer + * + * @return number of bytes written + */ +int alg_impl_get(alg_impl_conf_t *conf, char *buffer); + +/* + * Set the desired implementation to use. + * + * @param[in,out] conf + * @param[in] val one of supported[i].name + * + * @retval 0 success + * @retval -EINVAL unknown or unsupported implementation + */ +int alg_impl_set(alg_impl_conf_t *conf, const char *val); + +/* + * Get the implementation ops to use. + * + * - If !kfpu_allowed, then returns conf->generic + * - If alg_impl_set was set to "cycle", then returns the next implementation. + * - If alg_impl_set was set to any of supported[i].name, then returns that + * implementation. + * + * @param[in,out] conf + * + * @return one of conf->generic, conf->fastest, or conf->supported + */ +const alg_impl_ops_t *alg_impl_get_ops(alg_impl_conf_t *conf); + +/* + * Helper function that always returns true. + * + * @retval B_TRUE + */ +boolean_t alg_impl_will_always_work(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _ALG_IMPL_H */