Implementation of vdev_raidz generate and reconstruct routines

- specialized gen/rec routines for all RAIDZ levels,
- new scalar raidz implementation (unrolled),
- two x86_64 SIMD implementations (SSE_4.1 and AVX2 instructions sets),
- fastest routines selected on module load (benchmark).

New zfs module parameters:
- zfs_raidz_math_impl (int): selects a new implementation to use:
    "-1" - the fastest (DEFAULT),
     "0" - new scalar routines,
     "1" - new SSE routines,
     "2" - new AVX2 routines.

- zfs_raidz_new_math (uint): enables or disables new implementations:
    "1" - use new raidz implementation (DEFAULT),
    "0" - use old raidz implementation.

vdev_raidz_math: fix AVX2 code compilation against kernels older and newer than ~3.16
kernel comp: fix CPU feature check for new kernels
vdev_raidz_math_scalar: fix function prototype inconsistencies
autoconf: added m4 macros for userspace CPUID and SIMD toolchain support

cmd/zdb/zdb.c

@@ -42,6 +42,7 @@
 #include <sys/sa_impl.h>
 #include <sys/vdev.h>
 #include <sys/vdev_impl.h>
+#include <sys/vdev_raidz.h>
 #include <sys/metaslab_impl.h>
 #include <sys/dmu_objset.h>
 #include <sys/dsl_dir.h>

cmd/ztest/ztest.c

@@ -103,6 +103,7 @@
 #include <sys/zil_impl.h>
 #include <sys/vdev_impl.h>
 #include <sys/vdev_file.h>
+#include <sys/vdev_raidz.h>
 #include <sys/spa_impl.h>
 #include <sys/metaslab_impl.h>
 #include <sys/dsl_prop.h>

config/toolchain-simd.m4

@@ -0,0 +1,51 @@
+dnl #
+dnl # Checks whether toolchain supports SIMD instructions
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [
+
+	dnl # x86
+	AS_IF([test "t$host_cpu" = "tx86_64" -o "t$host_cpu" = "tx86" -o "t$host_cpu" = "ti686"],
+	[
+		ZFS_AC_CONFIG_TOOLCHIN_CAN_BUILD_SSSE3
+		ZFS_AC_CONFIG_TOOLCHIN_CAN_BUILD_AVX2
+	])
+
+	dnl # arm
+	AS_IF([test "t$host_cpu" = "taarch64"],
+	[
+
+	])
+])
+
+
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHIN_CAN_BUILD_SSSE3
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHIN_CAN_BUILD_SSSE3], [
+	AC_MSG_CHECKING([whether host toolchain supports SSSE3])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(){ __asm__ __volatile__("pshufb %xmm0, %xmm1"); return 0;}]])],
+	[
+		AC_DEFINE([HAVE_SSE], 1, [Define if host toolchain supports SSE])
+		AC_MSG_RESULT([yes])
+	],
+	[
+		AC_MSG_RESULT([no])
+	])
+])
+
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHIN_CAN_BUILD_AVX2
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHIN_CAN_BUILD_AVX2], [
+	AC_MSG_CHECKING([whether host toolchain supports AVX2])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(){ __asm__ __volatile__("vpshufb %ymm0, %ymm1, %ymm2"); return 0;}]])],
+	[
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_AVX2], 1, [Define if host toolchain supports AVX2])
+	],
+	[
+		AC_MSG_RESULT([no])
+	])
+])

config/user-cpuid.m4

@@ -0,0 +1,11 @@
+dnl #
+dnl # Checks whether <cpuid.h> exists
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_USER_CPUID], [
+
+	AC_CHECK_HEADER([cpuid.h],
+	[
+		AC_DEFINE([HAVE_CPUID_H], 1, [Define if host has <cpuid.h>])
+	],)
+
+])

config/user.m4

@@ -13,6 +13,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [
 	ZFS_AC_CONFIG_USER_LIBBLKID
 	ZFS_AC_CONFIG_USER_FRAME_LARGER_THAN
 	ZFS_AC_CONFIG_USER_RUNSTATEDIR
+	ZFS_AC_CONFIG_USER_CPUID
 dnl #
 dnl #	Checks for library functions
 	AC_CHECK_FUNCS([mlockall])

config/zfs-build.m4

@@ -63,6 +63,7 @@ AC_DEFUN([ZFS_AC_DEBUG_DMU_TX], [
 AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
 	ZFS_AC_CONFIG_ALWAYS_NO_UNUSED_BUT_SET_VARIABLE
 	ZFS_AC_CONFIG_ALWAYS_NO_BOOL_COMPARE
+	ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD
 ])
 
 AC_DEFUN([ZFS_AC_CONFIG], [
@@ -89,8 +90,8 @@ AC_DEFUN([ZFS_AC_CONFIG], [
 	case "$ZFS_CONFIG" in
 		kernel) ZFS_AC_CONFIG_KERNEL ;;
 		user)	ZFS_AC_CONFIG_USER   ;;
-		all)    ZFS_AC_CONFIG_KERNEL
-			ZFS_AC_CONFIG_USER   ;;
+		all)	ZFS_AC_CONFIG_USER
+				ZFS_AC_CONFIG_KERNEL ;;
 		srpm)                        ;;
 		*)
 		AC_MSG_RESULT([Error!])

include/sys/Makefile.am

@@ -73,6 +73,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/vdev_file.h \
 	$(top_srcdir)/include/sys/vdev.h \
 	$(top_srcdir)/include/sys/vdev_impl.h \
+	$(top_srcdir)/include/sys/vdev_raidz.h \
 	$(top_srcdir)/include/sys/xvattr.h \
 	$(top_srcdir)/include/sys/zap.h \
 	$(top_srcdir)/include/sys/zap_impl.h \

include/sys/vdev_raidz.h

@@ -0,0 +1,139 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (C) 2016 Gvozden Neskovic <[email protected]>.
+ */
+
+#ifndef _SYS_VDEV_RAIDZ_H
+#define	_SYS_VDEV_RAIDZ_H
+
+#include <sys/isa_defs.h>
+
+#if !defined(_KERNEL)
+#define RAIDZ_USERSPACE	1
+#endif
+
+#if defined(HAVE_SSE)
+#define	RAIDZ_SSE			1
+#if defined(__i386)
+#define	RAIDZ_SSE_x86		1
+#elif defined(__x86_64)
+#define	RAIDZ_SSE_x86_64	1
+#else
+#error "Unknown RAIDZ SSE implementation"
+#endif
+#endif /* defined(HAVE_SSE) */
+
+#if defined(HAVE_AVX2)
+#define	RAIDZ_AVX2			1
+#if defined(__i386)
+#define	RAIDZ_AVX2_x86		1
+#elif defined(__x86_64)
+#define	RAIDZ_AVX2_x86_64	1
+#else
+#error "Unknown RAIDZ AVX2 implementation"
+#endif
+#endif /* defined(HAVE_AVX2) */
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+void vdev_raidz_math_init(void);
+
+/* testing interface for userspace */
+#if !defined(_KERNEL)
+void vdev_raidz_cycle_impl(unsigned int v);
+#endif
+
+
+
+/* cpuid for userspace */
+#if defined(RAIDZ_USERSPACE)
+/* only for x86 and x86_64*/
+#if defined(__i686__) || defined(__i386) || defined(__x86_64__)
+
+static inline
+void cpuid(unsigned int *eax,
+							unsigned int *ebx,
+							unsigned int *ecx);
+
+#if defined(HAVE_CPUID_H)
+#include <cpuid.h>
+static inline
+void
+cpuid(unsigned int *eax,
+		unsigned int *ebx,
+		unsigned int *ecx)
+{
+	unsigned int edx;
+	*eax = 0, *ebx = 0, *ecx = 0, edx = 0;
+
+	__get_cpuid(1, eax, ebx, ecx, &edx);
+}
+#else /* !defined(HAVE_CPUID_H) */
+static inline
+void
+cpuid(unsigned int *eax,
+		unsigned int *ebx,
+		unsigned int *ecx)
+{
+	*eax = 0, *ebx = 0, *ecx = 0;
+	/* TODO: provide platform cpuid function (user-cpuid.m4) */
+}
+#endif /* defined(HAVE_CPUID_H) */
+
+static inline
+int cpuid_has_ssse3(void)
+{
+	static const unsigned int SSSE3_FLAG = (1 << 9);
+	unsigned int eax, ebx, ecx;
+
+	cpuid(&eax, &ebx, &ecx);
+
+	return (!!(ecx & SSSE3_FLAG));
+}
+
+static inline
+int cpuid_has_avx2(void)
+{
+	static const unsigned int AVX2_FLAG = 0x00000200U;
+	unsigned int eax, ebx, ecx;
+
+	cpuid(&eax, &ebx, &ecx);
+
+	return 	(
+				(eax == 7) &&
+				(!!(ebx & AVX2_FLAG))
+			);
+}
+#endif /* defined(__i686__) || defined(__i386) || defined(__x86_64__) */
+
+
+
+#endif /* defined(RAIDZ_USERSPACE) */
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _SYS_VDEV_RAIDZ_H */

lib/libzpool/Makefile.am

@@ -90,6 +90,10 @@ KERNEL_C = \
 	vdev_missing.c \
 	vdev_queue.c \
 	vdev_raidz.c \
+	vdev_raidz_math.c \
+	vdev_raidz_math_scalar.c \
+	vdev_raidz_math_sse.c \
+	vdev_raidz_math_avx2.c \
 	vdev_root.c \
 	zap.c \
 	zap_leaf.c \

module/zfs/Makefile.in

@@ -69,6 +69,8 @@ $(MODULE)-objs += vdev_mirror.o
 $(MODULE)-objs += vdev_missing.o
 $(MODULE)-objs += vdev_queue.o
 $(MODULE)-objs += vdev_raidz.o
+$(MODULE)-objs += vdev_raidz_math.o
+$(MODULE)-objs += vdev_raidz_math_scalar.o
 $(MODULE)-objs += vdev_root.o
 $(MODULE)-objs += zap.o
 $(MODULE)-objs += zap_leaf.o
@@ -107,3 +109,6 @@ $(MODULE)-objs += zrlock.o
 $(MODULE)-objs += zvol.o
 $(MODULE)-objs += dsl_destroy.o
 $(MODULE)-objs += dsl_userhold.o
+
+$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_sse.o
+$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_avx2.o