Implementation of AVX2 optimized Fletcher-4

New functionality: - Preserves existing scalar implementation. - Adds AVX2 optimized Fletcher-4 computation. - Fastest routines selected on module load (benchmark). - Test case for Fletcher-4 added to ztest. New zcommon module parameters: - zfs_fletcher_4_impl (str): selects the implementation to use. "fastest" - use the fastest version available "cycle" - cycle trough all available impl for ztest "scalar" - use the original version "avx2" - new AVX2 implementation if available Performance comparison (Intel i7 CPU, 1MB data buffers): - Scalar: 4216 MB/s - AVX2: 14499 MB/s See contents of `/sys/module/zcommon/parameters/zfs_fletcher_4_impl` to get list of supported values. If an implementation is not supported on the system, it will not be shown. Currently selected option is enclosed in `[]`. Signed-off-by: Jinshan Xiong <[email protected]> Signed-off-by: Andreas Dilger <[email protected]> Signed-off-by: Brian Behlendorf <[email protected]> Closes #4330
openzfs · Jun 2, 2016 · 1eeb456 · 1eeb456
1 parent 8fbbc6b
commit 1eeb456
Show file tree

Hide file tree

Showing 12 changed files with 588 additions and 69 deletions.
diff --git a/cmd/ztest/ztest.c b/cmd/ztest/ztest.c
@@ -123,6 +123,7 @@
 #include <ctype.h>
 #include <math.h>
 #include <sys/fs/zfs.h>
+#include <zfs_fletcher.h>
 #include <libnvpair.h>
 #ifdef __GLIBC__
 #include <execinfo.h> /* for backtrace() */
@@ -327,6 +328,7 @@ ztest_func_t ztest_vdev_aux_add_remove;
 ztest_func_t ztest_split_pool;
 ztest_func_t ztest_reguid;
 ztest_func_t ztest_spa_upgrade;
+ztest_func_t ztest_fletcher;
 
 uint64_t zopt_always = 0ULL * NANOSEC;		/* all the time */
 uint64_t zopt_incessant = 1ULL * NANOSEC / 10;	/* every 1/10 second */
@@ -372,6 +374,7 @@ ztest_info_t ztest_info[] = {
 	ZTI_INIT(ztest_vdev_LUN_growth, 1, &zopt_rarely),
 	ZTI_INIT(ztest_vdev_add_remove, 1, &ztest_opts.zo_vdevtime),
 	ZTI_INIT(ztest_vdev_aux_add_remove, 1, &ztest_opts.zo_vdevtime),
+	ZTI_INIT(ztest_fletcher, 1, &zopt_rarely),
 };
 
 #define	ZTEST_FUNCS	(sizeof (ztest_info) / sizeof (ztest_info_t))
@@ -5496,6 +5499,47 @@ ztest_spa_rename(ztest_ds_t *zd, uint64_t id)
 	(void) rw_unlock(&ztest_name_lock);
 }
 
+void
+ztest_fletcher(ztest_ds_t *zd, uint64_t id)
+{
+	hrtime_t end = gethrtime() + NANOSEC;
+
+	while (gethrtime() <= end) {
+		int run_count = 100;
+		void *buf;
+		uint32_t size;
+		int *ptr;
+		int i;
+		zio_cksum_t zc_ref;
+		zio_cksum_t zc_ref_byteswap;
+
+		size = ztest_random_blocksize();
+		buf = umem_alloc(size, UMEM_NOFAIL);
+
+		for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++)
+			*ptr = ztest_random(UINT_MAX);
+
+		VERIFY0(fletcher_4_impl_set("scalar"));
+		fletcher_4_native(buf, size, &zc_ref);
+		fletcher_4_byteswap(buf, size, &zc_ref_byteswap);
+
+		VERIFY0(fletcher_4_impl_set("cycle"));
+		while (run_count-- > 0) {
+			zio_cksum_t zc;
+			zio_cksum_t zc_byteswap;
+
+			fletcher_4_byteswap(buf, size, &zc_byteswap);
+			fletcher_4_native(buf, size, &zc);
+
+			VERIFY0(bcmp(&zc, &zc_ref, sizeof (zc)));
+			VERIFY0(bcmp(&zc_byteswap, &zc_ref_byteswap,
+			    sizeof (zc_byteswap)));
+		}
+
+		umem_free(buf, size);
+	}
+}
+
 static int
 ztest_check_path(char *path)
 {

diff --git a/include/sys/Makefile.am b/include/sys/Makefile.am
@@ -50,6 +50,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/space_reftree.h \
 	$(top_srcdir)/include/sys/spa.h \
 	$(top_srcdir)/include/sys/spa_impl.h \
+	$(top_srcdir)/include/sys/spa_checksum.h \
 	$(top_srcdir)/include/sys/trace.h \
 	$(top_srcdir)/include/sys/trace_acl.h \
 	$(top_srcdir)/include/sys/trace_arc.h \

diff --git a/include/sys/spa.h b/include/sys/spa.h
@@ -35,6 +35,7 @@
 #include <sys/sysmacros.h>
 #include <sys/types.h>
 #include <sys/fs/zfs.h>
+#include <sys/spa_checksum.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -142,12 +143,6 @@ typedef struct dva {
 	uint64_t	dva_word[2];
 } dva_t;
 
-/*
- * Each block has a 256-bit checksum -- strong enough for cryptographic hashes.
- */
-typedef struct zio_cksum {
-	uint64_t	zc_word[4];
-} zio_cksum_t;
 
 /*
  * Each block is described by its DVAs, time of birth, checksum, etc.
@@ -440,35 +435,9 @@ _NOTE(CONSTCOND) } while (0)
 	DVA_EQUAL(&(bp1)->blk_dva[1], &(bp2)->blk_dva[1]) &&	\
 	DVA_EQUAL(&(bp1)->blk_dva[2], &(bp2)->blk_dva[2]))
 
-#define	ZIO_CHECKSUM_EQUAL(zc1, zc2) \
-	(0 == (((zc1).zc_word[0] - (zc2).zc_word[0]) | \
-	((zc1).zc_word[1] - (zc2).zc_word[1]) | \
-	((zc1).zc_word[2] - (zc2).zc_word[2]) | \
-	((zc1).zc_word[3] - (zc2).zc_word[3])))
-
-#define	ZIO_CHECKSUM_IS_ZERO(zc) \
-	(0 == ((zc)->zc_word[0] | (zc)->zc_word[1] | \
-	(zc)->zc_word[2] | (zc)->zc_word[3]))
-
-#define	ZIO_CHECKSUM_BSWAP(zcp)					\
-{								\
-	(zcp)->zc_word[0] = BSWAP_64((zcp)->zc_word[0]);	\
-	(zcp)->zc_word[1] = BSWAP_64((zcp)->zc_word[1]);	\
-	(zcp)->zc_word[2] = BSWAP_64((zcp)->zc_word[2]);	\
-	(zcp)->zc_word[3] = BSWAP_64((zcp)->zc_word[3]);	\
-}
-
 
 #define	DVA_IS_VALID(dva)	(DVA_GET_ASIZE(dva) != 0)
 
-#define	ZIO_SET_CHECKSUM(zcp, w0, w1, w2, w3)	\
-{						\
-	(zcp)->zc_word[0] = w0;			\
-	(zcp)->zc_word[1] = w1;			\
-	(zcp)->zc_word[2] = w2;			\
-	(zcp)->zc_word[3] = w3;			\
-}
-
 #define	BP_IDENTITY(bp)		(ASSERT(!BP_IS_EMBEDDED(bp)), &(bp)->blk_dva[0])
 #define	BP_IS_GANG(bp)		\
 	(BP_IS_EMBEDDED(bp) ? B_FALSE : DVA_GET_GANG(BP_IDENTITY(bp)))

diff --git a/include/sys/spa_checksum.h b/include/sys/spa_checksum.h
@@ -0,0 +1,72 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_SPA_CHECKSUM_H
+#define	_SPA_CHECKSUM_H
+
+#include <sys/types.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/*
+ * Each block has a 256-bit checksum -- strong enough for cryptographic hashes.
+ */
+typedef struct zio_cksum {
+	uint64_t	zc_word[4];
+} zio_cksum_t;
+
+#define	ZIO_SET_CHECKSUM(zcp, w0, w1, w2, w3)	\
+{						\
+	(zcp)->zc_word[0] = w0;			\
+	(zcp)->zc_word[1] = w1;			\
+	(zcp)->zc_word[2] = w2;			\
+	(zcp)->zc_word[3] = w3;			\
+}
+
+#define	ZIO_CHECKSUM_EQUAL(zc1, zc2) \
+	(0 == (((zc1).zc_word[0] - (zc2).zc_word[0]) | \
+	((zc1).zc_word[1] - (zc2).zc_word[1]) | \
+	((zc1).zc_word[2] - (zc2).zc_word[2]) | \
+	((zc1).zc_word[3] - (zc2).zc_word[3])))
+
+#define	ZIO_CHECKSUM_IS_ZERO(zc) \
+	(0 == ((zc)->zc_word[0] | (zc)->zc_word[1] | \
+	(zc)->zc_word[2] | (zc)->zc_word[3]))
+
+#define	ZIO_CHECKSUM_BSWAP(zcp)					\
+{								\
+	(zcp)->zc_word[0] = BSWAP_64((zcp)->zc_word[0]);	\
+	(zcp)->zc_word[1] = BSWAP_64((zcp)->zc_word[1]);	\
+	(zcp)->zc_word[2] = BSWAP_64((zcp)->zc_word[2]);	\
+	(zcp)->zc_word[3] = BSWAP_64((zcp)->zc_word[3]);	\
+}
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif
diff --git a/include/zfs_fletcher.h b/include/zfs_fletcher.h
@@ -27,7 +27,7 @@
 #define	_ZFS_FLETCHER_H
 
 #include <sys/types.h>
-#include <sys/spa.h>
+#include <sys/spa_checksum.h>
 
 #ifdef	__cplusplus
 extern "C" {
@@ -45,6 +45,25 @@ void fletcher_4_incremental_native(const void *, uint64_t,
     zio_cksum_t *);
 void fletcher_4_incremental_byteswap(const void *, uint64_t,
     zio_cksum_t *);
+int fletcher_4_impl_set(const char *selector);
+void fletcher_4_init(void);
+void fletcher_4_fini(void);
+
+/*
+ * fletcher checksum struct
+ */
+typedef struct fletcher_4_func {
+	void (*init)(zio_cksum_t *);
+	void (*fini)(zio_cksum_t *);
+	void (*compute)(const void *, uint64_t, zio_cksum_t *);
+	void (*compute_byteswap)(const void *, uint64_t, zio_cksum_t *);
+	boolean_t (*valid)(void);
+	const char *name;
+} fletcher_4_ops_t;
+
+#if defined(HAVE_AVX) && defined(HAVE_AVX2)
+extern const fletcher_4_ops_t fletcher_4_avx2_ops;
+#endif
 
 #ifdef	__cplusplus
 }

diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am
@@ -22,6 +22,7 @@ KERNEL_C = \
 	zfs_comutil.c \
 	zfs_deleg.c \
 	zfs_fletcher.c \
+	zfs_fletcher_intel.c \
 	zfs_namecheck.c \
 	zfs_prop.c \
 	zfs_uio.c \

diff --git a/lib/libzpool/kernel.c b/lib/libzpool/kernel.c
@@ -40,6 +40,7 @@
 #include <sys/utsname.h>
 #include <sys/time.h>
 #include <sys/systeminfo.h>
+#include <zfs_fletcher.h>
 
 /*
  * Emulation of kernel services in userland.
@@ -1236,12 +1237,15 @@ kernel_init(int mode)
 
 	spa_init(mode);
 
+	fletcher_4_init();
+
 	tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
 }
 
 void
 kernel_fini(void)
 {
+	fletcher_4_fini();
 	spa_fini();
 
 	system_taskq_fini();

diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5
@@ -830,6 +830,23 @@ Start syncing out a transaction group if there is at least this much dirty data.
 Default value: \fB67,108,864\fR.
 .RE
 
+.sp
+.ne 2
+.na
+\fBzfs_fletcher_4_impl\fR (string)
+.ad
+.RS 12n
+Select a fletcher 4 implementation.
+.sp
+Supported selectors are: \fBfastest\fR, \fBscalar\fR, and \fBavx2\fR when
+AVX2 is supported by the processor.  If multiple implementations of fletcher 4
+are available the \fBfastest\fR will be chosen using a micro benchmark.
+Selecting \fBscalar\fR results in the original CPU based calculation being
+used, \fBavx2\fR uses the AVX2 vector instructions to compute a fletcher 4.
+.sp
+Default value: \fBfastest\fR.
+.RE
+
 .sp
 .ne 2
 .na

diff --git a/module/zcommon/Makefile.in b/module/zcommon/Makefile.in
@@ -15,3 +15,5 @@ $(MODULE)-objs += zfs_comutil.o
 $(MODULE)-objs += zfs_fletcher.o
 $(MODULE)-objs += zfs_uio.o
 $(MODULE)-objs += zpool_prop.o
+
+$(MODULE)-$(CONFIG_X86) += zfs_fletcher_intel.o