Fixes and enhancements of SIMD raidz parity

- Writer lock is now acquired when zfs_vdev_raidz_impl_set() is invoked for user
parameter change.

- Trailing whitespace is removed from user specified parameter, to enhance
experience when using commands that add newline, e.g. `echo`

- To enable better insight in what parity implementation is being used when
[fastest] option is selected for `zfs_vdev_raidz_impl` parameter, following
changes are made:
	- An unique ID is assigned to every physical parity method. Each parity
	implementation consists of 10 distinct parity methods.
	- Methods of the [fastest] implementation will have IDs that indicate
	physical implementation used (determined by a micro-benchmark).
	- List of all supported parity methods, with ID and relative performance,
	is available in `/proc/spl/kstat/zfs/vdev_raidz_bench`

- raidz_test: remove dependency on `getrusage()` and RUSAGE_THREAD, Issue openzfs#4813

- Minor fixes and cleanups

Signed-off-by: Gvozden Neskovic <[email protected]>

cmd/raidz_test/raidz_bench.c

@@ -32,7 +32,6 @@
 #include <stdio.h>
 
 #include <sys/time.h>
-#include <sys/resource.h>
 
 #include "raidz_test.h"
 
@@ -70,28 +69,18 @@ bench_fini_raidz_maps(void)
 	bzero(&zio_bench, sizeof (zio_t));
 }
 
-static double
-get_time_diff(struct rusage *start, struct rusage *stop)
-{
-	return (((double)stop->ru_utime.tv_sec * (double)MICROSEC +
-		(double)stop->ru_utime.tv_usec) -
-		((double)start->ru_utime.tv_sec * (double)MICROSEC +
-		(double)start->ru_utime.tv_usec)) / (double)MICROSEC;
-}
-
 static inline void
 run_gen_bench_impl(const char *impl)
 {
 	int fn, ncols;
 	uint64_t ds, iter_cnt, iter, disksize;
-	struct rusage start, stop;
+	hrtime_t start;
 	double elapsed, d_bw;
 
 	/* Benchmark generate functions */
 	for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) {
 
 		for (ds = MIN_CS_SHIFT; ds <= MAX_CS_SHIFT; ds++) {
-
 			/* create suitable raidz_map */
 			ncols = rto_opts.rto_dcols + fn + 1;
 			zio_bench.io_size = 1ULL << ds;
@@ -102,12 +91,11 @@ run_gen_bench_impl(const char *impl)
 			iter_cnt = GEN_BENCH_MEMORY;
 			iter_cnt /= zio_bench.io_size;
 
-			getrusage(RUSAGE_THREAD, &start);
+			start = gethrtime();
 			for (iter = 0; iter < iter_cnt; iter++)
 				vdev_raidz_generate_parity(rm_bench);
-			getrusage(RUSAGE_THREAD, &stop);
+			elapsed = NSEC2SEC((double) (gethrtime() - start));
 
-			elapsed = get_time_diff(&start, &stop);
 			disksize = (1ULL << ds) / rto_opts.rto_dcols;
 			d_bw = (double)iter_cnt * (double)disksize;
 			d_bw /= (1024.0 * 1024.0 * elapsed);
@@ -147,9 +135,9 @@ run_gen_bench(void)
 static void
 run_rec_bench_impl(const char *impl)
 {
-	struct rusage start, stop;
 	int fn, ncols, nbad;
 	uint64_t ds, iter_cnt, iter, disksize;
+	hrtime_t start;
 	double elapsed, d_bw;
 	static const int tgt[7][3] = {
 		{1, 2, 3},	/* rec_p:   bad QR & D[0]	*/
@@ -187,12 +175,11 @@ run_rec_bench_impl(const char *impl)
 			nbad = MIN(3, raidz_ncols(rm_bench) -
 			    raidz_parity(rm_bench));
 
-			getrusage(RUSAGE_THREAD, &start);
+			start = gethrtime();
 			for (iter = 0; iter < iter_cnt; iter++)
 				vdev_raidz_reconstruct(rm_bench, tgt[fn], nbad);
-			getrusage(RUSAGE_THREAD, &stop);
+			elapsed = NSEC2SEC((double) (gethrtime() - start));
 
-			elapsed = get_time_diff(&start, &stop);
 			disksize = (1ULL << ds) / rto_opts.rto_dcols;
 			d_bw = (double)iter_cnt * (double)(disksize);
 			d_bw /= (1024.0 * 1024.0 * elapsed);

include/sys/vdev_raidz.h

@@ -37,25 +37,28 @@ struct raidz_map;
 struct kernel_param {};
 #endif
 
+typedef struct raidz_map raidz_map_t;
+typedef struct raidz_impl_ops raidz_impl_ops_t;
+
 /*
  * vdev_raidz interface
  */
-struct raidz_map *	vdev_raidz_map_alloc(struct zio *, uint64_t, uint64_t,
-			    uint64_t);
+raidz_map_t *	vdev_raidz_map_alloc(struct zio *, uint64_t, uint64_t,
+		    uint64_t);
 void		vdev_raidz_map_free(struct raidz_map *);
 void 		vdev_raidz_generate_parity(struct raidz_map *);
 int 		vdev_raidz_reconstruct(struct raidz_map *, const int *, int);
 
 /*
  * vdev_raidz_math interface
  */
-void	vdev_raidz_math_init(void);
-void	vdev_raidz_math_fini(void);
-void	vdev_raidz_math_get_ops(struct raidz_map *);
-void	vdev_raidz_math_generate(struct raidz_map *);
-int	vdev_raidz_math_reconstruct(struct raidz_map *, const int *,
-	    const int *, const int);
-int	vdev_raidz_impl_set(const char *);
+void			vdev_raidz_math_init(void);
+void			vdev_raidz_math_fini(void);
+raidz_impl_ops_t *	vdev_raidz_math_get_ops(void);
+void			vdev_raidz_math_generate(struct raidz_map *);
+int			vdev_raidz_math_reconstruct(struct raidz_map *,
+			    const int *, const int *, const int);
+int			vdev_raidz_impl_set(const char *);
 
 #ifdef	__cplusplus
 }

include/sys/vdev_raidz_impl.h

@@ -89,13 +89,15 @@ typedef boolean_t	(*will_work_f)(void);
 typedef void		(*init_impl_f)(void);
 typedef void		(*fini_impl_f)(void);
 
+#define	RAIDZ_IMPL_NAME_MAX	(16)
+
 typedef struct raidz_impl_ops {
 	init_impl_f init;
 	fini_impl_f fini;
 	raidz_gen_f gen[RAIDZ_GEN_NUM];	/* Parity generate functions */
 	raidz_rec_f rec[RAIDZ_REC_NUM];	/* Data reconstruction functions */
 	will_work_f is_supported;	/* Support check function */
-	char *name;			/* Name of the implementation */
+	char name[RAIDZ_IMPL_NAME_MAX];	/* Name of the implementation */
 } raidz_impl_ops_t;
 
 typedef struct raidz_col {
@@ -227,12 +229,18 @@ impl ## _rec_ ## code(void *rmp, const int *tgtidx)			\
 	[RAIDZ_REC_PQR] = & impl ## _rec_pqr				\
 }
 
+#define	RAIDZ_IMPL_KSTAT_CNT	(2 * (RAIDZ_GEN_NUM + RAIDZ_REC_NUM))
 
 typedef struct raidz_impl_kstat {
-	kstat_named_t gen[RAIDZ_GEN_NUM];	/* gen method speed kiB/s */
-	kstat_named_t rec[RAIDZ_REC_NUM];	/* rec method speed kiB/s */
+	kstat_named_t gen[2 * RAIDZ_GEN_NUM];	/* gen method speed kiB/s */
+	kstat_named_t rec[2 * RAIDZ_REC_NUM];	/* rec method speed kiB/s */
 } raidz_impl_kstat_t;
 
+#define	RAIDZ_IMPL_GEN_ID(ks, fn)	(ks)->gen[2 * (fn)].value.ui64
+#define	RAIDZ_IMPL_GEN_BW(ks, fn)	(ks)->gen[2 * (fn) + 1].value.ui64
+#define	RAIDZ_IMPL_REC_ID(ks, fn)	(ks)->rec[2 * (fn)].value.ui64
+#define	RAIDZ_IMPL_REC_BW(ks, fn)	(ks)->rec[2 * (fn) + 1].value.ui64
+
 /*
  * Enumerate various multiplication constants
  * used in reconstruction methods

man/man5/zfs-module-parameters.5

@@ -1680,7 +1680,7 @@ Default value: \fB4,096\fR.
 \fBzfs_vdev_raidz_impl\fR (string)
 .ad
 .RS 12n
-Parameter for selecting raidz implementation to use.
+Parameter for selecting raidz parity implementation to use.
 
 Options marked (always) below may be selected on module load as they are
 supported on all systems.

module/zfs/vdev_raidz.c

@@ -458,8 +458,8 @@ vdev_raidz_map_alloc(zio_t *zio, uint64_t unit_shift, uint64_t dcols,
 	zio->io_vsd = rm;
 	zio->io_vsd_ops = &vdev_raidz_vsd_ops;
 
-	/* RAIDZ ops init */
-	vdev_raidz_math_get_ops(rm);
+	/* init RAIDZ parity ops */
+	rm->rm_ops = vdev_raidz_math_get_ops();
 
 	return (rm);
 }