Combine OS-independent ABD Code into Common Source File

Reorganizing ABD code base so OS-independent ABD code has been placed
into a common abd.c file. OS-dependent ABD code has been left in each
OS's ABD source files, and these source files have been renamed to
abd_os.

The OS-independent ABD code is now under:
module/zfs/abd.c
With the OS-dependent code in:
module/os/linux/zfs/abd_os.c
module/os/freebsd/zfs/abd_os.c

Reviewed-by: Matthew Ahrens <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Brian Atkinson <[email protected]>
Closes #10293

include/sys/Makefile.am

@@ -2,6 +2,7 @@ SUBDIRS = fm fs crypto lua sysevent
 
 COMMON_H = \
 	$(top_srcdir)/include/sys/abd.h \
+	$(top_srcdir)/include/sys/abd_impl.h \
 	$(top_srcdir)/include/sys/aggsum.h \
 	$(top_srcdir)/include/sys/arc.h \
 	$(top_srcdir)/include/sys/arc_impl.h \

include/sys/abd.h

@@ -35,56 +35,14 @@
 extern "C" {
 #endif
 
-typedef enum abd_flags {
-	ABD_FLAG_LINEAR	= 1 << 0,	/* is buffer linear (or scattered)? */
-	ABD_FLAG_OWNER	= 1 << 1,	/* does it own its data buffers? */
-	ABD_FLAG_META	= 1 << 2,	/* does this represent FS metadata? */
-	ABD_FLAG_MULTI_ZONE  = 1 << 3,	/* pages split over memory zones */
-	ABD_FLAG_MULTI_CHUNK = 1 << 4,	/* pages split over multiple chunks */
-	ABD_FLAG_LINEAR_PAGE = 1 << 5,	/* linear but allocd from page */
-} abd_flags_t;
-
-typedef struct abd {
-	abd_flags_t	abd_flags;
-	uint_t		abd_size;	/* excludes scattered abd_offset */
-	struct abd	*abd_parent;
-	zfs_refcount_t	abd_children;
-	union {
-		struct abd_scatter {
-			uint_t		abd_offset;
-#if defined(__FreeBSD__) && defined(_KERNEL)
-			uint_t  abd_chunk_size;
-			void    *abd_chunks[];
-#else
-			uint_t		abd_nents;
-			struct scatterlist *abd_sgl;
-#endif
-		} abd_scatter;
-		struct abd_linear {
-			void		*abd_buf;
-			struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
-		} abd_linear;
-	} abd_u;
-} abd_t;
+struct abd; /* forward declaration */
+typedef struct abd abd_t;
 
 typedef int abd_iter_func_t(void *buf, size_t len, void *private);
 typedef int abd_iter_func2_t(void *bufa, void *bufb, size_t len, void *private);
 
 extern int zfs_abd_scatter_enabled;
 
-static inline boolean_t
-abd_is_linear(abd_t *abd)
-{
-	return ((abd->abd_flags & ABD_FLAG_LINEAR) != 0 ? B_TRUE : B_FALSE);
-}
-
-static inline boolean_t
-abd_is_linear_page(abd_t *abd)
-{
-	return ((abd->abd_flags & ABD_FLAG_LINEAR_PAGE) != 0 ?
-	    B_TRUE : B_FALSE);
-}
-
 /*
  * Allocations and deallocations
  */
@@ -124,12 +82,8 @@ void abd_copy_to_buf_off(void *, abd_t *, size_t, size_t);
 int abd_cmp(abd_t *, abd_t *);
 int abd_cmp_buf_off(abd_t *, const void *, size_t, size_t);
 void abd_zero_off(abd_t *, size_t, size_t);
-
-#if defined(_KERNEL)
-unsigned int abd_scatter_bio_map_off(struct bio *, abd_t *, unsigned int,
-		size_t);
-unsigned long abd_nr_pages_off(abd_t *, unsigned int, size_t);
-#endif
+void abd_verify(abd_t *);
+uint_t abd_get_size(abd_t *);
 
 void abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
 	ssize_t csize, ssize_t dsize, const unsigned parity,
@@ -174,13 +128,29 @@ abd_zero(abd_t *abd, size_t size)
 	abd_zero_off(abd, 0, size);
 }
 
+/*
+ * ABD type check functions
+ */
+boolean_t abd_is_linear(abd_t *);
+boolean_t abd_is_linear_page(abd_t *);
+
 /*
  * Module lifecycle
+ * Defined in each specific OS's abd_os.c
  */
 
 void abd_init(void);
 void abd_fini(void);
 
+/*
+ * Linux ABD bio functions
+ */
+#if defined(__linux__) && defined(_KERNEL)
+unsigned int abd_scatter_bio_map_off(struct bio *, abd_t *, unsigned int,
+    size_t);
+unsigned long abd_nr_pages_off(abd_t *, unsigned int, size_t);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

include/sys/abd_impl.h

@@ -0,0 +1,126 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
+ * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
+ */
+
+#ifndef _ABD_IMPL_H
+#define	_ABD_IMPL_H
+
+#include <sys/abd.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum abd_flags {
+	ABD_FLAG_LINEAR		= 1 << 0, /* is buffer linear (or scattered)? */
+	ABD_FLAG_OWNER		= 1 << 1, /* does it own its data buffers? */
+	ABD_FLAG_META		= 1 << 2, /* does this represent FS metadata? */
+	ABD_FLAG_MULTI_ZONE  	= 1 << 3, /* pages split over memory zones */
+	ABD_FLAG_MULTI_CHUNK 	= 1 << 4, /* pages split over multiple chunks */
+	ABD_FLAG_LINEAR_PAGE 	= 1 << 5, /* linear but allocd from page */
+} abd_flags_t;
+
+typedef enum abd_stats_op {
+	ABDSTAT_INCR, /* Increase abdstat values */
+	ABDSTAT_DECR  /* Decrease abdstat values */
+} abd_stats_op_t;
+
+struct abd {
+	abd_flags_t	abd_flags;
+	uint_t		abd_size;	/* excludes scattered abd_offset */
+	struct abd	*abd_parent;
+	zfs_refcount_t	abd_children;
+	union {
+		struct abd_scatter {
+			uint_t		abd_offset;
+#if defined(__FreeBSD__) && defined(_KERNEL)
+			uint_t  abd_chunk_size;
+			void    *abd_chunks[];
+#else
+			uint_t		abd_nents;
+			struct scatterlist *abd_sgl;
+#endif
+		} abd_scatter;
+		struct abd_linear {
+			void		*abd_buf;
+			struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
+		} abd_linear;
+	} abd_u;
+};
+
+struct scatterlist; /* forward declaration */
+
+struct abd_iter {
+	/* public interface */
+	void		*iter_mapaddr;	/* addr corresponding to iter_pos */
+	size_t		iter_mapsize;	/* length of data valid at mapaddr */
+
+	/* private */
+	abd_t		*iter_abd;	/* ABD being iterated through */
+	size_t		iter_pos;
+	size_t		iter_offset;	/* offset in current sg/abd_buf, */
+					/* abd_offset included */
+	struct scatterlist *iter_sg;	/* current sg */
+};
+
+/*
+ * OS specific functions
+ */
+
+abd_t *abd_alloc_struct(size_t);
+abd_t *abd_get_offset_scatter(abd_t *, size_t);
+void abd_free_struct(abd_t *);
+void abd_alloc_chunks(abd_t *, size_t);
+void abd_free_chunks(abd_t *);
+boolean_t abd_size_alloc_linear(size_t);
+void abd_update_scatter_stats(abd_t *, abd_stats_op_t);
+void abd_update_linear_stats(abd_t *, abd_stats_op_t);
+void abd_verify_scatter(abd_t *);
+void abd_free_linear_page(abd_t *);
+void abd_enter_critical(unsigned long);
+void abd_exit_critical(unsigned long);
+/* OS specific abd_iter functions */
+void abd_iter_init(struct abd_iter  *, abd_t *);
+boolean_t abd_iter_at_end(struct abd_iter *);
+void abd_iter_advance(struct abd_iter *, size_t);
+void abd_iter_map(struct abd_iter *);
+void abd_iter_unmap(struct abd_iter *);
+
+/*
+ * Helper macros
+ */
+#define	ABDSTAT(stat)		(abd_stats.stat.value.ui64)
+#define	ABDSTAT_INCR(stat, val) \
+	atomic_add_64(&abd_stats.stat.value.ui64, (val))
+#define	ABDSTAT_BUMP(stat)	ABDSTAT_INCR(stat, 1)
+#define	ABDSTAT_BUMPDOWN(stat)	ABDSTAT_INCR(stat, -1)
+
+#define	ABD_SCATTER(abd)	(abd->abd_u.abd_scatter)
+#define	ABD_LINEAR_BUF(abd)	(abd->abd_u.abd_linear.abd_buf)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _ABD_IMPL_H */

lib/libzpool/Makefile.am

@@ -39,6 +39,7 @@ KERNEL_C = \
 	zpool_prop.c \
 	zprop_common.c \
 	abd.c \
+	abd_os.c \
 	aggsum.c \
 	arc.c \
 	arc_os.c \

module/Makefile.bsd

@@ -127,7 +127,7 @@ SRCS+= spl_atomic.c
 .endif
 
 #os/freebsd/zfs
-SRCS+=	abd.c \
+SRCS+=	abd_os.c \
 	crypto_os.c \
 	dmu_os.c \
 	hkdf.c \
@@ -169,7 +169,8 @@ SRCS+=	zfeature_common.c \
 	zprop_common.c
 
 #zfs
-SRCS+=	aggsum.c \
+SRCS+=	abd.c \
+	aggsum.c \
 	arc.c \
 	arc_os.c \
 	blkptr.c \