Separating ABD into OS Specific Code

Reorganizing ABD code base so the OS
dependant ABD code has been split into
their own independent files.
The shared ABD code is now under:
module/zfs/abd.c
With the independent OS code in:
module/os/linux/zfs/abd_os.c
module/os/freebsd/zfs/abd_os.c

Signed-off-by: Brian Atkinson <[email protected]>
Authored-by: Brian Atkinson <[email protected]>

include/sys/Makefile.am

@@ -2,6 +2,7 @@ SUBDIRS = fm fs crypto lua sysevent
 
 COMMON_H = \
 	$(top_srcdir)/include/sys/abd.h \
+	$(top_srcdir)/include/sys/abd_impl.h \
 	$(top_srcdir)/include/sys/aggsum.h \
 	$(top_srcdir)/include/sys/arc.h \
 	$(top_srcdir)/include/sys/arc_impl.h \

include/sys/abd.h

@@ -35,56 +35,14 @@
 extern "C" {
 #endif
 
-typedef enum abd_flags {
-	ABD_FLAG_LINEAR	= 1 << 0,	/* is buffer linear (or scattered)? */
-	ABD_FLAG_OWNER	= 1 << 1,	/* does it own its data buffers? */
-	ABD_FLAG_META	= 1 << 2,	/* does this represent FS metadata? */
-	ABD_FLAG_MULTI_ZONE  = 1 << 3,	/* pages split over memory zones */
-	ABD_FLAG_MULTI_CHUNK = 1 << 4,	/* pages split over multiple chunks */
-	ABD_FLAG_LINEAR_PAGE = 1 << 5,	/* linear but allocd from page */
-} abd_flags_t;
-
-typedef struct abd {
-	abd_flags_t	abd_flags;
-	uint_t		abd_size;	/* excludes scattered abd_offset */
-	struct abd	*abd_parent;
-	zfs_refcount_t	abd_children;
-	union {
-		struct abd_scatter {
-			uint_t		abd_offset;
-#if defined(__FreeBSD__) && defined(_KERNEL)
-			uint_t  abd_chunk_size;
-			void    *abd_chunks[];
-#else
-			uint_t		abd_nents;
-			struct scatterlist *abd_sgl;
-#endif
-		} abd_scatter;
-		struct abd_linear {
-			void		*abd_buf;
-			struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
-		} abd_linear;
-	} abd_u;
-} abd_t;
+struct abd; /* forward declaration */
+typedef struct abd abd_t;
 
 typedef int abd_iter_func_t(void *buf, size_t len, void *private);
 typedef int abd_iter_func2_t(void *bufa, void *bufb, size_t len, void *private);
 
 extern int zfs_abd_scatter_enabled;
 
-static inline boolean_t
-abd_is_linear(abd_t *abd)
-{
-	return ((abd->abd_flags & ABD_FLAG_LINEAR) != 0 ? B_TRUE : B_FALSE);
-}
-
-static inline boolean_t
-abd_is_linear_page(abd_t *abd)
-{
-	return ((abd->abd_flags & ABD_FLAG_LINEAR_PAGE) != 0 ?
-	    B_TRUE : B_FALSE);
-}
-
 /*
  * Allocations and deallocations
  */
@@ -124,12 +82,8 @@ void abd_copy_to_buf_off(void *, abd_t *, size_t, size_t);
 int abd_cmp(abd_t *, abd_t *);
 int abd_cmp_buf_off(abd_t *, const void *, size_t, size_t);
 void abd_zero_off(abd_t *, size_t, size_t);
-
-#if defined(_KERNEL)
-unsigned int abd_scatter_bio_map_off(struct bio *, abd_t *, unsigned int,
-		size_t);
-unsigned long abd_nr_pages_off(abd_t *, unsigned int, size_t);
-#endif
+void abd_verify(abd_t *);
+uint_t abd_get_size(abd_t *);
 
 void abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
 	ssize_t csize, ssize_t dsize, const unsigned parity,
@@ -174,13 +128,29 @@ abd_zero(abd_t *abd, size_t size)
 	abd_zero_off(abd, 0, size);
 }
 
+/*
+ * ABD type check functions
+ */
+boolean_t abd_is_linear(abd_t *);
+boolean_t abd_is_linear_page(abd_t *);
+
 /*
  * Module lifecycle
+ * Defined in each specific OS's abd.c
  */
 
 void abd_init(void);
 void abd_fini(void);
 
+/*
+ * Linux ABD bio functions
+ */
+#if defined(__linux__) && defined(_KERNEL)
+unsigned int abd_scatter_bio_map_off(struct bio *, abd_t *, unsigned int,
+    size_t);
+unsigned long abd_nr_pages_off(abd_t *, unsigned int, size_t);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

include/sys/abd_impl.h

@@ -0,0 +1,128 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
+ * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
+ */
+
+#ifndef _ABD_IMPL_H
+#define	_ABD_IMPL_H
+
+#include <sys/abd.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum abd_flags {
+	ABD_FLAG_LINEAR		= 1 << 0, /* is buffer linear (or scattered)? */
+	ABD_FLAG_OWNER		= 1 << 1, /* does it own its data buffers? */
+	ABD_FLAG_META		= 1 << 2, /* does this represent FS metadata? */
+	ABD_FLAG_MULTI_ZONE  	= 1 << 3, /* pages split over memory zones */
+	ABD_FLAG_MULTI_CHUNK 	= 1 << 4, /* pages split over multiple chunks */
+	ABD_FLAG_LINEAR_PAGE 	= 1 << 5, /* linear but allocd from page */
+} abd_flags_t;
+
+typedef enum abd_stats_op {
+	ABDSTAT_INCR, /* Increase abdstat values */
+	ABDSTAT_DECR  /* Decrease abdstat values */
+} abd_stats_op_t;
+
+struct abd {
+	abd_flags_t	abd_flags;
+	uint_t		abd_size;	/* excludes scattered abd_offset */
+	struct abd	*abd_parent;
+	zfs_refcount_t	abd_children;
+	union {
+		struct abd_scatter {
+			uint_t		abd_offset;
+#if defined(__FreeBSD__) && defined(_KERNEL)
+			uint_t  abd_chunk_size;
+			void    *abd_chunks[];
+#else
+			uint_t		abd_nents;
+			struct scatterlist *abd_sgl;
+#endif
+		} abd_scatter;
+		struct abd_linear {
+			void		*abd_buf;
+			struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
+		} abd_linear;
+	} abd_u;
+};
+
+struct scatterlist; /* forward declaration */
+
+struct abd_iter {
+	/* public interface */
+	void		*iter_mapaddr;	/* addr corresponding to iter_pos */
+	size_t		iter_mapsize;	/* length of data valid at mapaddr */
+
+	/* private */
+	abd_t		*iter_abd;	/* ABD being iterated through */
+	size_t		iter_pos;
+	size_t		iter_offset;	/* offset in current sg/abd_buf, */
+					/* abd_offset included */
+	struct scatterlist *iter_sg;	/* current sg */
+};
+
+/*
+ * OS specific functions
+ */
+
+size_t abd_chunkcnt_for_bytes(size_t);
+abd_t *abd_alloc_struct(size_t);
+abd_t *abd_get_offset_scatter(abd_t *, size_t);
+void abd_free_struct(abd_t *);
+void abd_alloc_pages(abd_t *, size_t);
+void abd_free_pages(abd_t *);
+boolean_t abd_size_alloc_linear(size_t);
+void abd_update_scatter_stats(abd_t *, abd_stats_op_t);
+void abd_update_linear_stats(abd_t *, abd_stats_op_t);
+void abd_verify_scatter(abd_t *);
+void abd_free_linear_page(abd_t *);
+void abd_enter_critical(unsigned long);
+void abd_exit_critical(unsigned long);
+/* OS specific abd_iter functions */
+void abd_iter_init(struct abd_iter  *, abd_t *);
+boolean_t abd_iter_at_end(struct abd_iter *);
+void abd_iter_advance(struct abd_iter *, size_t);
+void abd_iter_map(struct abd_iter *);
+void abd_iter_unmap(struct abd_iter *);
+
+/*
+ * Helper macros
+ */
+#define	ABDSTAT(stat)		(abd_stats.stat.value.ui64)
+#define	ABDSTAT_INCR(stat, val) \
+	atomic_add_64(&abd_stats.stat.value.ui64, (val))
+#define	ABDSTAT_BUMP(stat)	ABDSTAT_INCR(stat, 1)
+#define	ABDSTAT_BUMPDOWN(stat)	ABDSTAT_INCR(stat, -1)
+
+#define	ABD_SCATTER(abd)	(abd->abd_u.abd_scatter)
+#define	ABD_LINEAR_BUF(abd)	(abd->abd_u.abd_linear.abd_buf)
+#define	ABD_MULTI(abd)		(abd->abd_u.abd_multi)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _ABD_IMPL_H */

lib/libzpool/Makefile.am

@@ -39,6 +39,7 @@ KERNEL_C = \
 	zpool_prop.c \
 	zprop_common.c \
 	abd.c \
+	abd_os.c \
 	aggsum.c \
 	arc.c \
 	arc_os.c \

module/Makefile.bsd

@@ -127,7 +127,7 @@ SRCS+= spl_atomic.c
 .endif
 
 #os/freebsd/zfs
-SRCS+=	abd.c \
+SRCS+=	abd_os.c \
 	crypto_os.c \
 	dmu_os.c \
 	hkdf.c \
@@ -169,7 +169,8 @@ SRCS+=	zfeature_common.c \
 	zprop_common.c
 
 #zfs
-SRCS+=	aggsum.c \
+SRCS+=	abd.c \
+	aggsum.c \
 	arc.c \
 	arc_os.c \
 	blkptr.c \