OpenZFS 8484 - Implement aggregate sum and use for arc counters

In pursuit of improving performance on multi-core systems, we should
implements fanned out counters and use them to improve the performance of
some of the arc statistics. These stats are updated extremely frequently,
and can consume a significant amount of CPU time.

Authored by: Paul Dagnelie <[email protected]>
Reviewed by: Pavel Zakharov <[email protected]>
Reviewed by: Matthew Ahrens <[email protected]>
Approved by: Dan McDonald <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Ported-by: Paul Dagnelie <[email protected]>

OpenZFS-issue: https://www.illumos.org/issues/8484
OpenZFS-commit: openzfs/openzfs@7028a8b92b7
Issue openzfs#3752
Closes openzfs#7462

include/sys/Makefile.am

@@ -2,6 +2,7 @@ SUBDIRS = fm fs crypto lua sysevent
 
 COMMON_H = \
 	$(top_srcdir)/include/sys/abd.h \
+	$(top_srcdir)/include/sys/aggsum.h \
 	$(top_srcdir)/include/sys/arc.h \
 	$(top_srcdir)/include/sys/arc_impl.h \
 	$(top_srcdir)/include/sys/avl.h \
@@ -11,6 +12,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/bpobj.h \
 	$(top_srcdir)/include/sys/bptree.h \
 	$(top_srcdir)/include/sys/bqueue.h \
+	$(top_srcdir)/include/sys/cityhash.h \
 	$(top_srcdir)/include/sys/dbuf.h \
 	$(top_srcdir)/include/sys/ddt.h \
 	$(top_srcdir)/include/sys/dmu.h \

include/sys/aggsum.h

@@ -0,0 +1,59 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2017 by Delphix. All rights reserved.
+ */
+
+#ifndef	_SYS_AGGSUM_H
+#define	_SYS_AGGSUM_H
+
+#include <sys/zfs_context.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+typedef struct aggsum_bucket aggsum_bucket_t;
+
+struct aggsum_bucket {
+	kmutex_t asc_lock;
+	int64_t asc_delta;
+	uint64_t asc_borrowed;
+} ____cacheline_aligned;
+
+/*
+ * Fan out over FANOUT cpus.
+ */
+typedef struct aggsum {
+	kmutex_t as_lock;
+	int64_t as_lower_bound;
+	int64_t as_upper_bound;
+	uint64_t as_numbuckets;
+	aggsum_bucket_t *as_buckets;
+} aggsum_t;
+
+void aggsum_init(aggsum_t *, uint64_t);
+void aggsum_fini(aggsum_t *);
+int64_t aggsum_lower_bound(aggsum_t *);
+int64_t aggsum_upper_bound(aggsum_t *);
+int aggsum_compare(aggsum_t *, uint64_t);
+uint64_t aggsum_value(aggsum_t *);
+void aggsum_add(aggsum_t *, int64_t);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _SYS_AGGSUM_H */

include/sys/cityhash.h

@@ -0,0 +1,41 @@
+// Copyright (c) 2011 Google, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+
+/*
+ * Copyright (c) 2017 by Delphix. All rights reserved.
+ */
+
+#ifndef	_SYS_CITYHASH_H
+#define	_SYS_CITYHASH_H
+
+#include <sys/zfs_context.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+uint64_t cityhash4(uint64_t, uint64_t, uint64_t, uint64_t);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_CITYHASH_H */

include/sys/zfs_context.h

@@ -750,5 +750,7 @@ extern fstrans_cookie_t spl_fstrans_mark(void);
 extern void spl_fstrans_unmark(fstrans_cookie_t);
 extern int __spl_pf_fstrans_check(void);
 
+#define	____cacheline_aligned
+
 #endif /* _KERNEL */
 #endif	/* _SYS_ZFS_CONTEXT_H */

lib/libzpool/Makefile.am

@@ -40,12 +40,14 @@ KERNEL_C = \
 	zpool_prop.c \
 	zprop_common.c \
 	abd.c \
+	aggsum.c \
 	arc.c \
 	blkptr.c \
 	bplist.c \
 	bpobj.c \
 	bptree.c \
 	bqueue.c \
+	cityhash.c \
 	dbuf.c \
 	dbuf_stats.c \
 	ddt.c \

module/zfs/Makefile.in

@@ -17,10 +17,12 @@ endif
 ccflags-y += $(NO_UNUSED_BUT_SET_VARIABLE)
 
 $(MODULE)-objs += abd.o
+$(MODULE)-objs += aggsum.o
 $(MODULE)-objs += arc.o
 $(MODULE)-objs += blkptr.o
 $(MODULE)-objs += bplist.o
 $(MODULE)-objs += bpobj.o
+$(MODULE)-objs += cityhash.o
 $(MODULE)-objs += dbuf.o
 $(MODULE)-objs += dbuf_stats.o
 $(MODULE)-objs += bptree.o

module/zfs/THIRDPARTYLICENSE.cityhash

@@ -0,0 +1,19 @@
+Copyright (c) 2011 Google, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

module/zfs/THIRDPARTYLICENSE.cityhash.descrip

@@ -0,0 +1 @@
+CITYHASH CHECKSUM FUNCTIONALITY IN ZFS