Add SHA2 SIMD feature tests for Linux

These are added: - zfs_neon_available() for arm and aarch64 - zfs_sha256_available() for arm and aarch64 - zfs_sha512_available() for aarch64 - zfs_shani_available() for x86_64 Tested-by: Rich Ercolani <[email protected]> Tested-by: Sebastian Gottschall <[email protected]> Reviewed-by: Brian Behlendorf <[email protected]> Signed-off-by: Tino Reichardt <[email protected]> Co-Authored-By: Sebastian Gottschall <[email protected]> Closes openzfs#13741
pcd1193182 · Mar 8, 2023 · add2e9b · add2e9b
1 parent 0b58c9c
commit add2e9b
Show file tree

Hide file tree

Showing 6 changed files with 169 additions and 13 deletions.
diff --git a/include/os/linux/Makefile.am b/include/os/linux/Makefile.am
@@ -10,6 +10,7 @@ kernel_linux_HEADERS = \
 	%D%/kernel/linux/percpu_compat.h \
 	%D%/kernel/linux/simd.h \
 	%D%/kernel/linux/simd_aarch64.h \
+	%D%/kernel/linux/simd_arm.h \
 	%D%/kernel/linux/simd_powerpc.h \
 	%D%/kernel/linux/simd_x86.h \
 	%D%/kernel/linux/utsname_compat.h \

diff --git a/include/os/linux/kernel/linux/simd.h b/include/os/linux/kernel/linux/simd.h
@@ -28,13 +28,16 @@
 #if defined(__x86)
 #include <linux/simd_x86.h>
 
+#elif defined(__arm__)
+#include <linux/simd_arm.h>
+
 #elif defined(__aarch64__)
 #include <linux/simd_aarch64.h>
 
 #elif defined(__powerpc__)
 #include <linux/simd_powerpc.h>
-#else
 
+#else
 #define	kfpu_allowed()		0
 #define	kfpu_begin()		do {} while (0)
 #define	kfpu_end()		do {} while (0)

diff --git a/include/os/linux/kernel/linux/simd_aarch64.h b/include/os/linux/kernel/linux/simd_aarch64.h
@@ -18,8 +18,11 @@
  *
  * CDDL HEADER END
  */
+
 /*
  * Copyright (C) 2016 Romain Dolbeau <[email protected]>.
+ * Copyright (C) 2022 Tino Reichardt <[email protected]>
+ * Copyright (C) 2022 Sebastian Gottschall <[email protected]>
  */
 
 /*
@@ -31,24 +34,83 @@
  *	kfpu_end()
  *	kfpu_init()
  *	kfpu_fini()
+ *
+ * SIMD support:
+ *
+ * Following functions should be called to determine whether CPU feature
+ * is supported. All functions are usable in kernel and user space.
+ * If a SIMD algorithm is using more than one instruction set
+ * all relevant feature test functions should be called.
+ *
+ * Supported features:
+ *   zfs_neon_available()
+ *   zfs_sha256_available()
+ *   zfs_sha512_available()
  */
 
 #ifndef _LINUX_SIMD_AARCH64_H
 #define	_LINUX_SIMD_AARCH64_H
 
-#include <sys/isa_defs.h>
-
-#if defined(__aarch64__)
-
 #include <sys/types.h>
 #include <asm/neon.h>
+#include <asm/elf.h>
+#include <asm/hwcap.h>
+#include <linux/version.h>
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
+#include <asm/sysreg.h>
+#else
+#define	sys_reg(op0, op1, crn, crm, op2) ( \
+	((op0) << Op0_shift) | \
+	((op1) << Op1_shift) | \
+	((crn) << CRn_shift) | \
+	((crm) << CRm_shift) | \
+	((op2) << Op2_shift))
+#endif
+
+#define	ID_AA64PFR0_EL1		sys_reg(3, 0, 0, 1, 0)
+#define	ID_AA64ISAR0_EL1	sys_reg(3, 0, 0, 6, 0)
 
 #define	kfpu_allowed()		1
 #define	kfpu_begin()		kernel_neon_begin()
 #define	kfpu_end()		kernel_neon_end()
-#define	kfpu_init()		0
-#define	kfpu_fini()		((void) 0)
+#define	kfpu_init()		(0)
+#define	kfpu_fini()		do {} while (0)
+
+#define	get_ftr(id) {				\
+	unsigned long __val;			\
+	asm("mrs %0, "#id : "=r" (__val));	\
+	__val;					\
+}
 
-#endif /* __aarch64__ */
+/*
+ * Check if NEON is available
+ */
+static inline boolean_t
+zfs_neon_available(void)
+{
+	unsigned long ftr = ((get_ftr(ID_AA64PFR0_EL1)) >> 16) & 0xf;
+	return (ftr == 0 || ftr == 1);
+}
+
+/*
+ * Check if SHA256 is available
+ */
+static inline boolean_t
+zfs_sha256_available(void)
+{
+	unsigned long ftr = ((get_ftr(ID_AA64ISAR0_EL1)) >> 12) & 0x3;
+	return (ftr & 0x1);
+}
+
+/*
+ * Check if SHA512 is available
+ */
+static inline boolean_t
+zfs_sha512_available(void)
+{
+	unsigned long ftr = ((get_ftr(ID_AA64ISAR0_EL1)) >> 12) & 0x3;
+	return (ftr & 0x2);
+}
 
 #endif /* _LINUX_SIMD_AARCH64_H */
diff --git a/include/os/linux/kernel/linux/simd_arm.h b/include/os/linux/kernel/linux/simd_arm.h
@@ -0,0 +1,80 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or https://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (C) 2022 Tino Reichardt <[email protected]>
+ */
+
+/*
+ * USER API:
+ *
+ * Kernel fpu methods:
+ *	kfpu_allowed()
+ *	kfpu_begin()
+ *	kfpu_end()
+ *	kfpu_init()
+ *	kfpu_fini()
+ *
+ * SIMD support:
+ *
+ * Following functions should be called to determine whether CPU feature
+ * is supported. All functions are usable in kernel and user space.
+ * If a SIMD algorithm is using more than one instruction set
+ * all relevant feature test functions should be called.
+ *
+ * Supported features:
+ *   zfs_neon_available()
+ *   zfs_sha256_available()
+ */
+
+#ifndef _LINUX_SIMD_ARM_H
+#define	_LINUX_SIMD_ARM_H
+
+#include <sys/types.h>
+#include <asm/neon.h>
+#include <asm/elf.h>
+#include <asm/hwcap.h>
+
+#define	kfpu_allowed()		1
+#define	kfpu_begin()		kernel_neon_begin()
+#define	kfpu_end()		kernel_neon_end()
+#define	kfpu_init()		(0)
+#define	kfpu_fini()		do {} while (0)
+
+/*
+ * Check if NEON is available
+ */
+static inline boolean_t
+zfs_neon_available(void)
+{
+	return (elf_hwcap & HWCAP_NEON);
+}
+
+/*
+ * Check if SHA256 is available
+ */
+static inline boolean_t
+zfs_sha256_available(void)
+{
+	return (elf_hwcap2 & HWCAP2_SHA2);
+}
+
+#endif /* _LINUX_SIMD_ARM_H */
diff --git a/include/os/linux/kernel/linux/simd_powerpc.h b/include/os/linux/kernel/linux/simd_powerpc.h
@@ -50,9 +50,6 @@
 #ifndef _LINUX_SIMD_POWERPC_H
 #define	_LINUX_SIMD_POWERPC_H
 
-/* only for __powerpc__ */
-#if defined(__powerpc__)
-
 #include <linux/preempt.h>
 #include <linux/export.h>
 #include <linux/sched.h>
@@ -134,6 +131,4 @@ zfs_isa207_available(void)
 	return (cpu_has_feature(CPU_FTR_ARCH_207S));
 }
 
-#endif /* defined(__powerpc) */
-
 #endif /* _LINUX_SIMD_POWERPC_H */
diff --git a/include/os/linux/kernel/linux/simd_x86.h b/include/os/linux/kernel/linux/simd_x86.h
@@ -53,6 +53,8 @@
  *	zfs_bmi1_available()
  *	zfs_bmi2_available()
  *
+ *	zfs_shani_available()
+ *
  *	zfs_avx512f_available()
  *	zfs_avx512cd_available()
  *	zfs_avx512er_available()
@@ -586,6 +588,19 @@ zfs_movbe_available(void)
 #endif
 }
 
+/*
+ * Check if SHA_NI instruction set is available
+ */
+static inline boolean_t
+zfs_shani_available(void)
+{
+#if defined(X86_FEATURE_SHA_NI)
+	return (!!boot_cpu_has(X86_FEATURE_SHA_NI));
+#else
+	return (B_FALSE);
+#endif
+}
+
 /*
  * AVX-512 family of instruction sets:
  *