Linux 4.14, 4.19, 5.0+ compat: SIMD save/restore

Contrary to initial testing we cannot rely on these kernels to invalidate the per-cpu FPU state and restore the FPU registers. Nor can we guarantee that the kernel won't modify the FPU state which we saved in the task struck. Therefore, the kfpu_begin() and kfpu_end() functions have been updated to save and restore the FPU state using our own dedicated per-cpu FPU state variables. This has the additional advantage of allowing us to use the FPU again in user threads. So we remove the code which was added to use task queues to ensure some functions ran in kernel threads. Signed-off-by: Brian Behlendorf <[email protected]> Issue openzfs#9346
behlendorf · Oct 10, 2019 · b6a1d5b · b6a1d5b
1 parent f3dc4a8
commit b6a1d5b
Show file tree

Hide file tree

Showing 19 changed files with 276 additions and 294 deletions.
diff --git a/config/kernel-fpu.m4 b/config/kernel-fpu.m4
@@ -2,15 +2,9 @@ dnl #
 dnl # Handle differences in kernel FPU code.
 dnl #
 dnl # Kernel
-dnl # 5.2:	The fpu->initialized flag was replaced by TIF_NEED_FPU_LOAD.
-dnl #		HAVE_KERNEL_TIF_NEED_FPU_LOAD
-dnl #
-dnl # 5.0:	As an optimization SIMD operations performed by kernel
-dnl #		threads can skip saving and restoring their FPU context.
-dnl #		Wrappers have been introduced to determine the running
-dnl #		context and use either the SIMD or generic implementation.
+dnl # 5.0:	Wrappers have been introduced to save/restore the FPU state.
 dnl #		This change was made to the 4.19.38 and 4.14.120 LTS kernels.
-dnl #		HAVE_KERNEL_FPU_INITIALIZED
+dnl #		HAVE_KERNEL_FPU_INTERNAL
 dnl #
 dnl # 4.2:	Use __kernel_fpu_{begin,end}()
 dnl #		HAVE_UNDERSCORE_KERNEL_FPU & KERNEL_EXPORTS_X86_FPU
@@ -38,6 +32,7 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU_HEADER], [
 
 AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [
 	ZFS_LINUX_TEST_SRC([kernel_fpu], [
+		#include <linux/types.h>
 		#ifdef HAVE_KERNEL_FPU_API_HEADER
 		#include <asm/fpu/api.h>
 		#else
@@ -50,6 +45,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [
 	], [], [$ZFS_META_LICENSE])
 
 	ZFS_LINUX_TEST_SRC([__kernel_fpu], [
+		#include <linux/types.h>
 		#ifdef HAVE_KERNEL_FPU_API_HEADER
 		#include <asm/fpu/api.h>
 		#else
@@ -61,22 +57,41 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [
 		__kernel_fpu_end();
 	], [], [$ZFS_META_LICENSE])
 
-	ZFS_LINUX_TEST_SRC([fpu_initialized], [
-		#include <linux/module.h>
-		#include <linux/sched.h>
-	],[
-		struct fpu *fpu = &current->thread.fpu;
-		if (fpu->initialized) { return (0); };
-	])
+	ZFS_LINUX_TEST_SRC([fpu_internal], [
+		#if defined(__x86_64) || defined(__x86_64__) || \
+		    defined(__i386) || defined(__i386__)
+		#if !defined(__x86)
+		#define __x86
+		#endif
+		#endif
 
-	ZFS_LINUX_TEST_SRC([tif_need_fpu_load], [
-		#include <linux/module.h>
-		#include <asm/thread_info.h>
+		#if !defined(__x86)
+		#error Unsupported architecture
+		#endif
 
-		#if !defined(TIF_NEED_FPU_LOAD)
-		#error "TIF_NEED_FPU_LOAD undefined"
+		#include <linux/types.h>
+		#ifdef HAVE_KERNEL_FPU_API_HEADER
+		#include <asm/fpu/api.h>
+		#include <asm/fpu/internal.h>
+		#else
+		#include <asm/i387.h>
+		#include <asm/xcr.h>
+		#endif
+
+		#if !defined(XSTATE_XSAVE)
+		#error XSTATE_XSAVE not defined
 		#endif
-	],[])
+
+		#if !defined(XSTATE_XRESTORE)
+		#error XSTATE_XRESTORE not defined
+		#endif
+	],[
+		struct fpu *fpu = &current->thread.fpu;
+		union fpregs_state *st = &fpu->state;
+		struct fregs_state *fr __attribute__ ((unused)) = &st->fsave;
+		struct fxregs_state *fxr __attribute__ ((unused)) = &st->fxsave;
+		struct xregs_state *xr __attribute__ ((unused)) = &st->xsave;
+	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_FPU], [
@@ -104,25 +119,12 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU], [
 			AC_DEFINE(KERNEL_EXPORTS_X86_FPU, 1,
 			    [kernel exports FPU functions])
 		],[
-			dnl #
-			dnl # Linux 5.0 kernel
-			dnl #
-			ZFS_LINUX_TEST_RESULT([fpu_initialized], [
-				AC_MSG_RESULT(fpu.initialized)
-				AC_DEFINE(HAVE_KERNEL_FPU_INITIALIZED, 1,
-				    [kernel fpu.initialized exists])
+			ZFS_LINUX_TEST_RESULT([fpu_internal], [
+				AC_MSG_RESULT(internal)
+				AC_DEFINE(HAVE_KERNEL_FPU_INTERNAL, 1,
+				    [kernel fpu internal])
 			],[
-				dnl #
-				dnl # Linux 5.2 kernel
-				dnl #
-				ZFS_LINUX_TEST_RESULT([tif_need_fpu_load], [
-					AC_MSG_RESULT(TIF_NEED_FPU_LOAD)
-					AC_DEFINE(
-					    HAVE_KERNEL_TIF_NEED_FPU_LOAD, 1,
-					    [kernel TIF_NEED_FPU_LOAD exists])
-				],[
-					AC_MSG_RESULT(unavailable)
-				])
+				AC_MSG_RESULT(unavailable)
 			])
 		])
 	])

diff --git a/include/os/linux/kernel/linux/simd.h b/include/os/linux/kernel/linux/simd.h
@@ -33,9 +33,10 @@
 #else
 
 #define	kfpu_allowed()		0
-#define	kfpu_initialize(tsk)	do {} while (0)
 #define	kfpu_begin()		do {} while (0)
 #define	kfpu_end()		do {} while (0)
+#define	kfpu_init()		0
+#define	kfpu_fini()		((void) 0)
 
 #endif
 #endif /* _LINUX_SIMD_H */
diff --git a/include/os/linux/kernel/linux/simd_aarch64.h b/include/os/linux/kernel/linux/simd_aarch64.h
@@ -27,9 +27,10 @@
  *
  * Kernel fpu methods:
  *	kfpu_allowed()
- *	kfpu_initialize()
  *	kfpu_begin()
  *	kfpu_end()
+ *	kfpu_init()
+ *	kfpu_fini()
  */
 
 #ifndef _LINUX_SIMD_AARCH64_H
@@ -43,9 +44,10 @@
 #include <asm/neon.h>
 
 #define	kfpu_allowed()		1
-#define	kfpu_initialize(tsk)	do {} while (0)
 #define	kfpu_begin()		kernel_neon_begin()
 #define	kfpu_end()		kernel_neon_end()
+#define	kfpu_init()		0
+#define	kfpu_fini()		((void) 0)
 
 #endif /* __aarch64__ */