interleaved MUL2 x4

openzfs · Jul 5, 2016 · 6654e60 · 6654e60
1 parent 367c932
commit 6654e60
Showing 1 changed file with 23 additions and 16 deletions.
diff --git a/module/zfs/vdev_raidz_math_neon.c b/module/zfs/vdev_raidz_math_neon.c
@@ -334,9 +334,31 @@ typedef struct v {
 	:	WVR(16), WVR(17));					\
 }
 
-#define	_MUL2_x2(r...)							\
+#define	MUL2(r...)							\
 {									\
 	switch (REG_CNT(r)) {						\
+	case 4:								\
+		__asm(							\
+		"cmgt v19.16b," VR(17) ".16b," VR0(r) ".16b\n"		\
+		"cmgt v18.16b," VR(17) ".16b," VR1(r) ".16b\n"		\
+		"cmgt v21.16b," VR(17) ".16b," VR2(r) ".16b\n"		\
+		"cmgt v20.16b," VR(17) ".16b," VR3(r) ".16b\n"		\
+		"and v19.16b,v19.16b," VR(16) ".16b\n"			\
+		"and v18.16b,v18.16b," VR(16) ".16b\n"			\
+		"and v21.16b,v21.16b," VR(16) ".16b\n"			\
+		"and v20.16b,v20.16b," VR(16) ".16b\n"			\
+		"shl " VR0(r) ".16b," VR0(r) ".16b,#1\n"		\
+		"shl " VR1(r) ".16b," VR1(r) ".16b,#1\n"		\
+		"shl " VR2(r) ".16b," VR2(r) ".16b,#1\n"		\
+		"shl " VR3(r) ".16b," VR3(r) ".16b,#1\n"		\
+		"eor " VR0(r) ".16b,v19.16b," VR0(r) ".16b\n"		\
+		"eor " VR1(r) ".16b,v18.16b," VR1(r) ".16b\n"		\
+		"eor " VR2(r) ".16b,v21.16b," VR2(r) ".16b\n"		\
+		"eor " VR3(r) ".16b,v20.16b," VR3(r) ".16b\n"		\
+		:	UVR0(r), UVR1(r), UVR2(r), UVR3(r)		\
+		:	RVR(17), RVR(16)				\
+		:	"v18", "v19", "v20", "v21");			\
+		break;							\
 	case 2:								\
 		__asm(							\
 		"cmgt v19.16b," VR(17) ".16b," VR0(r) ".16b\n"		\
@@ -356,21 +378,6 @@ typedef struct v {
 	}								\
 }
 
-#define	MUL2(r...)							\
-{									\
-	switch (REG_CNT(r)) {						\
-	case 4:								\
-		_MUL2_x2(R_01(r));					\
-		_MUL2_x2(R_23(r));					\
-		break;							\
-	case 2:								\
-		_MUL2_x2(r);						\
-		break;							\
-	default:							\
-		ASM_BUG();						\
-	}								\
-}
-
 #define	MUL4(r...)							\
 {									\
 	MUL2(r);							\