feat(lscq): add arm64 support

collection/lscq/asm_arm64.go

@@ -0,0 +1,70 @@
+//go:build arm64 && !gccgo && !appengine
+// +build arm64,!gccgo,!appengine
+
+package lscq
+
+import (
+	"golang.org/x/sys/cpu"
+	"runtime"
+	"unsafe"
+)
+
+var arm64HasAtomics = detectArm64HasAtomics()
+
+type uint128 [2]uint64
+
+func compareAndSwapUint128(addr *uint128, old1, old2, new1, new2 uint64) (swapped bool)
+
+func loadUint128(addr *uint128) (val uint128)
+
+func loadSCQNodePointer(addr unsafe.Pointer) (val scqNodePointer)
+
+func loadSCQNodeUint64(addr unsafe.Pointer) (val scqNodeUint64)
+
+func atomicTestAndSetFirstBit(addr *uint64) (val uint64)
+
+func atomicTestAndSetSecondBit(addr *uint64) (val uint64)
+
+func resetNode(addr unsafe.Pointer)
+
+//go:nosplit
+func compareAndSwapSCQNodePointer(addr *scqNodePointer, old, new scqNodePointer) (swapped bool) {
+	// Ref: src/runtime/atomic_pointer.go:sync_atomic_CompareAndSwapPointer
+	if runtimeEnableWriteBarrier() {
+		runtimeatomicwb(&addr.data, new.data)
+	}
+	return compareAndSwapUint128((*uint128)(runtimenoescape(unsafe.Pointer(addr))), old.flags, uint64(uintptr(old.data)), new.flags, uint64(uintptr(new.data)))
+}
+
+func compareAndSwapSCQNodeUint64(addr *scqNodeUint64, old, new scqNodeUint64) (swapped bool) {
+	return compareAndSwapUint128((*uint128)(unsafe.Pointer(addr)), old.flags, old.data, new.flags, new.data)
+}
+
+func runtimeEnableWriteBarrier() bool
+
+//go:linkname runtimeatomicwb runtime.atomicwb
+//go:noescape
+func runtimeatomicwb(ptr *unsafe.Pointer, new unsafe.Pointer)
+
+//go:linkname runtimenoescape runtime.noescape
+func runtimenoescape(p unsafe.Pointer) unsafe.Pointer
+
+//go:nosplit
+func atomicWriteBarrier(ptr *unsafe.Pointer) {
+	// For SCQ dequeue only. (fastpath)
+	if runtimeEnableWriteBarrier() {
+		runtimeatomicwb(ptr, nil)
+	}
+}
+
+//go:linkname sysctlEnabled internal/cpu.sysctlEnabled
+func sysctlEnabled(name []byte) bool
+
+func detectArm64HasAtomics() bool {
+	switch runtime.GOOS {
+	case "darwin":
+		return sysctlEnabled([]byte("hw.optional.armv8_1_atomics\x00"))
+	default:
+		return cpu.ARM64.HasATOMICS
+	}
+}

collection/lscq/asm_arm64.s

@@ -0,0 +1,89 @@
+#include "textflag.h"
+#include "funcdata.h"
+
+TEXT ·compareAndSwapUint128(SB), NOSPLIT, $0-41
+	MOVD	addr+0(FP), R0
+	MOVD	old1+8(FP), R2
+	MOVD	old2+16(FP), R3
+	MOVD	new1+24(FP), R4
+	MOVD	new2+32(FP), R5
+	MOVBU	·arm64HasAtomics+0(SB), R1
+	CBZ 	R1, load_store_loop
+	MOVD	R2, R6
+	MOVD	R3, R7
+	CASPD	(R2, R3), (R0), (R4, R5)
+	CMP 	R2, R6
+	BNE 	ok
+	CMP 	R3, R7
+	CSET	EQ, R0
+	MOVB	R0, ret+40(FP)
+	RET
+load_store_loop:
+	LDAXP	(R0), (R6, R7)
+	CMP 	R2, R6
+	BNE 	ok
+	CMP 	R3, R7
+	BNE 	ok
+	STLXP	(R4, R5), (R0), R6
+	CBNZ	R6, load_store_loop
+ok:
+	CSET	EQ, R0
+	MOVB	R0, ret+40(FP)
+	RET
+
+TEXT ·loadUint128(SB),NOSPLIT,$0-24
+	MOVD	ptr+0(FP), R0
+	LDAXP	(R0), (R0, R1)
+	MOVD	R0, ret+8(FP)
+	MOVD	R1, ret+16(FP)
+	RET
+
+TEXT ·loadSCQNodeUint64(SB),NOSPLIT,$0
+	MOVD	ptr+0(FP), R0
+	LDAXP	(R0), (R0, R1)
+	MOVD	R0, ret+8(FP)
+	MOVD	R1, ret+16(FP)
+	RET
+
+TEXT ·loadSCQNodePointer(SB),NOSPLIT,$0
+	MOVD	ptr+0(FP), R0
+	LDAXP	(R0), (R0, R1)
+	MOVD	R0, ret+8(FP)
+	MOVD	R1, ret+16(FP)
+	RET
+
+TEXT ·atomicTestAndSetFirstBit(SB),NOSPLIT,$0
+	MOVD	addr+0(FP), R0
+load_store_loop:
+	LDAXR	(R0), R1
+	ORR 	$(1<<63), R1, R1
+	STLXR	R1, (R0), R2
+	CBNZ	R2, load_store_loop
+	MOVD	R1, val+8(FP)
+	RET
+
+
+TEXT ·atomicTestAndSetSecondBit(SB),NOSPLIT,$0
+	MOVD	addr+0(FP), R0
+load_store_loop:
+	LDAXR	(R0), R1
+	ORR 	$(1<<62), R1, R1
+	STLXR	R1, (R0), R2
+	CBNZ	R2, load_store_loop
+	MOVD	R1, val+8(FP)
+	RET
+
+TEXT ·resetNode(SB),NOSPLIT,$0
+	MOVD	addr+0(FP), R0
+	MOVD	$0, 8(R0)
+load_store_loop:
+	LDAXR	(R0), R1
+	ORR 	$(1<<62), R1, R1
+	STLXR	R1, (R0), R2
+	CBNZ	R2, load_store_loop
+	RET
+
+TEXT ·runtimeEnableWriteBarrier(SB),NOSPLIT,$0
+	MOVW	runtime·writeBarrier(SB), R0
+	MOVB	R0, ret+0(FP)
+	RET

go.mod

@@ -5,5 +5,5 @@ go 1.16
 require (
 	github.com/stretchr/testify v1.7.0
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe
+	golang.org/x/sys v0.0.0-20221010170243-090e33056c14
 )

go.sum

@@ -9,6 +9,10 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cO
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe h1:W8vbETX/n8S6EmY0Pu4Ix7VvpsJUESTwl0oCK8MJOgk=
 golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220907062415-87db552b00fd h1:AZeIEzg+8RCELJYq8w+ODLVxFgLMMigSwO/ffKPEd9U=
+golang.org/x/sys v0.0.0-20220907062415-87db552b00fd/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=