diff --git a/collection/skipmap/oparray.go b/collection/skipmap/oparray.go new file mode 100644 index 00000000..ce4cbc59 --- /dev/null +++ b/collection/skipmap/oparray.go @@ -0,0 +1,60 @@ +// Copyright 2021 ByteDance Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package skipmap + +import ( + "sync/atomic" + "unsafe" +) + +const ( + op1 = 4 + op2 = maxLevel - op1 +) + +type optionalArray struct { + base [op1]unsafe.Pointer + extra *([op2]unsafe.Pointer) +} + +func (a *optionalArray) load(i int) unsafe.Pointer { + if i < op1 { + return a.base[i] + } + return a.extra[i-op1] +} + +func (a *optionalArray) store(i int, p unsafe.Pointer) { + if i < op1 { + a.base[i] = p + return + } + a.extra[i-op1] = p +} + +func (a *optionalArray) atomicLoad(i int) unsafe.Pointer { + if i < op1 { + return atomic.LoadPointer(&a.base[i]) + } + return atomic.LoadPointer(&a.extra[i-op1]) +} + +func (a *optionalArray) atomicStore(i int, p unsafe.Pointer) { + if i < op1 { + atomic.StorePointer(&a.base[i], p) + return + } + atomic.StorePointer(&a.extra[i-op1], p) +} diff --git a/collection/skipmap/oparry_test.go b/collection/skipmap/oparry_test.go new file mode 100644 index 00000000..bab1fffb --- /dev/null +++ b/collection/skipmap/oparry_test.go @@ -0,0 +1,64 @@ +// Copyright 2021 ByteDance Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package skipmap + +import ( + "testing" + "unsafe" + + "github.com/bytedance/gopkg/lang/fastrand" +) + +type dummy struct { + data optionalArray +} + +func TestOpArray(t *testing.T) { + n := new(dummy) + n.data.extra = new([op2]unsafe.Pointer) + + var array [maxLevel]unsafe.Pointer + for i := 0; i < maxLevel; i++ { + value := unsafe.Pointer(&dummy{}) + array[i] = value + n.data.store(i, value) + } + + for i := 0; i < maxLevel; i++ { + if array[i] != n.data.load(i) || array[i] != n.data.atomicLoad(i) { + t.Fatal(i, array[i], n.data.load(i)) + } + } + + for i := 0; i < 1000; i++ { + r := int(fastrand.Uint32n(maxLevel)) + value := unsafe.Pointer(&dummy{}) + if i%100 == 0 { + value = nil + } + array[r] = value + if fastrand.Uint32n(2) == 0 { + n.data.store(r, value) + } else { + n.data.atomicStore(r, value) + } + } + + for i := 0; i < maxLevel; i++ { + if array[i] != n.data.load(i) || array[i] != n.data.atomicLoad(i) { + t.Fatal(i, array[i], n.data.load(i)) + } + } +} diff --git a/collection/skipmap/skipmap.go b/collection/skipmap/skipmap.go index 1a3e8869..d4da3ed8 100644 --- a/collection/skipmap/skipmap.go +++ b/collection/skipmap/skipmap.go @@ -32,19 +32,23 @@ type Int64Map struct { type int64Node struct { key int64 - value unsafe.Pointer - next []*int64Node + value unsafe.Pointer // *interface{} + next optionalArray // [level]*int64Node mu sync.Mutex flags bitflag + level uint32 } func newInt64Node(key int64, value interface{}, level int) *int64Node { - n := &int64Node{ - key: key, - next: make([]*int64Node, level), + node := &int64Node{ + key: key, + level: uint32(level), } - n.storeVal(value) - return n + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } func (n *int64Node) storeVal(value interface{}) { @@ -55,14 +59,20 @@ func (n *int64Node) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *int64Node) loadNext(i int) *int64Node { - return (*int64Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*int64Node)(n.next.load(i)) +} + +func (n *int64Node) storeNext(i int, node *int64Node) { + n.next.store(i, unsafe.Pointer(node)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *int64Node) storeNext(i int, value *int64Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *int64Node) atomicLoadNext(i int) *int64Node { + return (*int64Node)(n.next.atomicLoad(i)) +} + +func (n *int64Node) atomicStoreNext(i int, node *int64Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *int64Node) lessthan(key int64) bool { @@ -89,10 +99,10 @@ func NewInt64() *Int64Map { func (s *Int64Map) findNode(key int64, preds *[maxLevel]*int64Node, succs *[maxLevel]*int64Node) *int64Node { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -111,10 +121,10 @@ func (s *Int64Map) findNodeDelete(key int64, preds *[maxLevel]*int64Node, succs // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -173,7 +183,7 @@ func (s *Int64Map) Store(key int64, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt64(preds, highestLocked) @@ -182,8 +192,8 @@ func (s *Int64Map) Store(key int64, value interface{}) { nn := newInt64Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt64(preds, highestLocked) @@ -213,10 +223,10 @@ func (s *Int64Map) randomlevel() int { func (s *Int64Map) Load(key int64) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -242,7 +252,7 @@ func (s *Int64Map) LoadAndDelete(key int64) (value interface{}, loaded bool) { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -274,7 +284,7 @@ func (s *Int64Map) LoadAndDelete(key int64) (value interface{}, loaded bool) { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt64(preds, highestLocked) @@ -283,7 +293,7 @@ func (s *Int64Map) LoadAndDelete(key int64) (value interface{}, loaded bool) { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt64(preds, highestLocked) @@ -331,7 +341,7 @@ func (s *Int64Map) LoadOrStore(key int64, value interface{}) (actual interface{} // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt64(preds, highestLocked) @@ -340,8 +350,8 @@ func (s *Int64Map) LoadOrStore(key int64, value interface{}) (actual interface{} nn := newInt64Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt64(preds, highestLocked) @@ -361,7 +371,7 @@ func (s *Int64Map) Delete(key int64) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -393,7 +403,7 @@ func (s *Int64Map) Delete(key int64) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockInt64(preds, highestLocked) @@ -402,7 +412,7 @@ func (s *Int64Map) Delete(key int64) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt64(preds, highestLocked) @@ -421,16 +431,16 @@ func (s *Int64Map) Delete(key int64) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Int64Map) Range(f func(key int64, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } diff --git a/collection/skipmap/types.go b/collection/skipmap/types.go index 9b2a212b..2be978f2 100644 --- a/collection/skipmap/types.go +++ b/collection/skipmap/types.go @@ -31,19 +31,23 @@ type Float32Map struct { type float32Node struct { key float32 - value unsafe.Pointer - next []*float32Node + value unsafe.Pointer // *interface{} + next optionalArray // [level]*float32Node mu sync.Mutex flags bitflag + level uint32 } func newFloat32Node(key float32, value interface{}, level int) *float32Node { - n := &float32Node{ - key: key, - next: make([]*float32Node, level), + node := &float32Node{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *float32Node) storeVal(value interface{}) { @@ -54,14 +58,20 @@ func (n *float32Node) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *float32Node) loadNext(i int) *float32Node { - return (*float32Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*float32Node)(n.next.load(i)) +} + +func (n *float32Node) storeNext(i int, node *float32Node) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *float32Node) atomicLoadNext(i int) *float32Node { + return (*float32Node)(n.next.atomicLoad(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *float32Node) storeNext(i int, value *float32Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *float32Node) atomicStoreNext(i int, node *float32Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *float32Node) lessthan(key float32) bool { @@ -88,10 +98,10 @@ func NewFloat32() *Float32Map { func (s *Float32Map) findNode(key float32, preds *[maxLevel]*float32Node, succs *[maxLevel]*float32Node) *float32Node { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -110,10 +120,10 @@ func (s *Float32Map) findNodeDelete(key float32, preds *[maxLevel]*float32Node, // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -172,7 +182,7 @@ func (s *Float32Map) Store(key float32, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat32(preds, highestLocked) @@ -181,8 +191,8 @@ func (s *Float32Map) Store(key float32, value interface{}) { nn := newFloat32Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat32(preds, highestLocked) @@ -212,10 +222,10 @@ func (s *Float32Map) randomlevel() int { func (s *Float32Map) Load(key float32) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -241,7 +251,7 @@ func (s *Float32Map) LoadAndDelete(key float32) (value interface{}, loaded bool) for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -273,7 +283,7 @@ func (s *Float32Map) LoadAndDelete(key float32) (value interface{}, loaded bool) // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockFloat32(preds, highestLocked) @@ -282,7 +292,7 @@ func (s *Float32Map) LoadAndDelete(key float32) (value interface{}, loaded bool) for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockFloat32(preds, highestLocked) @@ -330,7 +340,7 @@ func (s *Float32Map) LoadOrStore(key float32, value interface{}) (actual interfa // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat32(preds, highestLocked) @@ -339,8 +349,8 @@ func (s *Float32Map) LoadOrStore(key float32, value interface{}) (actual interfa nn := newFloat32Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat32(preds, highestLocked) @@ -360,7 +370,7 @@ func (s *Float32Map) Delete(key float32) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -392,7 +402,7 @@ func (s *Float32Map) Delete(key float32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockFloat32(preds, highestLocked) @@ -401,7 +411,7 @@ func (s *Float32Map) Delete(key float32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockFloat32(preds, highestLocked) @@ -420,16 +430,16 @@ func (s *Float32Map) Delete(key float32) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Float32Map) Range(f func(key float32, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -447,19 +457,23 @@ type Float32MapDesc struct { type float32NodeDesc struct { key float32 - value unsafe.Pointer - next []*float32NodeDesc + value unsafe.Pointer // *interface{} + next optionalArray // [level]*float32NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newFloat32NodeDesc(key float32, value interface{}, level int) *float32NodeDesc { - n := &float32NodeDesc{ - key: key, - next: make([]*float32NodeDesc, level), + node := &float32NodeDesc{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *float32NodeDesc) storeVal(value interface{}) { @@ -470,14 +484,20 @@ func (n *float32NodeDesc) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *float32NodeDesc) loadNext(i int) *float32NodeDesc { - return (*float32NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*float32NodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *float32NodeDesc) storeNext(i int, value *float32NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *float32NodeDesc) storeNext(i int, node *float32NodeDesc) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *float32NodeDesc) atomicLoadNext(i int) *float32NodeDesc { + return (*float32NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *float32NodeDesc) atomicStoreNext(i int, node *float32NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *float32NodeDesc) lessthan(key float32) bool { @@ -504,10 +524,10 @@ func NewFloat32Desc() *Float32MapDesc { func (s *Float32MapDesc) findNode(key float32, preds *[maxLevel]*float32NodeDesc, succs *[maxLevel]*float32NodeDesc) *float32NodeDesc { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -526,10 +546,10 @@ func (s *Float32MapDesc) findNodeDelete(key float32, preds *[maxLevel]*float32No // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -588,7 +608,7 @@ func (s *Float32MapDesc) Store(key float32, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat32Desc(preds, highestLocked) @@ -597,8 +617,8 @@ func (s *Float32MapDesc) Store(key float32, value interface{}) { nn := newFloat32NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat32Desc(preds, highestLocked) @@ -628,10 +648,10 @@ func (s *Float32MapDesc) randomlevel() int { func (s *Float32MapDesc) Load(key float32) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -657,7 +677,7 @@ func (s *Float32MapDesc) LoadAndDelete(key float32) (value interface{}, loaded b for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -689,7 +709,7 @@ func (s *Float32MapDesc) LoadAndDelete(key float32) (value interface{}, loaded b // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockFloat32Desc(preds, highestLocked) @@ -698,7 +718,7 @@ func (s *Float32MapDesc) LoadAndDelete(key float32) (value interface{}, loaded b for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockFloat32Desc(preds, highestLocked) @@ -746,7 +766,7 @@ func (s *Float32MapDesc) LoadOrStore(key float32, value interface{}) (actual int // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat32Desc(preds, highestLocked) @@ -755,8 +775,8 @@ func (s *Float32MapDesc) LoadOrStore(key float32, value interface{}) (actual int nn := newFloat32NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat32Desc(preds, highestLocked) @@ -776,7 +796,7 @@ func (s *Float32MapDesc) Delete(key float32) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -808,7 +828,7 @@ func (s *Float32MapDesc) Delete(key float32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockFloat32Desc(preds, highestLocked) @@ -817,7 +837,7 @@ func (s *Float32MapDesc) Delete(key float32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockFloat32Desc(preds, highestLocked) @@ -836,16 +856,16 @@ func (s *Float32MapDesc) Delete(key float32) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Float32MapDesc) Range(f func(key float32, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -863,19 +883,23 @@ type Float64Map struct { type float64Node struct { key float64 - value unsafe.Pointer - next []*float64Node + value unsafe.Pointer // *interface{} + next optionalArray // [level]*float64Node mu sync.Mutex flags bitflag + level uint32 } func newFloat64Node(key float64, value interface{}, level int) *float64Node { - n := &float64Node{ - key: key, - next: make([]*float64Node, level), + node := &float64Node{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *float64Node) storeVal(value interface{}) { @@ -886,14 +910,20 @@ func (n *float64Node) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *float64Node) loadNext(i int) *float64Node { - return (*float64Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*float64Node)(n.next.load(i)) +} + +func (n *float64Node) storeNext(i int, node *float64Node) { + n.next.store(i, unsafe.Pointer(node)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *float64Node) storeNext(i int, value *float64Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *float64Node) atomicLoadNext(i int) *float64Node { + return (*float64Node)(n.next.atomicLoad(i)) +} + +func (n *float64Node) atomicStoreNext(i int, node *float64Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *float64Node) lessthan(key float64) bool { @@ -920,10 +950,10 @@ func NewFloat64() *Float64Map { func (s *Float64Map) findNode(key float64, preds *[maxLevel]*float64Node, succs *[maxLevel]*float64Node) *float64Node { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -942,10 +972,10 @@ func (s *Float64Map) findNodeDelete(key float64, preds *[maxLevel]*float64Node, // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -1004,7 +1034,7 @@ func (s *Float64Map) Store(key float64, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat64(preds, highestLocked) @@ -1013,8 +1043,8 @@ func (s *Float64Map) Store(key float64, value interface{}) { nn := newFloat64Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat64(preds, highestLocked) @@ -1044,10 +1074,10 @@ func (s *Float64Map) randomlevel() int { func (s *Float64Map) Load(key float64) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -1073,7 +1103,7 @@ func (s *Float64Map) LoadAndDelete(key float64) (value interface{}, loaded bool) for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -1105,7 +1135,7 @@ func (s *Float64Map) LoadAndDelete(key float64) (value interface{}, loaded bool) // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockFloat64(preds, highestLocked) @@ -1114,7 +1144,7 @@ func (s *Float64Map) LoadAndDelete(key float64) (value interface{}, loaded bool) for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockFloat64(preds, highestLocked) @@ -1162,7 +1192,7 @@ func (s *Float64Map) LoadOrStore(key float64, value interface{}) (actual interfa // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat64(preds, highestLocked) @@ -1171,8 +1201,8 @@ func (s *Float64Map) LoadOrStore(key float64, value interface{}) (actual interfa nn := newFloat64Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat64(preds, highestLocked) @@ -1192,7 +1222,7 @@ func (s *Float64Map) Delete(key float64) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -1224,7 +1254,7 @@ func (s *Float64Map) Delete(key float64) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockFloat64(preds, highestLocked) @@ -1233,7 +1263,7 @@ func (s *Float64Map) Delete(key float64) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockFloat64(preds, highestLocked) @@ -1252,16 +1282,16 @@ func (s *Float64Map) Delete(key float64) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Float64Map) Range(f func(key float64, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -1279,19 +1309,23 @@ type Float64MapDesc struct { type float64NodeDesc struct { key float64 - value unsafe.Pointer - next []*float64NodeDesc + value unsafe.Pointer // *interface{} + next optionalArray // [level]*float64NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newFloat64NodeDesc(key float64, value interface{}, level int) *float64NodeDesc { - n := &float64NodeDesc{ - key: key, - next: make([]*float64NodeDesc, level), + node := &float64NodeDesc{ + key: key, + level: uint32(level), } - n.storeVal(value) - return n + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } func (n *float64NodeDesc) storeVal(value interface{}) { @@ -1302,14 +1336,20 @@ func (n *float64NodeDesc) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *float64NodeDesc) loadNext(i int) *float64NodeDesc { - return (*float64NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*float64NodeDesc)(n.next.load(i)) +} + +func (n *float64NodeDesc) storeNext(i int, node *float64NodeDesc) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *float64NodeDesc) atomicLoadNext(i int) *float64NodeDesc { + return (*float64NodeDesc)(n.next.atomicLoad(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *float64NodeDesc) storeNext(i int, value *float64NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *float64NodeDesc) atomicStoreNext(i int, node *float64NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *float64NodeDesc) lessthan(key float64) bool { @@ -1336,10 +1376,10 @@ func NewFloat64Desc() *Float64MapDesc { func (s *Float64MapDesc) findNode(key float64, preds *[maxLevel]*float64NodeDesc, succs *[maxLevel]*float64NodeDesc) *float64NodeDesc { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -1358,10 +1398,10 @@ func (s *Float64MapDesc) findNodeDelete(key float64, preds *[maxLevel]*float64No // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -1420,7 +1460,7 @@ func (s *Float64MapDesc) Store(key float64, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat64Desc(preds, highestLocked) @@ -1429,8 +1469,8 @@ func (s *Float64MapDesc) Store(key float64, value interface{}) { nn := newFloat64NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat64Desc(preds, highestLocked) @@ -1460,10 +1500,10 @@ func (s *Float64MapDesc) randomlevel() int { func (s *Float64MapDesc) Load(key float64) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -1489,7 +1529,7 @@ func (s *Float64MapDesc) LoadAndDelete(key float64) (value interface{}, loaded b for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -1521,7 +1561,7 @@ func (s *Float64MapDesc) LoadAndDelete(key float64) (value interface{}, loaded b // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockFloat64Desc(preds, highestLocked) @@ -1530,7 +1570,7 @@ func (s *Float64MapDesc) LoadAndDelete(key float64) (value interface{}, loaded b for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockFloat64Desc(preds, highestLocked) @@ -1578,7 +1618,7 @@ func (s *Float64MapDesc) LoadOrStore(key float64, value interface{}) (actual int // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat64Desc(preds, highestLocked) @@ -1587,8 +1627,8 @@ func (s *Float64MapDesc) LoadOrStore(key float64, value interface{}) (actual int nn := newFloat64NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat64Desc(preds, highestLocked) @@ -1608,7 +1648,7 @@ func (s *Float64MapDesc) Delete(key float64) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -1640,7 +1680,7 @@ func (s *Float64MapDesc) Delete(key float64) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockFloat64Desc(preds, highestLocked) @@ -1649,7 +1689,7 @@ func (s *Float64MapDesc) Delete(key float64) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockFloat64Desc(preds, highestLocked) @@ -1668,16 +1708,16 @@ func (s *Float64MapDesc) Delete(key float64) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Float64MapDesc) Range(f func(key float64, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -1695,19 +1735,23 @@ type Int32Map struct { type int32Node struct { key int32 - value unsafe.Pointer - next []*int32Node + value unsafe.Pointer // *interface{} + next optionalArray // [level]*int32Node mu sync.Mutex flags bitflag + level uint32 } func newInt32Node(key int32, value interface{}, level int) *int32Node { - n := &int32Node{ - key: key, - next: make([]*int32Node, level), + node := &int32Node{ + key: key, + level: uint32(level), } - n.storeVal(value) - return n + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } func (n *int32Node) storeVal(value interface{}) { @@ -1718,14 +1762,20 @@ func (n *int32Node) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *int32Node) loadNext(i int) *int32Node { - return (*int32Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*int32Node)(n.next.load(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *int32Node) storeNext(i int, value *int32Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *int32Node) storeNext(i int, node *int32Node) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *int32Node) atomicLoadNext(i int) *int32Node { + return (*int32Node)(n.next.atomicLoad(i)) +} + +func (n *int32Node) atomicStoreNext(i int, node *int32Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *int32Node) lessthan(key int32) bool { @@ -1752,10 +1802,10 @@ func NewInt32() *Int32Map { func (s *Int32Map) findNode(key int32, preds *[maxLevel]*int32Node, succs *[maxLevel]*int32Node) *int32Node { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -1774,10 +1824,10 @@ func (s *Int32Map) findNodeDelete(key int32, preds *[maxLevel]*int32Node, succs // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -1836,7 +1886,7 @@ func (s *Int32Map) Store(key int32, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt32(preds, highestLocked) @@ -1845,8 +1895,8 @@ func (s *Int32Map) Store(key int32, value interface{}) { nn := newInt32Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt32(preds, highestLocked) @@ -1876,10 +1926,10 @@ func (s *Int32Map) randomlevel() int { func (s *Int32Map) Load(key int32) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -1905,7 +1955,7 @@ func (s *Int32Map) LoadAndDelete(key int32) (value interface{}, loaded bool) { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -1937,7 +1987,7 @@ func (s *Int32Map) LoadAndDelete(key int32) (value interface{}, loaded bool) { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt32(preds, highestLocked) @@ -1946,7 +1996,7 @@ func (s *Int32Map) LoadAndDelete(key int32) (value interface{}, loaded bool) { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt32(preds, highestLocked) @@ -1994,7 +2044,7 @@ func (s *Int32Map) LoadOrStore(key int32, value interface{}) (actual interface{} // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt32(preds, highestLocked) @@ -2003,8 +2053,8 @@ func (s *Int32Map) LoadOrStore(key int32, value interface{}) (actual interface{} nn := newInt32Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt32(preds, highestLocked) @@ -2024,7 +2074,7 @@ func (s *Int32Map) Delete(key int32) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -2056,7 +2106,7 @@ func (s *Int32Map) Delete(key int32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockInt32(preds, highestLocked) @@ -2065,7 +2115,7 @@ func (s *Int32Map) Delete(key int32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt32(preds, highestLocked) @@ -2084,16 +2134,16 @@ func (s *Int32Map) Delete(key int32) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Int32Map) Range(f func(key int32, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -2111,19 +2161,23 @@ type Int32MapDesc struct { type int32NodeDesc struct { key int32 - value unsafe.Pointer - next []*int32NodeDesc + value unsafe.Pointer // *interface{} + next optionalArray // [level]*int32NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newInt32NodeDesc(key int32, value interface{}, level int) *int32NodeDesc { - n := &int32NodeDesc{ - key: key, - next: make([]*int32NodeDesc, level), + node := &int32NodeDesc{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *int32NodeDesc) storeVal(value interface{}) { @@ -2134,14 +2188,20 @@ func (n *int32NodeDesc) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *int32NodeDesc) loadNext(i int) *int32NodeDesc { - return (*int32NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*int32NodeDesc)(n.next.load(i)) +} + +func (n *int32NodeDesc) storeNext(i int, node *int32NodeDesc) { + n.next.store(i, unsafe.Pointer(node)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *int32NodeDesc) storeNext(i int, value *int32NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *int32NodeDesc) atomicLoadNext(i int) *int32NodeDesc { + return (*int32NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *int32NodeDesc) atomicStoreNext(i int, node *int32NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *int32NodeDesc) lessthan(key int32) bool { @@ -2168,10 +2228,10 @@ func NewInt32Desc() *Int32MapDesc { func (s *Int32MapDesc) findNode(key int32, preds *[maxLevel]*int32NodeDesc, succs *[maxLevel]*int32NodeDesc) *int32NodeDesc { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2190,10 +2250,10 @@ func (s *Int32MapDesc) findNodeDelete(key int32, preds *[maxLevel]*int32NodeDesc // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2252,7 +2312,7 @@ func (s *Int32MapDesc) Store(key int32, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt32Desc(preds, highestLocked) @@ -2261,8 +2321,8 @@ func (s *Int32MapDesc) Store(key int32, value interface{}) { nn := newInt32NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt32Desc(preds, highestLocked) @@ -2292,10 +2352,10 @@ func (s *Int32MapDesc) randomlevel() int { func (s *Int32MapDesc) Load(key int32) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -2321,7 +2381,7 @@ func (s *Int32MapDesc) LoadAndDelete(key int32) (value interface{}, loaded bool) for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -2353,7 +2413,7 @@ func (s *Int32MapDesc) LoadAndDelete(key int32) (value interface{}, loaded bool) // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt32Desc(preds, highestLocked) @@ -2362,7 +2422,7 @@ func (s *Int32MapDesc) LoadAndDelete(key int32) (value interface{}, loaded bool) for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt32Desc(preds, highestLocked) @@ -2410,7 +2470,7 @@ func (s *Int32MapDesc) LoadOrStore(key int32, value interface{}) (actual interfa // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt32Desc(preds, highestLocked) @@ -2419,8 +2479,8 @@ func (s *Int32MapDesc) LoadOrStore(key int32, value interface{}) (actual interfa nn := newInt32NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt32Desc(preds, highestLocked) @@ -2440,7 +2500,7 @@ func (s *Int32MapDesc) Delete(key int32) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -2472,7 +2532,7 @@ func (s *Int32MapDesc) Delete(key int32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockInt32Desc(preds, highestLocked) @@ -2481,7 +2541,7 @@ func (s *Int32MapDesc) Delete(key int32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt32Desc(preds, highestLocked) @@ -2500,16 +2560,16 @@ func (s *Int32MapDesc) Delete(key int32) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Int32MapDesc) Range(f func(key int32, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -2527,19 +2587,23 @@ type Int16Map struct { type int16Node struct { key int16 - value unsafe.Pointer - next []*int16Node + value unsafe.Pointer // *interface{} + next optionalArray // [level]*int16Node mu sync.Mutex flags bitflag + level uint32 } func newInt16Node(key int16, value interface{}, level int) *int16Node { - n := &int16Node{ - key: key, - next: make([]*int16Node, level), + node := &int16Node{ + key: key, + level: uint32(level), } - n.storeVal(value) - return n + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } func (n *int16Node) storeVal(value interface{}) { @@ -2550,14 +2614,20 @@ func (n *int16Node) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *int16Node) loadNext(i int) *int16Node { - return (*int16Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*int16Node)(n.next.load(i)) +} + +func (n *int16Node) storeNext(i int, node *int16Node) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *int16Node) atomicLoadNext(i int) *int16Node { + return (*int16Node)(n.next.atomicLoad(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *int16Node) storeNext(i int, value *int16Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *int16Node) atomicStoreNext(i int, node *int16Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *int16Node) lessthan(key int16) bool { @@ -2584,10 +2654,10 @@ func NewInt16() *Int16Map { func (s *Int16Map) findNode(key int16, preds *[maxLevel]*int16Node, succs *[maxLevel]*int16Node) *int16Node { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2606,10 +2676,10 @@ func (s *Int16Map) findNodeDelete(key int16, preds *[maxLevel]*int16Node, succs // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2668,7 +2738,7 @@ func (s *Int16Map) Store(key int16, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt16(preds, highestLocked) @@ -2677,8 +2747,8 @@ func (s *Int16Map) Store(key int16, value interface{}) { nn := newInt16Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt16(preds, highestLocked) @@ -2708,10 +2778,10 @@ func (s *Int16Map) randomlevel() int { func (s *Int16Map) Load(key int16) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -2737,7 +2807,7 @@ func (s *Int16Map) LoadAndDelete(key int16) (value interface{}, loaded bool) { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -2769,7 +2839,7 @@ func (s *Int16Map) LoadAndDelete(key int16) (value interface{}, loaded bool) { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt16(preds, highestLocked) @@ -2778,7 +2848,7 @@ func (s *Int16Map) LoadAndDelete(key int16) (value interface{}, loaded bool) { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt16(preds, highestLocked) @@ -2826,7 +2896,7 @@ func (s *Int16Map) LoadOrStore(key int16, value interface{}) (actual interface{} // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt16(preds, highestLocked) @@ -2835,8 +2905,8 @@ func (s *Int16Map) LoadOrStore(key int16, value interface{}) (actual interface{} nn := newInt16Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt16(preds, highestLocked) @@ -2856,7 +2926,7 @@ func (s *Int16Map) Delete(key int16) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -2888,7 +2958,7 @@ func (s *Int16Map) Delete(key int16) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockInt16(preds, highestLocked) @@ -2897,7 +2967,7 @@ func (s *Int16Map) Delete(key int16) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt16(preds, highestLocked) @@ -2916,16 +2986,16 @@ func (s *Int16Map) Delete(key int16) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Int16Map) Range(f func(key int16, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -2943,19 +3013,23 @@ type Int16MapDesc struct { type int16NodeDesc struct { key int16 - value unsafe.Pointer - next []*int16NodeDesc + value unsafe.Pointer // *interface{} + next optionalArray // [level]*int16NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newInt16NodeDesc(key int16, value interface{}, level int) *int16NodeDesc { - n := &int16NodeDesc{ - key: key, - next: make([]*int16NodeDesc, level), + node := &int16NodeDesc{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *int16NodeDesc) storeVal(value interface{}) { @@ -2966,14 +3040,20 @@ func (n *int16NodeDesc) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *int16NodeDesc) loadNext(i int) *int16NodeDesc { - return (*int16NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*int16NodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *int16NodeDesc) storeNext(i int, value *int16NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *int16NodeDesc) storeNext(i int, node *int16NodeDesc) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *int16NodeDesc) atomicLoadNext(i int) *int16NodeDesc { + return (*int16NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *int16NodeDesc) atomicStoreNext(i int, node *int16NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *int16NodeDesc) lessthan(key int16) bool { @@ -3000,10 +3080,10 @@ func NewInt16Desc() *Int16MapDesc { func (s *Int16MapDesc) findNode(key int16, preds *[maxLevel]*int16NodeDesc, succs *[maxLevel]*int16NodeDesc) *int16NodeDesc { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3022,10 +3102,10 @@ func (s *Int16MapDesc) findNodeDelete(key int16, preds *[maxLevel]*int16NodeDesc // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3084,7 +3164,7 @@ func (s *Int16MapDesc) Store(key int16, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt16Desc(preds, highestLocked) @@ -3093,8 +3173,8 @@ func (s *Int16MapDesc) Store(key int16, value interface{}) { nn := newInt16NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt16Desc(preds, highestLocked) @@ -3124,10 +3204,10 @@ func (s *Int16MapDesc) randomlevel() int { func (s *Int16MapDesc) Load(key int16) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -3153,7 +3233,7 @@ func (s *Int16MapDesc) LoadAndDelete(key int16) (value interface{}, loaded bool) for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -3185,7 +3265,7 @@ func (s *Int16MapDesc) LoadAndDelete(key int16) (value interface{}, loaded bool) // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt16Desc(preds, highestLocked) @@ -3194,7 +3274,7 @@ func (s *Int16MapDesc) LoadAndDelete(key int16) (value interface{}, loaded bool) for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt16Desc(preds, highestLocked) @@ -3242,7 +3322,7 @@ func (s *Int16MapDesc) LoadOrStore(key int16, value interface{}) (actual interfa // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt16Desc(preds, highestLocked) @@ -3251,8 +3331,8 @@ func (s *Int16MapDesc) LoadOrStore(key int16, value interface{}) (actual interfa nn := newInt16NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt16Desc(preds, highestLocked) @@ -3272,7 +3352,7 @@ func (s *Int16MapDesc) Delete(key int16) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -3304,7 +3384,7 @@ func (s *Int16MapDesc) Delete(key int16) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockInt16Desc(preds, highestLocked) @@ -3313,7 +3393,7 @@ func (s *Int16MapDesc) Delete(key int16) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt16Desc(preds, highestLocked) @@ -3332,16 +3412,16 @@ func (s *Int16MapDesc) Delete(key int16) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Int16MapDesc) Range(f func(key int16, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -3359,19 +3439,23 @@ type IntMap struct { type intNode struct { key int - value unsafe.Pointer - next []*intNode + value unsafe.Pointer // *interface{} + next optionalArray // [level]*intNode mu sync.Mutex flags bitflag + level uint32 } func newIntNode(key int, value interface{}, level int) *intNode { - n := &intNode{ - key: key, - next: make([]*intNode, level), + node := &intNode{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *intNode) storeVal(value interface{}) { @@ -3382,14 +3466,20 @@ func (n *intNode) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *intNode) loadNext(i int) *intNode { - return (*intNode)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*intNode)(n.next.load(i)) +} + +func (n *intNode) storeNext(i int, node *intNode) { + n.next.store(i, unsafe.Pointer(node)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *intNode) storeNext(i int, value *intNode) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *intNode) atomicLoadNext(i int) *intNode { + return (*intNode)(n.next.atomicLoad(i)) +} + +func (n *intNode) atomicStoreNext(i int, node *intNode) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *intNode) lessthan(key int) bool { @@ -3416,10 +3506,10 @@ func NewInt() *IntMap { func (s *IntMap) findNode(key int, preds *[maxLevel]*intNode, succs *[maxLevel]*intNode) *intNode { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3438,10 +3528,10 @@ func (s *IntMap) findNodeDelete(key int, preds *[maxLevel]*intNode, succs *[maxL // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3500,7 +3590,7 @@ func (s *IntMap) Store(key int, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt(preds, highestLocked) @@ -3509,8 +3599,8 @@ func (s *IntMap) Store(key int, value interface{}) { nn := newIntNode(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt(preds, highestLocked) @@ -3540,10 +3630,10 @@ func (s *IntMap) randomlevel() int { func (s *IntMap) Load(key int) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -3569,7 +3659,7 @@ func (s *IntMap) LoadAndDelete(key int) (value interface{}, loaded bool) { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -3601,7 +3691,7 @@ func (s *IntMap) LoadAndDelete(key int) (value interface{}, loaded bool) { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt(preds, highestLocked) @@ -3610,7 +3700,7 @@ func (s *IntMap) LoadAndDelete(key int) (value interface{}, loaded bool) { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt(preds, highestLocked) @@ -3658,7 +3748,7 @@ func (s *IntMap) LoadOrStore(key int, value interface{}) (actual interface{}, lo // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt(preds, highestLocked) @@ -3667,8 +3757,8 @@ func (s *IntMap) LoadOrStore(key int, value interface{}) (actual interface{}, lo nn := newIntNode(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt(preds, highestLocked) @@ -3688,7 +3778,7 @@ func (s *IntMap) Delete(key int) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -3720,7 +3810,7 @@ func (s *IntMap) Delete(key int) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockInt(preds, highestLocked) @@ -3729,7 +3819,7 @@ func (s *IntMap) Delete(key int) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockInt(preds, highestLocked) @@ -3748,16 +3838,16 @@ func (s *IntMap) Delete(key int) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *IntMap) Range(f func(key int, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -3775,19 +3865,23 @@ type IntMapDesc struct { type intNodeDesc struct { key int - value unsafe.Pointer - next []*intNodeDesc + value unsafe.Pointer // *interface{} + next optionalArray // [level]*intNodeDesc mu sync.Mutex flags bitflag + level uint32 } func newIntNodeDesc(key int, value interface{}, level int) *intNodeDesc { - n := &intNodeDesc{ - key: key, - next: make([]*intNodeDesc, level), + node := &intNodeDesc{ + key: key, + level: uint32(level), } - n.storeVal(value) - return n + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } func (n *intNodeDesc) storeVal(value interface{}) { @@ -3798,14 +3892,20 @@ func (n *intNodeDesc) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *intNodeDesc) loadNext(i int) *intNodeDesc { - return (*intNodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*intNodeDesc)(n.next.load(i)) +} + +func (n *intNodeDesc) storeNext(i int, node *intNodeDesc) { + n.next.store(i, unsafe.Pointer(node)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *intNodeDesc) storeNext(i int, value *intNodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *intNodeDesc) atomicLoadNext(i int) *intNodeDesc { + return (*intNodeDesc)(n.next.atomicLoad(i)) +} + +func (n *intNodeDesc) atomicStoreNext(i int, node *intNodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *intNodeDesc) lessthan(key int) bool { @@ -3832,10 +3932,10 @@ func NewIntDesc() *IntMapDesc { func (s *IntMapDesc) findNode(key int, preds *[maxLevel]*intNodeDesc, succs *[maxLevel]*intNodeDesc) *intNodeDesc { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3854,10 +3954,10 @@ func (s *IntMapDesc) findNodeDelete(key int, preds *[maxLevel]*intNodeDesc, succ // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3916,7 +4016,7 @@ func (s *IntMapDesc) Store(key int, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockIntDesc(preds, highestLocked) @@ -3925,8 +4025,8 @@ func (s *IntMapDesc) Store(key int, value interface{}) { nn := newIntNodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockIntDesc(preds, highestLocked) @@ -3956,10 +4056,10 @@ func (s *IntMapDesc) randomlevel() int { func (s *IntMapDesc) Load(key int) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -3985,7 +4085,7 @@ func (s *IntMapDesc) LoadAndDelete(key int) (value interface{}, loaded bool) { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -4017,7 +4117,7 @@ func (s *IntMapDesc) LoadAndDelete(key int) (value interface{}, loaded bool) { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockIntDesc(preds, highestLocked) @@ -4026,7 +4126,7 @@ func (s *IntMapDesc) LoadAndDelete(key int) (value interface{}, loaded bool) { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockIntDesc(preds, highestLocked) @@ -4074,7 +4174,7 @@ func (s *IntMapDesc) LoadOrStore(key int, value interface{}) (actual interface{} // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockIntDesc(preds, highestLocked) @@ -4083,8 +4183,8 @@ func (s *IntMapDesc) LoadOrStore(key int, value interface{}) (actual interface{} nn := newIntNodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockIntDesc(preds, highestLocked) @@ -4104,7 +4204,7 @@ func (s *IntMapDesc) Delete(key int) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -4136,7 +4236,7 @@ func (s *IntMapDesc) Delete(key int) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockIntDesc(preds, highestLocked) @@ -4145,7 +4245,7 @@ func (s *IntMapDesc) Delete(key int) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockIntDesc(preds, highestLocked) @@ -4164,16 +4264,16 @@ func (s *IntMapDesc) Delete(key int) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *IntMapDesc) Range(f func(key int, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -4191,19 +4291,23 @@ type Uint64Map struct { type uint64Node struct { key uint64 - value unsafe.Pointer - next []*uint64Node + value unsafe.Pointer // *interface{} + next optionalArray // [level]*uint64Node mu sync.Mutex flags bitflag + level uint32 } func newUuint64Node(key uint64, value interface{}, level int) *uint64Node { - n := &uint64Node{ - key: key, - next: make([]*uint64Node, level), + node := &uint64Node{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *uint64Node) storeVal(value interface{}) { @@ -4214,14 +4318,20 @@ func (n *uint64Node) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *uint64Node) loadNext(i int) *uint64Node { - return (*uint64Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint64Node)(n.next.load(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *uint64Node) storeNext(i int, value *uint64Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *uint64Node) storeNext(i int, node *uint64Node) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uint64Node) atomicLoadNext(i int) *uint64Node { + return (*uint64Node)(n.next.atomicLoad(i)) +} + +func (n *uint64Node) atomicStoreNext(i int, node *uint64Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint64Node) lessthan(key uint64) bool { @@ -4248,10 +4358,10 @@ func NewUint64() *Uint64Map { func (s *Uint64Map) findNode(key uint64, preds *[maxLevel]*uint64Node, succs *[maxLevel]*uint64Node) *uint64Node { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -4270,10 +4380,10 @@ func (s *Uint64Map) findNodeDelete(key uint64, preds *[maxLevel]*uint64Node, suc // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -4332,7 +4442,7 @@ func (s *Uint64Map) Store(key uint64, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint64(preds, highestLocked) @@ -4341,8 +4451,8 @@ func (s *Uint64Map) Store(key uint64, value interface{}) { nn := newUuint64Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint64(preds, highestLocked) @@ -4372,10 +4482,10 @@ func (s *Uint64Map) randomlevel() int { func (s *Uint64Map) Load(key uint64) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -4401,7 +4511,7 @@ func (s *Uint64Map) LoadAndDelete(key uint64) (value interface{}, loaded bool) { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -4433,7 +4543,7 @@ func (s *Uint64Map) LoadAndDelete(key uint64) (value interface{}, loaded bool) { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint64(preds, highestLocked) @@ -4442,7 +4552,7 @@ func (s *Uint64Map) LoadAndDelete(key uint64) (value interface{}, loaded bool) { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint64(preds, highestLocked) @@ -4490,7 +4600,7 @@ func (s *Uint64Map) LoadOrStore(key uint64, value interface{}) (actual interface // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint64(preds, highestLocked) @@ -4499,8 +4609,8 @@ func (s *Uint64Map) LoadOrStore(key uint64, value interface{}) (actual interface nn := newUuint64Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint64(preds, highestLocked) @@ -4520,7 +4630,7 @@ func (s *Uint64Map) Delete(key uint64) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -4552,7 +4662,7 @@ func (s *Uint64Map) Delete(key uint64) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockUint64(preds, highestLocked) @@ -4561,7 +4671,7 @@ func (s *Uint64Map) Delete(key uint64) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint64(preds, highestLocked) @@ -4580,16 +4690,16 @@ func (s *Uint64Map) Delete(key uint64) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Uint64Map) Range(f func(key uint64, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -4607,19 +4717,23 @@ type Uint64MapDesc struct { type uint64NodeDesc struct { key uint64 - value unsafe.Pointer - next []*uint64NodeDesc + value unsafe.Pointer // *interface{} + next optionalArray // [level]*uint64NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newUuint64NodeDescDesc(key uint64, value interface{}, level int) *uint64NodeDesc { - n := &uint64NodeDesc{ - key: key, - next: make([]*uint64NodeDesc, level), + node := &uint64NodeDesc{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *uint64NodeDesc) storeVal(value interface{}) { @@ -4630,14 +4744,20 @@ func (n *uint64NodeDesc) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *uint64NodeDesc) loadNext(i int) *uint64NodeDesc { - return (*uint64NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint64NodeDesc)(n.next.load(i)) +} + +func (n *uint64NodeDesc) storeNext(i int, node *uint64NodeDesc) { + n.next.store(i, unsafe.Pointer(node)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *uint64NodeDesc) storeNext(i int, value *uint64NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *uint64NodeDesc) atomicLoadNext(i int) *uint64NodeDesc { + return (*uint64NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *uint64NodeDesc) atomicStoreNext(i int, node *uint64NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint64NodeDesc) lessthan(key uint64) bool { @@ -4664,10 +4784,10 @@ func NewUint64Desc() *Uint64MapDesc { func (s *Uint64MapDesc) findNode(key uint64, preds *[maxLevel]*uint64NodeDesc, succs *[maxLevel]*uint64NodeDesc) *uint64NodeDesc { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -4686,10 +4806,10 @@ func (s *Uint64MapDesc) findNodeDelete(key uint64, preds *[maxLevel]*uint64NodeD // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -4748,7 +4868,7 @@ func (s *Uint64MapDesc) Store(key uint64, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint64Desc(preds, highestLocked) @@ -4757,8 +4877,8 @@ func (s *Uint64MapDesc) Store(key uint64, value interface{}) { nn := newUuint64NodeDescDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint64Desc(preds, highestLocked) @@ -4788,10 +4908,10 @@ func (s *Uint64MapDesc) randomlevel() int { func (s *Uint64MapDesc) Load(key uint64) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -4817,7 +4937,7 @@ func (s *Uint64MapDesc) LoadAndDelete(key uint64) (value interface{}, loaded boo for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -4849,7 +4969,7 @@ func (s *Uint64MapDesc) LoadAndDelete(key uint64) (value interface{}, loaded boo // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint64Desc(preds, highestLocked) @@ -4858,7 +4978,7 @@ func (s *Uint64MapDesc) LoadAndDelete(key uint64) (value interface{}, loaded boo for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint64Desc(preds, highestLocked) @@ -4906,7 +5026,7 @@ func (s *Uint64MapDesc) LoadOrStore(key uint64, value interface{}) (actual inter // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint64Desc(preds, highestLocked) @@ -4915,8 +5035,8 @@ func (s *Uint64MapDesc) LoadOrStore(key uint64, value interface{}) (actual inter nn := newUuint64NodeDescDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint64Desc(preds, highestLocked) @@ -4936,7 +5056,7 @@ func (s *Uint64MapDesc) Delete(key uint64) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -4968,7 +5088,7 @@ func (s *Uint64MapDesc) Delete(key uint64) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockUint64Desc(preds, highestLocked) @@ -4977,7 +5097,7 @@ func (s *Uint64MapDesc) Delete(key uint64) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint64Desc(preds, highestLocked) @@ -4996,16 +5116,16 @@ func (s *Uint64MapDesc) Delete(key uint64) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Uint64MapDesc) Range(f func(key uint64, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -5023,19 +5143,23 @@ type Uint32Map struct { type uint32Node struct { key uint32 - value unsafe.Pointer - next []*uint32Node + value unsafe.Pointer // *interface{} + next optionalArray // [level]*uint32Node mu sync.Mutex flags bitflag + level uint32 } func newUint32Node(key uint32, value interface{}, level int) *uint32Node { - n := &uint32Node{ - key: key, - next: make([]*uint32Node, level), + node := &uint32Node{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *uint32Node) storeVal(value interface{}) { @@ -5046,14 +5170,20 @@ func (n *uint32Node) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *uint32Node) loadNext(i int) *uint32Node { - return (*uint32Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint32Node)(n.next.load(i)) +} + +func (n *uint32Node) storeNext(i int, node *uint32Node) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uint32Node) atomicLoadNext(i int) *uint32Node { + return (*uint32Node)(n.next.atomicLoad(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *uint32Node) storeNext(i int, value *uint32Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *uint32Node) atomicStoreNext(i int, node *uint32Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint32Node) lessthan(key uint32) bool { @@ -5080,10 +5210,10 @@ func NewUint32() *Uint32Map { func (s *Uint32Map) findNode(key uint32, preds *[maxLevel]*uint32Node, succs *[maxLevel]*uint32Node) *uint32Node { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -5102,10 +5232,10 @@ func (s *Uint32Map) findNodeDelete(key uint32, preds *[maxLevel]*uint32Node, suc // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -5164,7 +5294,7 @@ func (s *Uint32Map) Store(key uint32, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint32(preds, highestLocked) @@ -5173,8 +5303,8 @@ func (s *Uint32Map) Store(key uint32, value interface{}) { nn := newUint32Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint32(preds, highestLocked) @@ -5204,10 +5334,10 @@ func (s *Uint32Map) randomlevel() int { func (s *Uint32Map) Load(key uint32) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -5233,7 +5363,7 @@ func (s *Uint32Map) LoadAndDelete(key uint32) (value interface{}, loaded bool) { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -5265,7 +5395,7 @@ func (s *Uint32Map) LoadAndDelete(key uint32) (value interface{}, loaded bool) { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint32(preds, highestLocked) @@ -5274,7 +5404,7 @@ func (s *Uint32Map) LoadAndDelete(key uint32) (value interface{}, loaded bool) { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint32(preds, highestLocked) @@ -5322,7 +5452,7 @@ func (s *Uint32Map) LoadOrStore(key uint32, value interface{}) (actual interface // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint32(preds, highestLocked) @@ -5331,8 +5461,8 @@ func (s *Uint32Map) LoadOrStore(key uint32, value interface{}) (actual interface nn := newUint32Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint32(preds, highestLocked) @@ -5352,7 +5482,7 @@ func (s *Uint32Map) Delete(key uint32) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -5384,7 +5514,7 @@ func (s *Uint32Map) Delete(key uint32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockUint32(preds, highestLocked) @@ -5393,7 +5523,7 @@ func (s *Uint32Map) Delete(key uint32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint32(preds, highestLocked) @@ -5412,16 +5542,16 @@ func (s *Uint32Map) Delete(key uint32) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Uint32Map) Range(f func(key uint32, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -5439,19 +5569,23 @@ type Uint32MapDesc struct { type uint32NodeDesc struct { key uint32 - value unsafe.Pointer - next []*uint32NodeDesc + value unsafe.Pointer // *interface{} + next optionalArray // [level]*uint32NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newUint32NodeDesc(key uint32, value interface{}, level int) *uint32NodeDesc { - n := &uint32NodeDesc{ - key: key, - next: make([]*uint32NodeDesc, level), + node := &uint32NodeDesc{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *uint32NodeDesc) storeVal(value interface{}) { @@ -5462,14 +5596,20 @@ func (n *uint32NodeDesc) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *uint32NodeDesc) loadNext(i int) *uint32NodeDesc { - return (*uint32NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint32NodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *uint32NodeDesc) storeNext(i int, value *uint32NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *uint32NodeDesc) storeNext(i int, node *uint32NodeDesc) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uint32NodeDesc) atomicLoadNext(i int) *uint32NodeDesc { + return (*uint32NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *uint32NodeDesc) atomicStoreNext(i int, node *uint32NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint32NodeDesc) lessthan(key uint32) bool { @@ -5496,10 +5636,10 @@ func NewUint32Desc() *Uint32MapDesc { func (s *Uint32MapDesc) findNode(key uint32, preds *[maxLevel]*uint32NodeDesc, succs *[maxLevel]*uint32NodeDesc) *uint32NodeDesc { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -5518,10 +5658,10 @@ func (s *Uint32MapDesc) findNodeDelete(key uint32, preds *[maxLevel]*uint32NodeD // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -5580,7 +5720,7 @@ func (s *Uint32MapDesc) Store(key uint32, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint32Desc(preds, highestLocked) @@ -5589,8 +5729,8 @@ func (s *Uint32MapDesc) Store(key uint32, value interface{}) { nn := newUint32NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint32Desc(preds, highestLocked) @@ -5620,10 +5760,10 @@ func (s *Uint32MapDesc) randomlevel() int { func (s *Uint32MapDesc) Load(key uint32) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -5649,7 +5789,7 @@ func (s *Uint32MapDesc) LoadAndDelete(key uint32) (value interface{}, loaded boo for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -5681,7 +5821,7 @@ func (s *Uint32MapDesc) LoadAndDelete(key uint32) (value interface{}, loaded boo // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint32Desc(preds, highestLocked) @@ -5690,7 +5830,7 @@ func (s *Uint32MapDesc) LoadAndDelete(key uint32) (value interface{}, loaded boo for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint32Desc(preds, highestLocked) @@ -5738,7 +5878,7 @@ func (s *Uint32MapDesc) LoadOrStore(key uint32, value interface{}) (actual inter // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint32Desc(preds, highestLocked) @@ -5747,8 +5887,8 @@ func (s *Uint32MapDesc) LoadOrStore(key uint32, value interface{}) (actual inter nn := newUint32NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint32Desc(preds, highestLocked) @@ -5768,7 +5908,7 @@ func (s *Uint32MapDesc) Delete(key uint32) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -5800,7 +5940,7 @@ func (s *Uint32MapDesc) Delete(key uint32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockUint32Desc(preds, highestLocked) @@ -5809,7 +5949,7 @@ func (s *Uint32MapDesc) Delete(key uint32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint32Desc(preds, highestLocked) @@ -5828,16 +5968,16 @@ func (s *Uint32MapDesc) Delete(key uint32) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Uint32MapDesc) Range(f func(key uint32, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -5855,19 +5995,23 @@ type Uint16Map struct { type uint16Node struct { key uint16 - value unsafe.Pointer - next []*uint16Node + value unsafe.Pointer // *interface{} + next optionalArray // [level]*uint16Node mu sync.Mutex flags bitflag + level uint32 } func newUint16Node(key uint16, value interface{}, level int) *uint16Node { - n := &uint16Node{ - key: key, - next: make([]*uint16Node, level), + node := &uint16Node{ + key: key, + level: uint32(level), } - n.storeVal(value) - return n + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } func (n *uint16Node) storeVal(value interface{}) { @@ -5878,14 +6022,20 @@ func (n *uint16Node) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *uint16Node) loadNext(i int) *uint16Node { - return (*uint16Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint16Node)(n.next.load(i)) +} + +func (n *uint16Node) storeNext(i int, node *uint16Node) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uint16Node) atomicLoadNext(i int) *uint16Node { + return (*uint16Node)(n.next.atomicLoad(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *uint16Node) storeNext(i int, value *uint16Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *uint16Node) atomicStoreNext(i int, node *uint16Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint16Node) lessthan(key uint16) bool { @@ -5912,10 +6062,10 @@ func NewUint16() *Uint16Map { func (s *Uint16Map) findNode(key uint16, preds *[maxLevel]*uint16Node, succs *[maxLevel]*uint16Node) *uint16Node { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -5934,10 +6084,10 @@ func (s *Uint16Map) findNodeDelete(key uint16, preds *[maxLevel]*uint16Node, suc // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -5996,7 +6146,7 @@ func (s *Uint16Map) Store(key uint16, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint16(preds, highestLocked) @@ -6005,8 +6155,8 @@ func (s *Uint16Map) Store(key uint16, value interface{}) { nn := newUint16Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint16(preds, highestLocked) @@ -6036,10 +6186,10 @@ func (s *Uint16Map) randomlevel() int { func (s *Uint16Map) Load(key uint16) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -6065,7 +6215,7 @@ func (s *Uint16Map) LoadAndDelete(key uint16) (value interface{}, loaded bool) { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -6097,7 +6247,7 @@ func (s *Uint16Map) LoadAndDelete(key uint16) (value interface{}, loaded bool) { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint16(preds, highestLocked) @@ -6106,7 +6256,7 @@ func (s *Uint16Map) LoadAndDelete(key uint16) (value interface{}, loaded bool) { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint16(preds, highestLocked) @@ -6154,7 +6304,7 @@ func (s *Uint16Map) LoadOrStore(key uint16, value interface{}) (actual interface // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint16(preds, highestLocked) @@ -6163,8 +6313,8 @@ func (s *Uint16Map) LoadOrStore(key uint16, value interface{}) (actual interface nn := newUint16Node(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint16(preds, highestLocked) @@ -6184,7 +6334,7 @@ func (s *Uint16Map) Delete(key uint16) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -6216,7 +6366,7 @@ func (s *Uint16Map) Delete(key uint16) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockUint16(preds, highestLocked) @@ -6225,7 +6375,7 @@ func (s *Uint16Map) Delete(key uint16) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint16(preds, highestLocked) @@ -6244,16 +6394,16 @@ func (s *Uint16Map) Delete(key uint16) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Uint16Map) Range(f func(key uint16, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -6271,19 +6421,23 @@ type Uint16MapDesc struct { type uint16NodeDesc struct { key uint16 - value unsafe.Pointer - next []*uint16NodeDesc + value unsafe.Pointer // *interface{} + next optionalArray // [level]*uint16NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newUint16NodeDesc(key uint16, value interface{}, level int) *uint16NodeDesc { - n := &uint16NodeDesc{ - key: key, - next: make([]*uint16NodeDesc, level), + node := &uint16NodeDesc{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *uint16NodeDesc) storeVal(value interface{}) { @@ -6294,14 +6448,20 @@ func (n *uint16NodeDesc) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *uint16NodeDesc) loadNext(i int) *uint16NodeDesc { - return (*uint16NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint16NodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *uint16NodeDesc) storeNext(i int, value *uint16NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *uint16NodeDesc) storeNext(i int, node *uint16NodeDesc) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uint16NodeDesc) atomicLoadNext(i int) *uint16NodeDesc { + return (*uint16NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *uint16NodeDesc) atomicStoreNext(i int, node *uint16NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint16NodeDesc) lessthan(key uint16) bool { @@ -6328,10 +6488,10 @@ func NewUint16Desc() *Uint16MapDesc { func (s *Uint16MapDesc) findNode(key uint16, preds *[maxLevel]*uint16NodeDesc, succs *[maxLevel]*uint16NodeDesc) *uint16NodeDesc { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -6350,10 +6510,10 @@ func (s *Uint16MapDesc) findNodeDelete(key uint16, preds *[maxLevel]*uint16NodeD // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -6412,7 +6572,7 @@ func (s *Uint16MapDesc) Store(key uint16, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint16Desc(preds, highestLocked) @@ -6421,8 +6581,8 @@ func (s *Uint16MapDesc) Store(key uint16, value interface{}) { nn := newUint16NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint16Desc(preds, highestLocked) @@ -6452,10 +6612,10 @@ func (s *Uint16MapDesc) randomlevel() int { func (s *Uint16MapDesc) Load(key uint16) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -6481,7 +6641,7 @@ func (s *Uint16MapDesc) LoadAndDelete(key uint16) (value interface{}, loaded boo for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -6513,7 +6673,7 @@ func (s *Uint16MapDesc) LoadAndDelete(key uint16) (value interface{}, loaded boo // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint16Desc(preds, highestLocked) @@ -6522,7 +6682,7 @@ func (s *Uint16MapDesc) LoadAndDelete(key uint16) (value interface{}, loaded boo for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint16Desc(preds, highestLocked) @@ -6570,7 +6730,7 @@ func (s *Uint16MapDesc) LoadOrStore(key uint16, value interface{}) (actual inter // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint16Desc(preds, highestLocked) @@ -6579,8 +6739,8 @@ func (s *Uint16MapDesc) LoadOrStore(key uint16, value interface{}) (actual inter nn := newUint16NodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint16Desc(preds, highestLocked) @@ -6600,7 +6760,7 @@ func (s *Uint16MapDesc) Delete(key uint16) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -6632,7 +6792,7 @@ func (s *Uint16MapDesc) Delete(key uint16) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockUint16Desc(preds, highestLocked) @@ -6641,7 +6801,7 @@ func (s *Uint16MapDesc) Delete(key uint16) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint16Desc(preds, highestLocked) @@ -6660,16 +6820,16 @@ func (s *Uint16MapDesc) Delete(key uint16) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *Uint16MapDesc) Range(f func(key uint16, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -6687,19 +6847,23 @@ type UintMap struct { type uintNode struct { key uint - value unsafe.Pointer - next []*uintNode + value unsafe.Pointer // *interface{} + next optionalArray // [level]*uintNode mu sync.Mutex flags bitflag + level uint32 } func newUintNode(key uint, value interface{}, level int) *uintNode { - n := &uintNode{ - key: key, - next: make([]*uintNode, level), + node := &uintNode{ + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *uintNode) storeVal(value interface{}) { @@ -6710,14 +6874,20 @@ func (n *uintNode) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *uintNode) loadNext(i int) *uintNode { - return (*uintNode)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uintNode)(n.next.load(i)) +} + +func (n *uintNode) storeNext(i int, node *uintNode) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uintNode) atomicLoadNext(i int) *uintNode { + return (*uintNode)(n.next.atomicLoad(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *uintNode) storeNext(i int, value *uintNode) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *uintNode) atomicStoreNext(i int, node *uintNode) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uintNode) lessthan(key uint) bool { @@ -6744,10 +6914,10 @@ func NewUint() *UintMap { func (s *UintMap) findNode(key uint, preds *[maxLevel]*uintNode, succs *[maxLevel]*uintNode) *uintNode { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -6766,10 +6936,10 @@ func (s *UintMap) findNodeDelete(key uint, preds *[maxLevel]*uintNode, succs *[m // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -6828,7 +6998,7 @@ func (s *UintMap) Store(key uint, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint(preds, highestLocked) @@ -6837,8 +7007,8 @@ func (s *UintMap) Store(key uint, value interface{}) { nn := newUintNode(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint(preds, highestLocked) @@ -6868,10 +7038,10 @@ func (s *UintMap) randomlevel() int { func (s *UintMap) Load(key uint) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -6897,7 +7067,7 @@ func (s *UintMap) LoadAndDelete(key uint) (value interface{}, loaded bool) { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -6929,7 +7099,7 @@ func (s *UintMap) LoadAndDelete(key uint) (value interface{}, loaded bool) { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint(preds, highestLocked) @@ -6938,7 +7108,7 @@ func (s *UintMap) LoadAndDelete(key uint) (value interface{}, loaded bool) { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint(preds, highestLocked) @@ -6986,7 +7156,7 @@ func (s *UintMap) LoadOrStore(key uint, value interface{}) (actual interface{}, // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint(preds, highestLocked) @@ -6995,8 +7165,8 @@ func (s *UintMap) LoadOrStore(key uint, value interface{}) (actual interface{}, nn := newUintNode(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint(preds, highestLocked) @@ -7016,7 +7186,7 @@ func (s *UintMap) Delete(key uint) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -7048,7 +7218,7 @@ func (s *UintMap) Delete(key uint) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockUint(preds, highestLocked) @@ -7057,7 +7227,7 @@ func (s *UintMap) Delete(key uint) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUint(preds, highestLocked) @@ -7076,16 +7246,16 @@ func (s *UintMap) Delete(key uint) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *UintMap) Range(f func(key uint, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -7103,19 +7273,23 @@ type UintMapDesc struct { type uintNodeDesc struct { key uint - value unsafe.Pointer - next []*uintNodeDesc + value unsafe.Pointer // *interface{} + next optionalArray // [level]*uintNodeDesc mu sync.Mutex flags bitflag + level uint32 } func newUintNodeDesc(key uint, value interface{}, level int) *uintNodeDesc { - n := &uintNodeDesc{ - key: key, - next: make([]*uintNodeDesc, level), + node := &uintNodeDesc{ + key: key, + level: uint32(level), } - n.storeVal(value) - return n + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } func (n *uintNodeDesc) storeVal(value interface{}) { @@ -7126,14 +7300,20 @@ func (n *uintNodeDesc) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *uintNodeDesc) loadNext(i int) *uintNodeDesc { - return (*uintNodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uintNodeDesc)(n.next.load(i)) +} + +func (n *uintNodeDesc) storeNext(i int, node *uintNodeDesc) { + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uintNodeDesc) atomicLoadNext(i int) *uintNodeDesc { + return (*uintNodeDesc)(n.next.atomicLoad(i)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *uintNodeDesc) storeNext(i int, value *uintNodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *uintNodeDesc) atomicStoreNext(i int, node *uintNodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uintNodeDesc) lessthan(key uint) bool { @@ -7160,10 +7340,10 @@ func NewUintDesc() *UintMapDesc { func (s *UintMapDesc) findNode(key uint, preds *[maxLevel]*uintNodeDesc, succs *[maxLevel]*uintNodeDesc) *uintNodeDesc { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -7182,10 +7362,10 @@ func (s *UintMapDesc) findNodeDelete(key uint, preds *[maxLevel]*uintNodeDesc, s // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(key) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -7244,7 +7424,7 @@ func (s *UintMapDesc) Store(key uint, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUintDesc(preds, highestLocked) @@ -7253,8 +7433,8 @@ func (s *UintMapDesc) Store(key uint, value interface{}) { nn := newUintNodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUintDesc(preds, highestLocked) @@ -7284,10 +7464,10 @@ func (s *UintMapDesc) randomlevel() int { func (s *UintMapDesc) Load(key uint) (value interface{}, ok bool) { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(key) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -7313,7 +7493,7 @@ func (s *UintMapDesc) LoadAndDelete(key uint) (value interface{}, loaded bool) { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -7345,7 +7525,7 @@ func (s *UintMapDesc) LoadAndDelete(key uint) (value interface{}, loaded bool) { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUintDesc(preds, highestLocked) @@ -7354,7 +7534,7 @@ func (s *UintMapDesc) LoadAndDelete(key uint) (value interface{}, loaded bool) { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUintDesc(preds, highestLocked) @@ -7402,7 +7582,7 @@ func (s *UintMapDesc) LoadOrStore(key uint, value interface{}) (actual interface // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUintDesc(preds, highestLocked) @@ -7411,8 +7591,8 @@ func (s *UintMapDesc) LoadOrStore(key uint, value interface{}) (actual interface nn := newUintNodeDesc(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUintDesc(preds, highestLocked) @@ -7432,7 +7612,7 @@ func (s *UintMapDesc) Delete(key uint) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -7464,7 +7644,7 @@ func (s *UintMapDesc) Delete(key uint) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockUintDesc(preds, highestLocked) @@ -7473,7 +7653,7 @@ func (s *UintMapDesc) Delete(key uint) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockUintDesc(preds, highestLocked) @@ -7492,16 +7672,16 @@ func (s *UintMapDesc) Delete(key uint) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *UintMapDesc) Range(f func(key uint, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -7520,20 +7700,24 @@ type StringMap struct { type stringNode struct { key string score uint64 - value unsafe.Pointer - next []*stringNode + value unsafe.Pointer // *interface{} + next optionalArray // [level]*stringNode mu sync.Mutex flags bitflag + level uint32 } func newStringNode(key string, value interface{}, level int) *stringNode { - n := &stringNode{ - key: key, + node := &stringNode{ score: hash(key), - next: make([]*stringNode, level), + key: key, + level: uint32(level), + } + node.storeVal(value) + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } - n.storeVal(value) - return n + return node } func (n *stringNode) storeVal(value interface{}) { @@ -7544,14 +7728,20 @@ func (n *stringNode) loadVal() interface{} { return *(*interface{})(atomic.LoadPointer(&n.value)) } -// loadNext return `n.next[i]`(atomic) func (n *stringNode) loadNext(i int) *stringNode { - return (*stringNode)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*stringNode)(n.next.load(i)) +} + +func (n *stringNode) storeNext(i int, node *stringNode) { + n.next.store(i, unsafe.Pointer(node)) } -// storeNext same with `n.next[i] = value`(atomic) -func (n *stringNode) storeNext(i int, value *stringNode) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(value)) +func (n *stringNode) atomicLoadNext(i int) *stringNode { + return (*stringNode)(n.next.atomicLoad(i)) +} + +func (n *stringNode) atomicStoreNext(i int, node *stringNode) { + n.next.atomicStore(i, unsafe.Pointer(node)) } // NewString return an empty int64 skipmap. @@ -7571,10 +7761,10 @@ func (s *StringMap) findNode(key string, preds *[maxLevel]*stringNode, succs *[m score := hash(key) x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.cmp(score, key) < 0 { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -7594,10 +7784,10 @@ func (s *StringMap) findNodeDelete(key string, preds *[maxLevel]*stringNode, suc // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.cmp(score, key) < 0 { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -7656,7 +7846,7 @@ func (s *StringMap) Store(key string, value interface{}) { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockString(preds, highestLocked) @@ -7665,8 +7855,8 @@ func (s *StringMap) Store(key string, value interface{}) { nn := newStringNode(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockString(preds, highestLocked) @@ -7697,10 +7887,10 @@ func (s *StringMap) Load(key string) (value interface{}, ok bool) { score := hash(key) x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.cmp(score, key) < 0 { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the key already in the skip list. @@ -7726,7 +7916,7 @@ func (s *StringMap) LoadAndDelete(key string) (value interface{}, loaded bool) { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -7758,7 +7948,7 @@ func (s *StringMap) LoadAndDelete(key string) (value interface{}, loaded bool) { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockString(preds, highestLocked) @@ -7767,7 +7957,7 @@ func (s *StringMap) LoadAndDelete(key string) (value interface{}, loaded bool) { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockString(preds, highestLocked) @@ -7815,7 +8005,7 @@ func (s *StringMap) LoadOrStore(key string, value interface{}) (actual interface // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockString(preds, highestLocked) @@ -7824,8 +8014,8 @@ func (s *StringMap) LoadOrStore(key string, value interface{}) (actual interface nn := newStringNode(key, value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockString(preds, highestLocked) @@ -7845,7 +8035,7 @@ func (s *StringMap) Delete(key string) bool { for { lFound := s.findNodeDelete(key, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToDelete = succs[lFound] topLayer = lFound @@ -7877,7 +8067,7 @@ func (s *StringMap) Delete(key string) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ + valid = !pred.flags.Get(marked) && pred.atomicLoadNext(layer) == succ } if !valid { unlockString(preds, highestLocked) @@ -7886,7 +8076,7 @@ func (s *StringMap) Delete(key string) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToDelete`, no other goroutine will modify it. // So we don't need `nodeToDelete.loadNext` - preds[i].storeNext(i, nodeToDelete.next[i]) + preds[i].atomicStoreNext(i, nodeToDelete.loadNext(i)) } nodeToDelete.mu.Unlock() unlockString(preds, highestLocked) @@ -7905,16 +8095,16 @@ func (s *StringMap) Delete(key string) bool { // is stored or deleted concurrently, Range may reflect any mapping for that key // from any point during the Range call. func (s *StringMap) Range(f func(key string, value interface{}) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.key, x.loadVal()) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } diff --git a/collection/skipmap/types_gen.go b/collection/skipmap/types_gen.go index ba271181..c89ecded 100644 --- a/collection/skipmap/types_gen.go +++ b/collection/skipmap/types_gen.go @@ -125,10 +125,8 @@ func replaceString(data string) string { score uint64`, -1) data = strings.Replace(data, + `&int64Node{`, `&int64Node{ - key: key,`, - `&int64Node{ - key: key, score: hash(key),`, -1) // Refactor comparsion. diff --git a/collection/skipset/oparry.go b/collection/skipset/oparry.go new file mode 100644 index 00000000..9b3ad902 --- /dev/null +++ b/collection/skipset/oparry.go @@ -0,0 +1,60 @@ +// Copyright 2021 ByteDance Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package skipset + +import ( + "sync/atomic" + "unsafe" +) + +const ( + op1 = 4 + op2 = maxLevel - op1 +) + +type optionalArray struct { + base [op1]unsafe.Pointer + extra *([op2]unsafe.Pointer) +} + +func (a *optionalArray) load(i int) unsafe.Pointer { + if i < op1 { + return a.base[i] + } + return a.extra[i-op1] +} + +func (a *optionalArray) store(i int, p unsafe.Pointer) { + if i < op1 { + a.base[i] = p + return + } + a.extra[i-op1] = p +} + +func (a *optionalArray) atomicLoad(i int) unsafe.Pointer { + if i < op1 { + return atomic.LoadPointer(&a.base[i]) + } + return atomic.LoadPointer(&a.extra[i-op1]) +} + +func (a *optionalArray) atomicStore(i int, p unsafe.Pointer) { + if i < op1 { + atomic.StorePointer(&a.base[i], p) + return + } + atomic.StorePointer(&a.extra[i-op1], p) +} diff --git a/collection/skipset/oparry_test.go b/collection/skipset/oparry_test.go new file mode 100644 index 00000000..23711260 --- /dev/null +++ b/collection/skipset/oparry_test.go @@ -0,0 +1,64 @@ +// Copyright 2021 ByteDance Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package skipset + +import ( + "testing" + "unsafe" + + "github.com/bytedance/gopkg/lang/fastrand" +) + +type dummy struct { + data optionalArray +} + +func TestOpArray(t *testing.T) { + n := new(dummy) + n.data.extra = new([op2]unsafe.Pointer) + + var array [maxLevel]unsafe.Pointer + for i := 0; i < maxLevel; i++ { + value := unsafe.Pointer(&dummy{}) + array[i] = value + n.data.store(i, value) + } + + for i := 0; i < maxLevel; i++ { + if array[i] != n.data.load(i) || array[i] != n.data.atomicLoad(i) { + t.Fatal(i, array[i], n.data.load(i)) + } + } + + for i := 0; i < 1000; i++ { + r := int(fastrand.Uint32n(maxLevel)) + value := unsafe.Pointer(&dummy{}) + if i%100 == 0 { + value = nil + } + array[r] = value + if fastrand.Uint32n(2) == 0 { + n.data.store(r, value) + } else { + n.data.atomicStore(r, value) + } + } + + for i := 0; i < maxLevel; i++ { + if array[i] != n.data.load(i) || array[i] != n.data.atomicLoad(i) { + t.Fatal(i, array[i], n.data.load(i)) + } + } +} diff --git a/collection/skipset/skipset.go b/collection/skipset/skipset.go index 935ddf3e..970de9e1 100644 --- a/collection/skipset/skipset.go +++ b/collection/skipset/skipset.go @@ -32,26 +32,37 @@ type Int64Set struct { type int64Node struct { value int64 - next []*int64Node + next optionalArray // [level]*int64Node mu sync.Mutex flags bitflag + level uint32 } func newInt64Node(value int64, level int) *int64Node { - return &int64Node{ + node := &int64Node{ value: value, - next: make([]*int64Node, level), + level: uint32(level), } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } -// loadNext return `n.next[i]`(atomic) func (n *int64Node) loadNext(i int) *int64Node { - return (*int64Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*int64Node)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *int64Node) storeNext(i int, node *int64Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *int64Node) atomicLoadNext(i int) *int64Node { + return (*int64Node)(n.next.atomicLoad(i)) +} + +func (n *int64Node) atomicStoreNext(i int, node *int64Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *int64Node) lessthan(value int64) bool { @@ -78,10 +89,10 @@ func (s *Int64Set) findNodeRemove(value int64, preds *[maxLevel]*int64Node, succ // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -99,10 +110,10 @@ func (s *Int64Set) findNodeRemove(value int64, preds *[maxLevel]*int64Node, succ func (s *Int64Set) findNodeAdd(value int64, preds *[maxLevel]*int64Node, succs *[maxLevel]*int64Node) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -164,7 +175,7 @@ func (s *Int64Set) Add(value int64) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt64(preds, highestLocked) @@ -173,8 +184,8 @@ func (s *Int64Set) Add(value int64) bool { nn := newInt64Node(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt64(preds, highestLocked) @@ -203,10 +214,10 @@ func (s *Int64Set) randomlevel() int { func (s *Int64Set) Contains(value int64) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -228,7 +239,7 @@ func (s *Int64Set) Remove(value int64) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -260,7 +271,7 @@ func (s *Int64Set) Remove(value int64) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt64(preds, highestLocked) @@ -269,7 +280,7 @@ func (s *Int64Set) Remove(value int64) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockInt64(preds, highestLocked) @@ -283,16 +294,16 @@ func (s *Int64Set) Remove(value int64) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Int64Set) Range(f func(value int64) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } diff --git a/collection/skipset/types.go b/collection/skipset/types.go index f665dff4..2ead6a29 100644 --- a/collection/skipset/types.go +++ b/collection/skipset/types.go @@ -31,26 +31,37 @@ type Float32Set struct { type float32Node struct { value float32 - next []*float32Node + next optionalArray // [level]*float32Node mu sync.Mutex flags bitflag + level uint32 } func newFloat32Node(value float32, level int) *float32Node { - return &float32Node{ + node := &float32Node{ value: value, - next: make([]*float32Node, level), + level: uint32(level), } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } -// loadNext return `n.next[i]`(atomic) func (n *float32Node) loadNext(i int) *float32Node { - return (*float32Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*float32Node)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *float32Node) storeNext(i int, node *float32Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *float32Node) atomicLoadNext(i int) *float32Node { + return (*float32Node)(n.next.atomicLoad(i)) +} + +func (n *float32Node) atomicStoreNext(i int, node *float32Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *float32Node) lessthan(value float32) bool { @@ -77,10 +88,10 @@ func (s *Float32Set) findNodeRemove(value float32, preds *[maxLevel]*float32Node // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -98,10 +109,10 @@ func (s *Float32Set) findNodeRemove(value float32, preds *[maxLevel]*float32Node func (s *Float32Set) findNodeAdd(value float32, preds *[maxLevel]*float32Node, succs *[maxLevel]*float32Node) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -163,7 +174,7 @@ func (s *Float32Set) Add(value float32) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat32(preds, highestLocked) @@ -172,8 +183,8 @@ func (s *Float32Set) Add(value float32) bool { nn := newFloat32Node(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat32(preds, highestLocked) @@ -202,10 +213,10 @@ func (s *Float32Set) randomlevel() int { func (s *Float32Set) Contains(value float32) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -227,7 +238,7 @@ func (s *Float32Set) Remove(value float32) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -259,7 +270,7 @@ func (s *Float32Set) Remove(value float32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockFloat32(preds, highestLocked) @@ -268,7 +279,7 @@ func (s *Float32Set) Remove(value float32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockFloat32(preds, highestLocked) @@ -282,16 +293,16 @@ func (s *Float32Set) Remove(value float32) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Float32Set) Range(f func(value float32) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -309,26 +320,37 @@ type Float32SetDesc struct { type float32NodeDesc struct { value float32 - next []*float32NodeDesc + next optionalArray // [level]*float32NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newFloat32NodeDesc(value float32, level int) *float32NodeDesc { - return &float32NodeDesc{ + node := &float32NodeDesc{ value: value, - next: make([]*float32NodeDesc, level), + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *float32NodeDesc) loadNext(i int) *float32NodeDesc { - return (*float32NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*float32NodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *float32NodeDesc) storeNext(i int, node *float32NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *float32NodeDesc) atomicLoadNext(i int) *float32NodeDesc { + return (*float32NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *float32NodeDesc) atomicStoreNext(i int, node *float32NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *float32NodeDesc) lessthan(value float32) bool { @@ -355,10 +377,10 @@ func (s *Float32SetDesc) findNodeRemove(value float32, preds *[maxLevel]*float32 // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -376,10 +398,10 @@ func (s *Float32SetDesc) findNodeRemove(value float32, preds *[maxLevel]*float32 func (s *Float32SetDesc) findNodeAdd(value float32, preds *[maxLevel]*float32NodeDesc, succs *[maxLevel]*float32NodeDesc) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -441,7 +463,7 @@ func (s *Float32SetDesc) Add(value float32) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat32Desc(preds, highestLocked) @@ -450,8 +472,8 @@ func (s *Float32SetDesc) Add(value float32) bool { nn := newFloat32NodeDesc(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat32Desc(preds, highestLocked) @@ -480,10 +502,10 @@ func (s *Float32SetDesc) randomlevel() int { func (s *Float32SetDesc) Contains(value float32) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -505,7 +527,7 @@ func (s *Float32SetDesc) Remove(value float32) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -537,7 +559,7 @@ func (s *Float32SetDesc) Remove(value float32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockFloat32Desc(preds, highestLocked) @@ -546,7 +568,7 @@ func (s *Float32SetDesc) Remove(value float32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockFloat32Desc(preds, highestLocked) @@ -560,16 +582,16 @@ func (s *Float32SetDesc) Remove(value float32) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Float32SetDesc) Range(f func(value float32) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -587,26 +609,37 @@ type Float64Set struct { type float64Node struct { value float64 - next []*float64Node + next optionalArray // [level]*float64Node mu sync.Mutex flags bitflag + level uint32 } func newFloat64Node(value float64, level int) *float64Node { - return &float64Node{ + node := &float64Node{ value: value, - next: make([]*float64Node, level), + level: uint32(level), } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } -// loadNext return `n.next[i]`(atomic) func (n *float64Node) loadNext(i int) *float64Node { - return (*float64Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*float64Node)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *float64Node) storeNext(i int, node *float64Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *float64Node) atomicLoadNext(i int) *float64Node { + return (*float64Node)(n.next.atomicLoad(i)) +} + +func (n *float64Node) atomicStoreNext(i int, node *float64Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *float64Node) lessthan(value float64) bool { @@ -633,10 +666,10 @@ func (s *Float64Set) findNodeRemove(value float64, preds *[maxLevel]*float64Node // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -654,10 +687,10 @@ func (s *Float64Set) findNodeRemove(value float64, preds *[maxLevel]*float64Node func (s *Float64Set) findNodeAdd(value float64, preds *[maxLevel]*float64Node, succs *[maxLevel]*float64Node) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -719,7 +752,7 @@ func (s *Float64Set) Add(value float64) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat64(preds, highestLocked) @@ -728,8 +761,8 @@ func (s *Float64Set) Add(value float64) bool { nn := newFloat64Node(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat64(preds, highestLocked) @@ -758,10 +791,10 @@ func (s *Float64Set) randomlevel() int { func (s *Float64Set) Contains(value float64) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -783,7 +816,7 @@ func (s *Float64Set) Remove(value float64) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -815,7 +848,7 @@ func (s *Float64Set) Remove(value float64) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockFloat64(preds, highestLocked) @@ -824,7 +857,7 @@ func (s *Float64Set) Remove(value float64) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockFloat64(preds, highestLocked) @@ -838,16 +871,16 @@ func (s *Float64Set) Remove(value float64) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Float64Set) Range(f func(value float64) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -865,26 +898,37 @@ type Float64SetDesc struct { type float64NodeDesc struct { value float64 - next []*float64NodeDesc + next optionalArray // [level]*float64NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newFloat64NodeDesc(value float64, level int) *float64NodeDesc { - return &float64NodeDesc{ + node := &float64NodeDesc{ value: value, - next: make([]*float64NodeDesc, level), + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *float64NodeDesc) loadNext(i int) *float64NodeDesc { - return (*float64NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*float64NodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *float64NodeDesc) storeNext(i int, node *float64NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *float64NodeDesc) atomicLoadNext(i int) *float64NodeDesc { + return (*float64NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *float64NodeDesc) atomicStoreNext(i int, node *float64NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *float64NodeDesc) lessthan(value float64) bool { @@ -911,10 +955,10 @@ func (s *Float64SetDesc) findNodeRemove(value float64, preds *[maxLevel]*float64 // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -932,10 +976,10 @@ func (s *Float64SetDesc) findNodeRemove(value float64, preds *[maxLevel]*float64 func (s *Float64SetDesc) findNodeAdd(value float64, preds *[maxLevel]*float64NodeDesc, succs *[maxLevel]*float64NodeDesc) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -997,7 +1041,7 @@ func (s *Float64SetDesc) Add(value float64) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockFloat64Desc(preds, highestLocked) @@ -1006,8 +1050,8 @@ func (s *Float64SetDesc) Add(value float64) bool { nn := newFloat64NodeDesc(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockFloat64Desc(preds, highestLocked) @@ -1036,10 +1080,10 @@ func (s *Float64SetDesc) randomlevel() int { func (s *Float64SetDesc) Contains(value float64) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -1061,7 +1105,7 @@ func (s *Float64SetDesc) Remove(value float64) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -1093,7 +1137,7 @@ func (s *Float64SetDesc) Remove(value float64) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockFloat64Desc(preds, highestLocked) @@ -1102,7 +1146,7 @@ func (s *Float64SetDesc) Remove(value float64) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockFloat64Desc(preds, highestLocked) @@ -1116,16 +1160,16 @@ func (s *Float64SetDesc) Remove(value float64) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Float64SetDesc) Range(f func(value float64) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -1143,26 +1187,37 @@ type Int32Set struct { type int32Node struct { value int32 - next []*int32Node + next optionalArray // [level]*int32Node mu sync.Mutex flags bitflag + level uint32 } func newInt32Node(value int32, level int) *int32Node { - return &int32Node{ + node := &int32Node{ value: value, - next: make([]*int32Node, level), + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *int32Node) loadNext(i int) *int32Node { - return (*int32Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*int32Node)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *int32Node) storeNext(i int, node *int32Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *int32Node) atomicLoadNext(i int) *int32Node { + return (*int32Node)(n.next.atomicLoad(i)) +} + +func (n *int32Node) atomicStoreNext(i int, node *int32Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *int32Node) lessthan(value int32) bool { @@ -1189,10 +1244,10 @@ func (s *Int32Set) findNodeRemove(value int32, preds *[maxLevel]*int32Node, succ // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -1210,10 +1265,10 @@ func (s *Int32Set) findNodeRemove(value int32, preds *[maxLevel]*int32Node, succ func (s *Int32Set) findNodeAdd(value int32, preds *[maxLevel]*int32Node, succs *[maxLevel]*int32Node) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -1275,7 +1330,7 @@ func (s *Int32Set) Add(value int32) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt32(preds, highestLocked) @@ -1284,8 +1339,8 @@ func (s *Int32Set) Add(value int32) bool { nn := newInt32Node(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt32(preds, highestLocked) @@ -1314,10 +1369,10 @@ func (s *Int32Set) randomlevel() int { func (s *Int32Set) Contains(value int32) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -1339,7 +1394,7 @@ func (s *Int32Set) Remove(value int32) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -1371,7 +1426,7 @@ func (s *Int32Set) Remove(value int32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt32(preds, highestLocked) @@ -1380,7 +1435,7 @@ func (s *Int32Set) Remove(value int32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockInt32(preds, highestLocked) @@ -1394,16 +1449,16 @@ func (s *Int32Set) Remove(value int32) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Int32Set) Range(f func(value int32) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -1421,26 +1476,37 @@ type Int32SetDesc struct { type int32NodeDesc struct { value int32 - next []*int32NodeDesc + next optionalArray // [level]*int32NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newInt32NodeDesc(value int32, level int) *int32NodeDesc { - return &int32NodeDesc{ + node := &int32NodeDesc{ value: value, - next: make([]*int32NodeDesc, level), + level: uint32(level), } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } -// loadNext return `n.next[i]`(atomic) func (n *int32NodeDesc) loadNext(i int) *int32NodeDesc { - return (*int32NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*int32NodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *int32NodeDesc) storeNext(i int, node *int32NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *int32NodeDesc) atomicLoadNext(i int) *int32NodeDesc { + return (*int32NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *int32NodeDesc) atomicStoreNext(i int, node *int32NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *int32NodeDesc) lessthan(value int32) bool { @@ -1467,10 +1533,10 @@ func (s *Int32SetDesc) findNodeRemove(value int32, preds *[maxLevel]*int32NodeDe // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -1488,10 +1554,10 @@ func (s *Int32SetDesc) findNodeRemove(value int32, preds *[maxLevel]*int32NodeDe func (s *Int32SetDesc) findNodeAdd(value int32, preds *[maxLevel]*int32NodeDesc, succs *[maxLevel]*int32NodeDesc) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -1553,7 +1619,7 @@ func (s *Int32SetDesc) Add(value int32) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt32Desc(preds, highestLocked) @@ -1562,8 +1628,8 @@ func (s *Int32SetDesc) Add(value int32) bool { nn := newInt32NodeDesc(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt32Desc(preds, highestLocked) @@ -1592,10 +1658,10 @@ func (s *Int32SetDesc) randomlevel() int { func (s *Int32SetDesc) Contains(value int32) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -1617,7 +1683,7 @@ func (s *Int32SetDesc) Remove(value int32) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -1649,7 +1715,7 @@ func (s *Int32SetDesc) Remove(value int32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt32Desc(preds, highestLocked) @@ -1658,7 +1724,7 @@ func (s *Int32SetDesc) Remove(value int32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockInt32Desc(preds, highestLocked) @@ -1672,16 +1738,16 @@ func (s *Int32SetDesc) Remove(value int32) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Int32SetDesc) Range(f func(value int32) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -1699,26 +1765,37 @@ type Int16Set struct { type int16Node struct { value int16 - next []*int16Node + next optionalArray // [level]*int16Node mu sync.Mutex flags bitflag + level uint32 } func newInt16Node(value int16, level int) *int16Node { - return &int16Node{ + node := &int16Node{ value: value, - next: make([]*int16Node, level), + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *int16Node) loadNext(i int) *int16Node { - return (*int16Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*int16Node)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *int16Node) storeNext(i int, node *int16Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *int16Node) atomicLoadNext(i int) *int16Node { + return (*int16Node)(n.next.atomicLoad(i)) +} + +func (n *int16Node) atomicStoreNext(i int, node *int16Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *int16Node) lessthan(value int16) bool { @@ -1745,10 +1822,10 @@ func (s *Int16Set) findNodeRemove(value int16, preds *[maxLevel]*int16Node, succ // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -1766,10 +1843,10 @@ func (s *Int16Set) findNodeRemove(value int16, preds *[maxLevel]*int16Node, succ func (s *Int16Set) findNodeAdd(value int16, preds *[maxLevel]*int16Node, succs *[maxLevel]*int16Node) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -1831,7 +1908,7 @@ func (s *Int16Set) Add(value int16) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt16(preds, highestLocked) @@ -1840,8 +1917,8 @@ func (s *Int16Set) Add(value int16) bool { nn := newInt16Node(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt16(preds, highestLocked) @@ -1870,10 +1947,10 @@ func (s *Int16Set) randomlevel() int { func (s *Int16Set) Contains(value int16) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -1895,7 +1972,7 @@ func (s *Int16Set) Remove(value int16) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -1927,7 +2004,7 @@ func (s *Int16Set) Remove(value int16) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt16(preds, highestLocked) @@ -1936,7 +2013,7 @@ func (s *Int16Set) Remove(value int16) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockInt16(preds, highestLocked) @@ -1950,16 +2027,16 @@ func (s *Int16Set) Remove(value int16) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Int16Set) Range(f func(value int16) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -1977,26 +2054,37 @@ type Int16SetDesc struct { type int16NodeDesc struct { value int16 - next []*int16NodeDesc + next optionalArray // [level]*int16NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newInt16NodeDesc(value int16, level int) *int16NodeDesc { - return &int16NodeDesc{ + node := &int16NodeDesc{ value: value, - next: make([]*int16NodeDesc, level), + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *int16NodeDesc) loadNext(i int) *int16NodeDesc { - return (*int16NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*int16NodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *int16NodeDesc) storeNext(i int, node *int16NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *int16NodeDesc) atomicLoadNext(i int) *int16NodeDesc { + return (*int16NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *int16NodeDesc) atomicStoreNext(i int, node *int16NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *int16NodeDesc) lessthan(value int16) bool { @@ -2023,10 +2111,10 @@ func (s *Int16SetDesc) findNodeRemove(value int16, preds *[maxLevel]*int16NodeDe // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2044,10 +2132,10 @@ func (s *Int16SetDesc) findNodeRemove(value int16, preds *[maxLevel]*int16NodeDe func (s *Int16SetDesc) findNodeAdd(value int16, preds *[maxLevel]*int16NodeDesc, succs *[maxLevel]*int16NodeDesc) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2109,7 +2197,7 @@ func (s *Int16SetDesc) Add(value int16) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt16Desc(preds, highestLocked) @@ -2118,8 +2206,8 @@ func (s *Int16SetDesc) Add(value int16) bool { nn := newInt16NodeDesc(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt16Desc(preds, highestLocked) @@ -2148,10 +2236,10 @@ func (s *Int16SetDesc) randomlevel() int { func (s *Int16SetDesc) Contains(value int16) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -2173,7 +2261,7 @@ func (s *Int16SetDesc) Remove(value int16) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -2205,7 +2293,7 @@ func (s *Int16SetDesc) Remove(value int16) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt16Desc(preds, highestLocked) @@ -2214,7 +2302,7 @@ func (s *Int16SetDesc) Remove(value int16) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockInt16Desc(preds, highestLocked) @@ -2228,16 +2316,16 @@ func (s *Int16SetDesc) Remove(value int16) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Int16SetDesc) Range(f func(value int16) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -2255,26 +2343,37 @@ type IntSet struct { type intNode struct { value int - next []*intNode + next optionalArray // [level]*intNode mu sync.Mutex flags bitflag + level uint32 } func newIntNode(value int, level int) *intNode { - return &intNode{ + node := &intNode{ value: value, - next: make([]*intNode, level), + level: uint32(level), } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } -// loadNext return `n.next[i]`(atomic) func (n *intNode) loadNext(i int) *intNode { - return (*intNode)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*intNode)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *intNode) storeNext(i int, node *intNode) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *intNode) atomicLoadNext(i int) *intNode { + return (*intNode)(n.next.atomicLoad(i)) +} + +func (n *intNode) atomicStoreNext(i int, node *intNode) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *intNode) lessthan(value int) bool { @@ -2301,10 +2400,10 @@ func (s *IntSet) findNodeRemove(value int, preds *[maxLevel]*intNode, succs *[ma // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2322,10 +2421,10 @@ func (s *IntSet) findNodeRemove(value int, preds *[maxLevel]*intNode, succs *[ma func (s *IntSet) findNodeAdd(value int, preds *[maxLevel]*intNode, succs *[maxLevel]*intNode) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2387,7 +2486,7 @@ func (s *IntSet) Add(value int) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockInt(preds, highestLocked) @@ -2396,8 +2495,8 @@ func (s *IntSet) Add(value int) bool { nn := newIntNode(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockInt(preds, highestLocked) @@ -2426,10 +2525,10 @@ func (s *IntSet) randomlevel() int { func (s *IntSet) Contains(value int) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -2451,7 +2550,7 @@ func (s *IntSet) Remove(value int) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -2483,7 +2582,7 @@ func (s *IntSet) Remove(value int) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockInt(preds, highestLocked) @@ -2492,7 +2591,7 @@ func (s *IntSet) Remove(value int) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockInt(preds, highestLocked) @@ -2506,16 +2605,16 @@ func (s *IntSet) Remove(value int) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *IntSet) Range(f func(value int) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -2533,26 +2632,37 @@ type IntSetDesc struct { type intNodeDesc struct { value int - next []*intNodeDesc + next optionalArray // [level]*intNodeDesc mu sync.Mutex flags bitflag + level uint32 } func newIntNodeDesc(value int, level int) *intNodeDesc { - return &intNodeDesc{ + node := &intNodeDesc{ value: value, - next: make([]*intNodeDesc, level), + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *intNodeDesc) loadNext(i int) *intNodeDesc { - return (*intNodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*intNodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *intNodeDesc) storeNext(i int, node *intNodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *intNodeDesc) atomicLoadNext(i int) *intNodeDesc { + return (*intNodeDesc)(n.next.atomicLoad(i)) +} + +func (n *intNodeDesc) atomicStoreNext(i int, node *intNodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *intNodeDesc) lessthan(value int) bool { @@ -2579,10 +2689,10 @@ func (s *IntSetDesc) findNodeRemove(value int, preds *[maxLevel]*intNodeDesc, su // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2600,10 +2710,10 @@ func (s *IntSetDesc) findNodeRemove(value int, preds *[maxLevel]*intNodeDesc, su func (s *IntSetDesc) findNodeAdd(value int, preds *[maxLevel]*intNodeDesc, succs *[maxLevel]*intNodeDesc) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2665,7 +2775,7 @@ func (s *IntSetDesc) Add(value int) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockIntDesc(preds, highestLocked) @@ -2674,8 +2784,8 @@ func (s *IntSetDesc) Add(value int) bool { nn := newIntNodeDesc(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockIntDesc(preds, highestLocked) @@ -2704,10 +2814,10 @@ func (s *IntSetDesc) randomlevel() int { func (s *IntSetDesc) Contains(value int) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -2729,7 +2839,7 @@ func (s *IntSetDesc) Remove(value int) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -2761,7 +2871,7 @@ func (s *IntSetDesc) Remove(value int) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockIntDesc(preds, highestLocked) @@ -2770,7 +2880,7 @@ func (s *IntSetDesc) Remove(value int) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockIntDesc(preds, highestLocked) @@ -2784,16 +2894,16 @@ func (s *IntSetDesc) Remove(value int) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *IntSetDesc) Range(f func(value int) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -2811,26 +2921,37 @@ type Uint64Set struct { type uint64Node struct { value uint64 - next []*uint64Node + next optionalArray // [level]*uint64Node mu sync.Mutex flags bitflag + level uint32 } func newUuint64Node(value uint64, level int) *uint64Node { - return &uint64Node{ + node := &uint64Node{ value: value, - next: make([]*uint64Node, level), + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *uint64Node) loadNext(i int) *uint64Node { - return (*uint64Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint64Node)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *uint64Node) storeNext(i int, node *uint64Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uint64Node) atomicLoadNext(i int) *uint64Node { + return (*uint64Node)(n.next.atomicLoad(i)) +} + +func (n *uint64Node) atomicStoreNext(i int, node *uint64Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint64Node) lessthan(value uint64) bool { @@ -2857,10 +2978,10 @@ func (s *Uint64Set) findNodeRemove(value uint64, preds *[maxLevel]*uint64Node, s // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2878,10 +2999,10 @@ func (s *Uint64Set) findNodeRemove(value uint64, preds *[maxLevel]*uint64Node, s func (s *Uint64Set) findNodeAdd(value uint64, preds *[maxLevel]*uint64Node, succs *[maxLevel]*uint64Node) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -2943,7 +3064,7 @@ func (s *Uint64Set) Add(value uint64) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint64(preds, highestLocked) @@ -2952,8 +3073,8 @@ func (s *Uint64Set) Add(value uint64) bool { nn := newUuint64Node(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint64(preds, highestLocked) @@ -2982,10 +3103,10 @@ func (s *Uint64Set) randomlevel() int { func (s *Uint64Set) Contains(value uint64) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -3007,7 +3128,7 @@ func (s *Uint64Set) Remove(value uint64) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -3039,7 +3160,7 @@ func (s *Uint64Set) Remove(value uint64) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint64(preds, highestLocked) @@ -3048,7 +3169,7 @@ func (s *Uint64Set) Remove(value uint64) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockUint64(preds, highestLocked) @@ -3062,16 +3183,16 @@ func (s *Uint64Set) Remove(value uint64) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Uint64Set) Range(f func(value uint64) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -3089,26 +3210,37 @@ type Uint64SetDesc struct { type uint64NodeDesc struct { value uint64 - next []*uint64NodeDesc + next optionalArray // [level]*uint64NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newUuint64NodeDescDesc(value uint64, level int) *uint64NodeDesc { - return &uint64NodeDesc{ + node := &uint64NodeDesc{ value: value, - next: make([]*uint64NodeDesc, level), + level: uint32(level), } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } -// loadNext return `n.next[i]`(atomic) func (n *uint64NodeDesc) loadNext(i int) *uint64NodeDesc { - return (*uint64NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint64NodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *uint64NodeDesc) storeNext(i int, node *uint64NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uint64NodeDesc) atomicLoadNext(i int) *uint64NodeDesc { + return (*uint64NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *uint64NodeDesc) atomicStoreNext(i int, node *uint64NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint64NodeDesc) lessthan(value uint64) bool { @@ -3135,10 +3267,10 @@ func (s *Uint64SetDesc) findNodeRemove(value uint64, preds *[maxLevel]*uint64Nod // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3156,10 +3288,10 @@ func (s *Uint64SetDesc) findNodeRemove(value uint64, preds *[maxLevel]*uint64Nod func (s *Uint64SetDesc) findNodeAdd(value uint64, preds *[maxLevel]*uint64NodeDesc, succs *[maxLevel]*uint64NodeDesc) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3221,7 +3353,7 @@ func (s *Uint64SetDesc) Add(value uint64) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint64Desc(preds, highestLocked) @@ -3230,8 +3362,8 @@ func (s *Uint64SetDesc) Add(value uint64) bool { nn := newUuint64NodeDescDesc(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint64Desc(preds, highestLocked) @@ -3260,10 +3392,10 @@ func (s *Uint64SetDesc) randomlevel() int { func (s *Uint64SetDesc) Contains(value uint64) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -3285,7 +3417,7 @@ func (s *Uint64SetDesc) Remove(value uint64) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -3317,7 +3449,7 @@ func (s *Uint64SetDesc) Remove(value uint64) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint64Desc(preds, highestLocked) @@ -3326,7 +3458,7 @@ func (s *Uint64SetDesc) Remove(value uint64) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockUint64Desc(preds, highestLocked) @@ -3340,16 +3472,16 @@ func (s *Uint64SetDesc) Remove(value uint64) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Uint64SetDesc) Range(f func(value uint64) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -3367,26 +3499,37 @@ type Uint32Set struct { type uint32Node struct { value uint32 - next []*uint32Node + next optionalArray // [level]*uint32Node mu sync.Mutex flags bitflag + level uint32 } func newUint32Node(value uint32, level int) *uint32Node { - return &uint32Node{ + node := &uint32Node{ value: value, - next: make([]*uint32Node, level), + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *uint32Node) loadNext(i int) *uint32Node { - return (*uint32Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint32Node)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *uint32Node) storeNext(i int, node *uint32Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uint32Node) atomicLoadNext(i int) *uint32Node { + return (*uint32Node)(n.next.atomicLoad(i)) +} + +func (n *uint32Node) atomicStoreNext(i int, node *uint32Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint32Node) lessthan(value uint32) bool { @@ -3413,10 +3556,10 @@ func (s *Uint32Set) findNodeRemove(value uint32, preds *[maxLevel]*uint32Node, s // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3434,10 +3577,10 @@ func (s *Uint32Set) findNodeRemove(value uint32, preds *[maxLevel]*uint32Node, s func (s *Uint32Set) findNodeAdd(value uint32, preds *[maxLevel]*uint32Node, succs *[maxLevel]*uint32Node) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3499,7 +3642,7 @@ func (s *Uint32Set) Add(value uint32) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint32(preds, highestLocked) @@ -3508,8 +3651,8 @@ func (s *Uint32Set) Add(value uint32) bool { nn := newUint32Node(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint32(preds, highestLocked) @@ -3538,10 +3681,10 @@ func (s *Uint32Set) randomlevel() int { func (s *Uint32Set) Contains(value uint32) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -3563,7 +3706,7 @@ func (s *Uint32Set) Remove(value uint32) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -3595,7 +3738,7 @@ func (s *Uint32Set) Remove(value uint32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint32(preds, highestLocked) @@ -3604,7 +3747,7 @@ func (s *Uint32Set) Remove(value uint32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockUint32(preds, highestLocked) @@ -3618,16 +3761,16 @@ func (s *Uint32Set) Remove(value uint32) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Uint32Set) Range(f func(value uint32) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -3645,26 +3788,37 @@ type Uint32SetDesc struct { type uint32NodeDesc struct { value uint32 - next []*uint32NodeDesc + next optionalArray // [level]*uint32NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newUint32NodeDesc(value uint32, level int) *uint32NodeDesc { - return &uint32NodeDesc{ + node := &uint32NodeDesc{ value: value, - next: make([]*uint32NodeDesc, level), + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *uint32NodeDesc) loadNext(i int) *uint32NodeDesc { - return (*uint32NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint32NodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *uint32NodeDesc) storeNext(i int, node *uint32NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uint32NodeDesc) atomicLoadNext(i int) *uint32NodeDesc { + return (*uint32NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *uint32NodeDesc) atomicStoreNext(i int, node *uint32NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint32NodeDesc) lessthan(value uint32) bool { @@ -3691,10 +3845,10 @@ func (s *Uint32SetDesc) findNodeRemove(value uint32, preds *[maxLevel]*uint32Nod // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3712,10 +3866,10 @@ func (s *Uint32SetDesc) findNodeRemove(value uint32, preds *[maxLevel]*uint32Nod func (s *Uint32SetDesc) findNodeAdd(value uint32, preds *[maxLevel]*uint32NodeDesc, succs *[maxLevel]*uint32NodeDesc) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3777,7 +3931,7 @@ func (s *Uint32SetDesc) Add(value uint32) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint32Desc(preds, highestLocked) @@ -3786,8 +3940,8 @@ func (s *Uint32SetDesc) Add(value uint32) bool { nn := newUint32NodeDesc(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint32Desc(preds, highestLocked) @@ -3816,10 +3970,10 @@ func (s *Uint32SetDesc) randomlevel() int { func (s *Uint32SetDesc) Contains(value uint32) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -3841,7 +3995,7 @@ func (s *Uint32SetDesc) Remove(value uint32) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -3873,7 +4027,7 @@ func (s *Uint32SetDesc) Remove(value uint32) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint32Desc(preds, highestLocked) @@ -3882,7 +4036,7 @@ func (s *Uint32SetDesc) Remove(value uint32) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockUint32Desc(preds, highestLocked) @@ -3896,16 +4050,16 @@ func (s *Uint32SetDesc) Remove(value uint32) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Uint32SetDesc) Range(f func(value uint32) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -3923,26 +4077,37 @@ type Uint16Set struct { type uint16Node struct { value uint16 - next []*uint16Node + next optionalArray // [level]*uint16Node mu sync.Mutex flags bitflag + level uint32 } func newUint16Node(value uint16, level int) *uint16Node { - return &uint16Node{ + node := &uint16Node{ value: value, - next: make([]*uint16Node, level), + level: uint32(level), } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } -// loadNext return `n.next[i]`(atomic) func (n *uint16Node) loadNext(i int) *uint16Node { - return (*uint16Node)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint16Node)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *uint16Node) storeNext(i int, node *uint16Node) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uint16Node) atomicLoadNext(i int) *uint16Node { + return (*uint16Node)(n.next.atomicLoad(i)) +} + +func (n *uint16Node) atomicStoreNext(i int, node *uint16Node) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint16Node) lessthan(value uint16) bool { @@ -3969,10 +4134,10 @@ func (s *Uint16Set) findNodeRemove(value uint16, preds *[maxLevel]*uint16Node, s // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -3990,10 +4155,10 @@ func (s *Uint16Set) findNodeRemove(value uint16, preds *[maxLevel]*uint16Node, s func (s *Uint16Set) findNodeAdd(value uint16, preds *[maxLevel]*uint16Node, succs *[maxLevel]*uint16Node) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -4055,7 +4220,7 @@ func (s *Uint16Set) Add(value uint16) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint16(preds, highestLocked) @@ -4064,8 +4229,8 @@ func (s *Uint16Set) Add(value uint16) bool { nn := newUint16Node(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint16(preds, highestLocked) @@ -4094,10 +4259,10 @@ func (s *Uint16Set) randomlevel() int { func (s *Uint16Set) Contains(value uint16) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -4119,7 +4284,7 @@ func (s *Uint16Set) Remove(value uint16) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -4151,7 +4316,7 @@ func (s *Uint16Set) Remove(value uint16) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint16(preds, highestLocked) @@ -4160,7 +4325,7 @@ func (s *Uint16Set) Remove(value uint16) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockUint16(preds, highestLocked) @@ -4174,16 +4339,16 @@ func (s *Uint16Set) Remove(value uint16) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Uint16Set) Range(f func(value uint16) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -4201,26 +4366,37 @@ type Uint16SetDesc struct { type uint16NodeDesc struct { value uint16 - next []*uint16NodeDesc + next optionalArray // [level]*uint16NodeDesc mu sync.Mutex flags bitflag + level uint32 } func newUint16NodeDesc(value uint16, level int) *uint16NodeDesc { - return &uint16NodeDesc{ + node := &uint16NodeDesc{ value: value, - next: make([]*uint16NodeDesc, level), + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *uint16NodeDesc) loadNext(i int) *uint16NodeDesc { - return (*uint16NodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uint16NodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *uint16NodeDesc) storeNext(i int, node *uint16NodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uint16NodeDesc) atomicLoadNext(i int) *uint16NodeDesc { + return (*uint16NodeDesc)(n.next.atomicLoad(i)) +} + +func (n *uint16NodeDesc) atomicStoreNext(i int, node *uint16NodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uint16NodeDesc) lessthan(value uint16) bool { @@ -4247,10 +4423,10 @@ func (s *Uint16SetDesc) findNodeRemove(value uint16, preds *[maxLevel]*uint16Nod // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -4268,10 +4444,10 @@ func (s *Uint16SetDesc) findNodeRemove(value uint16, preds *[maxLevel]*uint16Nod func (s *Uint16SetDesc) findNodeAdd(value uint16, preds *[maxLevel]*uint16NodeDesc, succs *[maxLevel]*uint16NodeDesc) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -4333,7 +4509,7 @@ func (s *Uint16SetDesc) Add(value uint16) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint16Desc(preds, highestLocked) @@ -4342,8 +4518,8 @@ func (s *Uint16SetDesc) Add(value uint16) bool { nn := newUint16NodeDesc(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint16Desc(preds, highestLocked) @@ -4372,10 +4548,10 @@ func (s *Uint16SetDesc) randomlevel() int { func (s *Uint16SetDesc) Contains(value uint16) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -4397,7 +4573,7 @@ func (s *Uint16SetDesc) Remove(value uint16) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -4429,7 +4605,7 @@ func (s *Uint16SetDesc) Remove(value uint16) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint16Desc(preds, highestLocked) @@ -4438,7 +4614,7 @@ func (s *Uint16SetDesc) Remove(value uint16) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockUint16Desc(preds, highestLocked) @@ -4452,16 +4628,16 @@ func (s *Uint16SetDesc) Remove(value uint16) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *Uint16SetDesc) Range(f func(value uint16) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -4479,26 +4655,37 @@ type UintSet struct { type uintNode struct { value uint - next []*uintNode + next optionalArray // [level]*uintNode mu sync.Mutex flags bitflag + level uint32 } func newUintNode(value uint, level int) *uintNode { - return &uintNode{ + node := &uintNode{ value: value, - next: make([]*uintNode, level), + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *uintNode) loadNext(i int) *uintNode { - return (*uintNode)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uintNode)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *uintNode) storeNext(i int, node *uintNode) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uintNode) atomicLoadNext(i int) *uintNode { + return (*uintNode)(n.next.atomicLoad(i)) +} + +func (n *uintNode) atomicStoreNext(i int, node *uintNode) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uintNode) lessthan(value uint) bool { @@ -4525,10 +4712,10 @@ func (s *UintSet) findNodeRemove(value uint, preds *[maxLevel]*uintNode, succs * // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -4546,10 +4733,10 @@ func (s *UintSet) findNodeRemove(value uint, preds *[maxLevel]*uintNode, succs * func (s *UintSet) findNodeAdd(value uint, preds *[maxLevel]*uintNode, succs *[maxLevel]*uintNode) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -4611,7 +4798,7 @@ func (s *UintSet) Add(value uint) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUint(preds, highestLocked) @@ -4620,8 +4807,8 @@ func (s *UintSet) Add(value uint) bool { nn := newUintNode(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUint(preds, highestLocked) @@ -4650,10 +4837,10 @@ func (s *UintSet) randomlevel() int { func (s *UintSet) Contains(value uint) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -4675,7 +4862,7 @@ func (s *UintSet) Remove(value uint) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -4707,7 +4894,7 @@ func (s *UintSet) Remove(value uint) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUint(preds, highestLocked) @@ -4716,7 +4903,7 @@ func (s *UintSet) Remove(value uint) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockUint(preds, highestLocked) @@ -4730,16 +4917,16 @@ func (s *UintSet) Remove(value uint) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *UintSet) Range(f func(value uint) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -4757,26 +4944,37 @@ type UintSetDesc struct { type uintNodeDesc struct { value uint - next []*uintNodeDesc + next optionalArray // [level]*uintNodeDesc mu sync.Mutex flags bitflag + level uint32 } func newUintNodeDesc(value uint, level int) *uintNodeDesc { - return &uintNodeDesc{ + node := &uintNodeDesc{ value: value, - next: make([]*uintNodeDesc, level), + level: uint32(level), } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) + } + return node } -// loadNext return `n.next[i]`(atomic) func (n *uintNodeDesc) loadNext(i int) *uintNodeDesc { - return (*uintNodeDesc)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*uintNodeDesc)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *uintNodeDesc) storeNext(i int, node *uintNodeDesc) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *uintNodeDesc) atomicLoadNext(i int) *uintNodeDesc { + return (*uintNodeDesc)(n.next.atomicLoad(i)) +} + +func (n *uintNodeDesc) atomicStoreNext(i int, node *uintNodeDesc) { + n.next.atomicStore(i, unsafe.Pointer(node)) } func (n *uintNodeDesc) lessthan(value uint) bool { @@ -4803,10 +5001,10 @@ func (s *UintSetDesc) findNodeRemove(value uint, preds *[maxLevel]*uintNodeDesc, // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -4824,10 +5022,10 @@ func (s *UintSetDesc) findNodeRemove(value uint, preds *[maxLevel]*uintNodeDesc, func (s *UintSetDesc) findNodeAdd(value uint, preds *[maxLevel]*uintNodeDesc, succs *[maxLevel]*uintNodeDesc) int { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.lessthan(value) { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -4889,7 +5087,7 @@ func (s *UintSetDesc) Add(value uint) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockUintDesc(preds, highestLocked) @@ -4898,8 +5096,8 @@ func (s *UintSetDesc) Add(value uint) bool { nn := newUintNodeDesc(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockUintDesc(preds, highestLocked) @@ -4928,10 +5126,10 @@ func (s *UintSetDesc) randomlevel() int { func (s *UintSetDesc) Contains(value uint) bool { x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.lessthan(value) { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -4953,7 +5151,7 @@ func (s *UintSetDesc) Remove(value uint) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -4985,7 +5183,7 @@ func (s *UintSetDesc) Remove(value uint) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockUintDesc(preds, highestLocked) @@ -4994,7 +5192,7 @@ func (s *UintSetDesc) Remove(value uint) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockUintDesc(preds, highestLocked) @@ -5008,16 +5206,16 @@ func (s *UintSetDesc) Remove(value uint) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *UintSetDesc) Range(f func(value uint) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } @@ -5036,27 +5234,38 @@ type StringSet struct { type stringNode struct { value string score uint64 - next []*stringNode + next optionalArray // [level]*stringNode mu sync.Mutex flags bitflag + level uint32 } func newStringNode(value string, level int) *stringNode { - return &stringNode{ - value: value, + node := &stringNode{ score: hash(value), - next: make([]*stringNode, level), + value: value, + level: uint32(level), + } + if level > op1 { + node.next.extra = new([op2]unsafe.Pointer) } + return node } -// loadNext return `n.next[i]`(atomic) func (n *stringNode) loadNext(i int) *stringNode { - return (*stringNode)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])))) + return (*stringNode)(n.next.load(i)) } -// storeNext same with `n.next[i] = node`(atomic) func (n *stringNode) storeNext(i int, node *stringNode) { - atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&n.next[i])), unsafe.Pointer(node)) + n.next.store(i, unsafe.Pointer(node)) +} + +func (n *stringNode) atomicLoadNext(i int) *stringNode { + return (*stringNode)(n.next.atomicLoad(i)) +} + +func (n *stringNode) atomicStoreNext(i int, node *stringNode) { + n.next.atomicStore(i, unsafe.Pointer(node)) } // NewString return an empty string skip set. @@ -5076,10 +5285,10 @@ func (s *StringSet) findNodeRemove(value string, preds *[maxLevel]*stringNode, s // lFound represents the index of the first layer at which it found a node. lFound, x := -1, s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.cmp(score, value) < 0 { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -5098,10 +5307,10 @@ func (s *StringSet) findNodeAdd(value string, preds *[maxLevel]*stringNode, succ score := hash(value) x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - succ := x.loadNext(i) + succ := x.atomicLoadNext(i) for succ != nil && succ.cmp(score, value) < 0 { x = succ - succ = x.loadNext(i) + succ = x.atomicLoadNext(i) } preds[i] = x succs[i] = succ @@ -5163,7 +5372,7 @@ func (s *StringSet) Add(value string) bool { // It is valid if: // 1. The previous node and next node both are not marked. // 2. The previous node's next node is succ in this layer. - valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && (succ == nil || !succ.flags.Get(marked)) && pred.loadNext(layer) == succ } if !valid { unlockString(preds, highestLocked) @@ -5172,8 +5381,8 @@ func (s *StringSet) Add(value string) bool { nn := newStringNode(value, level) for layer := 0; layer < level; layer++ { - nn.next[layer] = succs[layer] - preds[layer].storeNext(layer, nn) + nn.storeNext(layer, succs[layer]) + preds[layer].atomicStoreNext(layer, nn) } nn.flags.SetTrue(fullyLinked) unlockString(preds, highestLocked) @@ -5203,10 +5412,10 @@ func (s *StringSet) Contains(value string) bool { score := hash(value) x := s.header for i := int(atomic.LoadInt64(&s.highestLevel)) - 1; i >= 0; i-- { - nex := x.loadNext(i) + nex := x.atomicLoadNext(i) for nex != nil && nex.cmp(score, value) < 0 { x = nex - nex = x.loadNext(i) + nex = x.atomicLoadNext(i) } // Check if the value already in the skip list. @@ -5228,7 +5437,7 @@ func (s *StringSet) Remove(value string) bool { for { lFound := s.findNodeRemove(value, &preds, &succs) if isMarked || // this process mark this node or we can find this node in the skip list - lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (len(succs[lFound].next)-1) == lFound { + lFound != -1 && succs[lFound].flags.MGet(fullyLinked|marked, fullyLinked) && (int(succs[lFound].level)-1) == lFound { if !isMarked { // we don't mark this node for now nodeToRemove = succs[lFound] topLayer = lFound @@ -5260,7 +5469,7 @@ func (s *StringSet) Remove(value string) bool { // It is valid if: // 1. the previous node exists. // 2. no another node has inserted into the skip list in this layer. - valid = !pred.flags.Get(marked) && pred.next[layer] == succ + valid = !pred.flags.Get(marked) && pred.loadNext(layer) == succ } if !valid { unlockString(preds, highestLocked) @@ -5269,7 +5478,7 @@ func (s *StringSet) Remove(value string) bool { for i := topLayer; i >= 0; i-- { // Now we own the `nodeToRemove`, no other goroutine will modify it. // So we don't need `nodeToRemove.loadNext` - preds[i].storeNext(i, nodeToRemove.next[i]) + preds[i].atomicStoreNext(i, nodeToRemove.loadNext(i)) } nodeToRemove.mu.Unlock() unlockString(preds, highestLocked) @@ -5283,16 +5492,16 @@ func (s *StringSet) Remove(value string) bool { // Range calls f sequentially for each value present in the skip set. // If f returns false, range stops the iteration. func (s *StringSet) Range(f func(value string) bool) { - x := s.header.loadNext(0) + x := s.header.atomicLoadNext(0) for x != nil { if !x.flags.MGet(fullyLinked|marked, fullyLinked) { - x = x.loadNext(0) + x = x.atomicLoadNext(0) continue } if !f(x.value) { break } - x = x.loadNext(0) + x = x.atomicLoadNext(0) } } diff --git a/collection/skipset/types_gen.go b/collection/skipset/types_gen.go index 087bd721..18537fd4 100644 --- a/collection/skipset/types_gen.go +++ b/collection/skipset/types_gen.go @@ -123,10 +123,8 @@ func replaceString(data string) string { score uint64`, -1) data = strings.Replace(data, - `return &int64Node{ - value: value,`, - `return &int64Node{ - value: value, + `&int64Node{`, + `&int64Node{ score: hash(value),`, -1) // Refactor comparsion.