From 6efbd096e3e928b63ef110ce387f4ee775eeb479 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 26 Feb 2022 13:10:58 +0800 Subject: [PATCH 1/2] improve comment --- .idea/.gitignore | 8 + .idea/dictionaries | 6 + .idea/encodings.xml | 4 + .idea/markdown.xml | 9 + .idea/misc.xml | 14 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .idea/watcherTasks.xml | 10 + .idea/xmm.iml | 9 + README.md | 61 +- bit_map.go | 8 +- bit_map_test.go | 6 +- chunk.go | 2 +- class_span.go | 24 +- common.go | 13 +- common_test.go | 2 +- docs/README.md | 2 +- docs/XMM-DesignImplementation.md | 67 +- docs/XMM-INTRO-EN.md | 117 ++-- docs/XMM-InvestigateResearch.md | 26 +- docs/XMM-Usage-EN.md | 30 +- docs/XMM-Usage.md | 1036 +++++++++++++++--------------- entry.go | 151 ++--- example/xmm-test00.go | 45 +- example/xmm-test01.go | 63 +- example/xmm-test02.go | 81 ++- example/xmm-test03.go | 233 ++++--- go.mod | 6 +- go.sum | 6 + heap.go | 46 +- linear_alloc_test.go | 7 +- metadata.go | 12 +- metadata_alloc.go | 14 +- raw_linear_memory.go | 2 +- raw_memory.go | 12 +- raw_memory_test.go | 34 +- span.go | 24 +- span_pool.go | 18 +- span_pool_test.go | 2 +- string_alloc.go | 12 +- trees.go | 2 +- xmm.go | 20 +- xmm_test.go | 27 +- 43 files changed, 1186 insertions(+), 1099 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/dictionaries create mode 100644 .idea/encodings.xml create mode 100644 .idea/markdown.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/watcherTasks.xml create mode 100644 .idea/xmm.iml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/dictionaries b/.idea/dictionaries new file mode 100644 index 0000000..a88a91f --- /dev/null +++ b/.idea/dictionaries @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..72dd4ff --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/markdown.xml b/.idea/markdown.xml new file mode 100644 index 0000000..1e34094 --- /dev/null +++ b/.idea/markdown.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..c49ff23 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..50a1fe7 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/watcherTasks.xml b/.idea/watcherTasks.xml new file mode 100644 index 0000000..bd32330 --- /dev/null +++ b/.idea/watcherTasks.xml @@ -0,0 +1,10 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/xmm.iml b/.idea/xmm.iml new file mode 100644 index 0000000..5e764c4 --- /dev/null +++ b/.idea/xmm.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index f59e9a7..94cf8a6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -

GoDoc @@ -6,33 +5,28 @@ Go Report Card

- ## ➤ [XMM English Introduction Document](https://github.com/heiyeluren/XMM/blob/main/docs/XMM-INTRO-EN.md) -

- ## ➤ XMM中文文档 + ## XMM (eXtensible) Memory Manager - 完全自主第三方 Go 内存分配管理器 - [XMM (eXtensible) Memory Manager - 完全自主第三方 Go 内存分配管理器](#xmm-extensible-memory-manager---完全自主第三方-go-内存分配管理器) - - [XMM 是什么？](#xmm-是什么) - - [XMM 主要具备以下特点](#xmm-主要具备以下特点) - - [为什么要设计 XMM？](#为什么要设计-xmm) - - [为什么要设计自主的内存管理器？](#为什么要设计自主的内存管理器) - - [为什么不使用内置的 map/slice 等数据结构？](#为什么不使用内置的-mapslice-等数据结构) - - [为什么不使用其他开源的内存池？](#为什么不使用其他开源的内存池) - - [XMM 的最终设计结论是什么？](#xmm-的最终设计结论是什么) - - [XMM 设计的目标是什么？](#xmm-设计的目标是什么) - - [XMM 快速使用入门](#xmm-快速使用入门) - - [☆ XMM 使用案例 ☆](#-xmm-使用案例-) - - [XMM 实现原理介绍](#xmm-实现原理介绍) - - [XMM 技术交流](#xmm-技术交流) - - - + - [XMM 是什么？](#xmm-是什么) + - [XMM 主要具备以下特点](#xmm-主要具备以下特点) + - [为什么要设计 XMM？](#为什么要设计-xmm) + - [为什么要设计自主的内存管理器？](#为什么要设计自主的内存管理器) + - [为什么不使用内置的 map/slice 等数据结构？](#为什么不使用内置的-mapslice-等数据结构) + - [为什么不使用其他开源的内存池？](#为什么不使用其他开源的内存池) + - [XMM 的最终设计结论是什么？](#xmm-的最终设计结论是什么) + - [XMM 设计的目标是什么？](#xmm-设计的目标是什么) + - [XMM 快速使用入门](#xmm-快速使用入门) + - [☆ XMM 使用案例 ☆](#-xmm-使用案例-) + - [XMM 实现原理介绍](#xmm-实现原理介绍) + - [XMM 技术交流](#xmm-技术交流)
@@ -40,7 +34,8 @@ XMM - X(eXtensible) Memory Manager（完全自主研发的第三方 Go 内存分配管理器） -XMM 是一个在 Go 语言环境中完全自主实现的第三方内存管理库，不依赖于 Go 本身的任何内存管理能力，纯自主实现的 Go 内存管理库；能够应对各种场景下大小内存的 分配/释放/管理 等工作，能够帮助适用于任何复杂数据结构的构建（链表/数组/树/hash 等场景），能够良好完美的逃逸掉 Go 内置的 GC 机制，保证程序的超高性能，是构建高性能程序基础设施。 +XMM 是一个在 Go 语言环境中完全自主实现的第三方内存管理库，不依赖于 Go 本身的任何内存管理能力，纯自主实现的 Go 内存管理库；能够应对各种场景下大小内存的 分配/释放/管理 +等工作，能够帮助适用于任何复杂数据结构的构建（链表/数组/树/hash 等场景），能够良好完美的逃逸掉 Go 内置的 GC 机制，保证程序的超高性能，是构建高性能程序基础设施。
@@ -50,11 +45,13 @@ XMM 是一个在 Go 语言环境中完全自主实现的第三方内存管理库 2. XMM 能够应对各种场景下大小内存的 分配/释放/管理 等工作，能够帮助适用于任何复杂数据结构的构建，比如链表/数组/树/哈希表等等场景；XMM 可以让你像 C/C++ 一样方便便捷使用系统内存，并且不用担心性能问题。 -3. XMM 能够良好完美的逃逸掉 Go 内置的 GC 机制，保证程序的超高性能，是构建高性能程序基础设施；但与 sync.Pool 等实现机制完全不同，sync.Pool 等使用字节流实现来逃逸 GC，XMM 是纯使用 Linux 系统的 mmap 作为底层内存存储，XMM 更像 TcMalloc 等内存分配器。 +3. XMM 能够良好完美的逃逸掉 Go 内置的 GC 机制，保证程序的超高性能，是构建高性能程序基础设施；但与 sync.Pool 等实现机制完全不同，sync.Pool 等使用字节流实现来逃逸 GC，XMM 是纯使用 Linux 系统的 + mmap 作为底层内存存储，XMM 更像 TcMalloc 等内存分配器。 4. XMM 协程安全，且分配性能超高，目前在普通 Linux 服务器上面可以达到 350w alloc/s，就是每秒可以进行 350 万次的内存分配操作不卡顿，非常适合想要自主管理内存且超高性能场景。 -5. XMM 内存库使用接口简单，兼容性强，能够兼容 Go 1.8 以上版本，容易上手（推荐 go 1.12+ 版本更好），可以在 XMM 之上重构你所有想要的高性能数据结构，比如 map/slice 等等。（案例部分可以做一些数据结构实现的参考） +5. XMM 内存库使用接口简单，兼容性强，能够兼容 Go 1.8 以上版本，容易上手（推荐 go 1.12+ 版本更好），可以在 XMM 之上重构你所有想要的高性能数据结构，比如 map/slice + 等等。（案例部分可以做一些数据结构实现的参考）

@@ -65,23 +62,27 @@ XMM 是一个在 Go 语言环境中完全自主实现的第三方内存管理库 ### 为什么要设计自主的内存管理器？ -为了应对在多种内存管理的场景下的使用，可能需要有一些除了内置数据结构外的一些内存自主调度使用的场景，比如构建复杂的高性能的数据结构，在大规模内存占用，或者是非常多的小内存块占用场景下，能够尽量减少 Go 的 GC 机制，保障服务性能稳定不会因为 GC 而产生抖动。 +为了应对在多种内存管理的场景下的使用，可能需要有一些除了内置数据结构外的一些内存自主调度使用的场景，比如构建复杂的高性能的数据结构，在大规模内存占用，或者是非常多的小内存块占用场景下，能够尽量减少 Go 的 GC +机制，保障服务性能稳定不会因为 GC 而产生抖动。
- ### 为什么不使用内置的 map/slice 等数据结构？ -Golang 本身为了性能和内存可控，整个内存管理是完全封闭不对外的，并且有自主的 GC 机制，需要自主内存管理比较麻烦；Go 中自带的 GC 机制经过很多个版本的迭代，到目前性能已经很不错，但是在大规模的碎片化内存块下面，GC 还是会有一定损耗，在极端高性能场景下，GC 会让整个后台应用服务性能上不去（或偶尔卡顿）。所以一句话，Go 本身指针等还有性能会受到 GC 的影响，导致服务性能总是上不去。 +Golang 本身为了性能和内存可控，整个内存管理是完全封闭不对外的，并且有自主的 GC 机制，需要自主内存管理比较麻烦；Go 中自带的 GC 机制经过很多个版本的迭代，到目前性能已经很不错，但是在大规模的碎片化内存块下面，GC +还是会有一定损耗，在极端高性能场景下，GC 会让整个后台应用服务性能上不去（或偶尔卡顿）。所以一句话，Go 本身指针等还有性能会受到 GC 的影响，导致服务性能总是上不去。
### 为什么不使用其他开源的内存池？ 1. 除 Go 本身的内存模块，调研了解现有大部分的第三方 对象池/内存池/字节池 等需要某块自主内存操作的场景中基本是 Map/sync.Pool/Bytes[] 等方式。 -2. Map 数据结构适合保存各类型数据，但 GC 概率大； sync.Pool 这类保存复用临时对象，也可以各种数据机构，可适当减少 GC（无法避免 GC）； Bytes[] 方式来保存字节数据，并且只能保存字节数据，通过某些处理，尽量逃避 GC 扫描；（对比参考 [Go 语言基于 channel 实现的并发安全的字节池](https://zhuanlan.zhihu.com/p/265790840) ） +2. Map 数据结构适合保存各类型数据，但 GC 概率大； sync.Pool 这类保存复用临时对象，也可以各种数据机构，可适当减少 GC（无法避免 GC）； Bytes[] + 方式来保存字节数据，并且只能保存字节数据，通过某些处理，尽量逃避 GC 扫描；（对比参考 [Go 语言基于 channel 实现的并发安全的字节池](https://zhuanlan.zhihu.com/p/265790840) ） -3. 现有开源库包括：依赖于 sync.Pool 的比如字节的 mcache [gopkg/mcache.go](https://github.com/bytedance/gopkg/blob/main/lang/mcache/mcache.go)；采用 Bytes[] 方式的比如 MinIO 的的 [bpool minio/bpool.go](https://github.com/minio/minio/blob/master/internal/bpool/bpool.go) ，都可以学习参考。 +3. 现有开源库包括：依赖于 sync.Pool 的比如字节的 + mcache [gopkg/mcache.go](https://github.com/bytedance/gopkg/blob/main/lang/mcache/mcache.go)；采用 Bytes[] 方式的比如 MinIO + 的的 [bpool minio/bpool.go](https://github.com/minio/minio/blob/master/internal/bpool/bpool.go) ，都可以学习参考。 4. 结论：XMM 与他们实现机制完全不同，XMM 更靠近 Go 内置内存分配机制原理 @@ -89,11 +90,13 @@ Golang 本身为了性能和内存可控，整个内存管理是完全封闭不 ### XMM 的最终设计结论是什么？ -为了完全实现最终为了逃逸掉 Golang 的 GC 机制，以及拥有完全自主可控的内存管理分配操作，在面对成千上万的小对象场景中，不会因为 Go 本身 GC 机制带来任何的抖动，所以自主从零开始实现了 XMM 模块，达到在 Go 程序中调用 XMM 模块可以达到完美的自主内存 申请/释放/管理 的功能，并可以完美逃逸掉 Go 的 GC 机制。 +为了完全实现最终为了逃逸掉 Golang 的 GC 机制，以及拥有完全自主可控的内存管理分配操作，在面对成千上万的小对象场景中，不会因为 Go 本身 GC 机制带来任何的抖动，所以自主从零开始实现了 XMM 模块，达到在 Go 程序中调用 +XMM 模块可以达到完美的自主内存 申请/释放/管理 的功能，并可以完美逃逸掉 Go 的 GC 机制。

### XMM 设计的目标是什么？ + 为了保证高性能，XMM 设计之初，就定下了三个核心目标： 1. 单机（6 核心 KVM 或物理机）内存分配性能达到 350w+ alloc/s；（每秒内存分配速度）； @@ -102,7 +105,6 @@ Golang 本身为了性能和内存可控，整个内存管理是完全封闭不 3. 不会内存泄露，并且内存管理不是粗糙的，而颗粒度细致的，完全尽量可媲美行业主流的内存管理分配器。 -

@@ -121,7 +123,6 @@ Golang 本身为了性能和内存可控，整个内存管理是完全封闭不

- ## XMM 实现原理介绍 1. [XMM 的核心设计与实现流程](https://github.com/heiyeluren/XMM/blob/main/docs/XMM-DesignImplementation.md) diff --git a/bit_map.go b/bit_map.go index 6d5c918..4a843b6 100644 --- a/bit_map.go +++ b/bit_map.go @@ -152,14 +152,14 @@ func (m markBits) setMarked() { // Might be racing with other updates, so use atomic update always. // We used to be clever here and use a non-atomic update in certain // cases, but it's not worth the risk. - //atomic.Or32(m.uint32p, m.mask) + // atomic.Or32(m.uint32p, m.mask) for { val := atomic.LoadUint32(m.uint32p) if atomic.CompareAndSwapUint32(m.uint32p, val, val^m.mask) { return } } - //atomic.StoreUint32(m.uint32p, *m.uint32p^m.mask) + // atomic.StoreUint32(m.uint32p, *m.uint32p^m.mask) } // setMarkedNonAtomic sets the marked bit in the markbits, non-atomically. @@ -172,14 +172,14 @@ func (m markBits) clearMarked() { // Might be racing with other updates, so use atomic update always. // We used to be clever here and use a non-atomic update in certain // cases, but it's not worth the risk. - //atomic.And32(m.uint32p, ^m.mask) + // atomic.And32(m.uint32p, ^m.mask) for { val := atomic.LoadUint32(m.uint32p) if atomic.CompareAndSwapUint32(m.uint32p, val, val&(^m.mask)) { return } } - //atomic.StoreUint32(m.uint32p, *m.uint32p&(^m.mask)) + // atomic.StoreUint32(m.uint32p, *m.uint32p&(^m.mask)) } // markBitsForSpan returns the markBits for the span base address base. diff --git a/bit_map_test.go b/bit_map_test.go index 6e14b53..708d42a 100644 --- a/bit_map_test.go +++ b/bit_map_test.go @@ -30,19 +30,19 @@ func TestMarkBits(t *testing.T) { if err != nil { t.Fatal(err) } - //fmt.Println(String(bit, 4), len(String(bit, 4))) + // fmt.Println(String(bit, 4), len(String(bit, 4))) bytep, mask := bit.bitp(objIndex) mb := markBits{bytep, mask, objIndex} fmt.Printf("bytep:%.32b , mask:%.32b isMarked:%t \n", *bytep, mask, mb.isMarked()) mb.setMarked() - //fmt.Println(String(bit, 4)) + // fmt.Println(String(bit, 4)) bytep, mask = bit.bitp(objIndex) mb = markBits{bytep, mask, objIndex} fmt.Printf("bytep:%.32b , mask:%.32b isMarked:%t \n", *bytep, mask, mb.isMarked()) - //fmt.Println(String(bit, 4)) + // fmt.Println(String(bit, 4)) bytep, mask = bit.bitp(uintptr(5000000)) mb = markBits{bytep, mask, objIndex} diff --git a/chunk.go b/chunk.go index ec8d3bd..2736537 100644 --- a/chunk.go +++ b/chunk.go @@ -18,7 +18,7 @@ package xmm -//连续page的管理 +// 连续page的管理 type xChunk struct { startAddr uintptr npages uintptr diff --git a/class_span.go b/class_span.go index 5364cba..1ef848b 100644 --- a/class_span.go +++ b/class_span.go @@ -24,14 +24,13 @@ import ( type xClassSpan struct { - //lock sync.Mutex - + // lock sync.Mutex classIndex uint // class的索引 - //空的， + // 空的， free *mSpanList - //满的span。 + // 满的span。 full *mSpanList // nmalloc is the cumulative count of objects allocated from @@ -42,6 +41,7 @@ type xClassSpan struct { heap *xHeap } +// Init initializes func (x *xClassSpan) Init(classIndex uint, heap *xHeap) error { if heap == nil { return errors.New("heap is nil") @@ -59,7 +59,7 @@ func (x *xClassSpan) allocSpan(index int, f float32) (*xSpan, error) { return x.free.moveHead(), nil }() if span != nil && err == nil { - //log.Printf("xClassSpan class:%d free申请 span:%d\n", x.classIndex, unsafe.Pointer(span)) + // log.Printf("xClassSpan class:%d free申请 span:%d\n", x.classIndex, unsafe.Pointer(span)) return span, nil } } @@ -68,7 +68,7 @@ func (x *xClassSpan) allocSpan(index int, f float32) (*xSpan, error) { size := class_to_size[index] pageNum = uint8(Align(Align(uintptr(size), _PageSize)/uintptr(_PageSize), uintptr(pageNum))) span, err := heap.allocSpan(uintptr(pageNum), uint(index), uintptr(size), f) - //log.Printf("xClassSpan heap.allocSpan class:%d free申请 span:%d\n", x.classIndex, unsafe.Pointer(span)) + // log.Printf("xClassSpan heap.allocSpan class:%d free申请 span:%d\n", x.classIndex, unsafe.Pointer(span)) if err != nil { return nil, err } @@ -92,7 +92,7 @@ func (x *xClassSpan) freeSpan(span *xSpan) (swap bool, size uint, err error) { needFree := false err = func() error { gcCount := span.countGcMarkBits() - //没有达到gc阈值或者当前span正在被分配不做GC（异步gc该类span） + // 没有达到gc阈值或者当前span正在被分配不做GC（异步gc该类span） if gcCount <= uintptr(float64(span.nelems)*SpanGCFactor) { return nil } @@ -104,7 +104,7 @@ func (x *xClassSpan) freeSpan(span *xSpan) (swap bool, size uint, err error) { allocCount := span.allocCount gcCount = span.countGcMarkBits() x.heap.addFreeCapacity(-int64(allocCount)) - //没有达到gc阈值或者当前span正在被分配不做GC（异步gc该类span） + // 没有达到gc阈值或者当前span正在被分配不做GC（异步gc该类span） if gcCount < uintptr(float64(span.nelems)*SpanGCFactor) { return nil } @@ -115,14 +115,14 @@ func (x *xClassSpan) freeSpan(span *xSpan) (swap bool, size uint, err error) { needFree = true span.freeIndex = 0 span.allocCount = span.nelems - gcCount - //span.gcmarkBits.show64(span.nelems) + // span.gcmarkBits.show64(span.nelems) span.allocBits = span.gcmarkBits span.gcmarkBits, err = newMarkBits(span.nelems, true) if err != nil { return err } span.refillAllocCache(0) - //log.Printf("refillAllocCache allocCache:%.64b gcCount:%d allocCount:%d gcCount:%d oldallocCount:%d\n", span.allocCache, gcCount, span.allocCount, gcCount, allocCount) + // log.Printf("refillAllocCache allocCache:%.64b gcCount:%d allocCount:%d gcCount:%d oldallocCount:%d\n", span.allocCache, gcCount, span.allocCount, gcCount, allocCount) return nil }() if err != nil { @@ -131,8 +131,8 @@ func (x *xClassSpan) freeSpan(span *xSpan) (swap bool, size uint, err error) { if !needFree { return false, 0, nil } - //判断当前span是否不在使用，不在使用存放进去。在使用则 - //log.Printf("xClassSpan class:%d 回收 span:%d\n", x.classIndex, unsafe.Pointer(span)) + // 判断当前span是否不在使用，不在使用存放进去。在使用则 + // log.Printf("xClassSpan class:%d 回收 span:%d\n", x.classIndex, unsafe.Pointer(span)) x.free.insert(span) return true, size, nil } diff --git a/common.go b/common.go index b3499d3..0c61cd9 100644 --- a/common.go +++ b/common.go @@ -19,12 +19,13 @@ package xmm import ( - "github.com/spf13/cast" "log" "runtime/debug" "sync" "sync/atomic" "unsafe" + + "github.com/spf13/cast" ) func Recover() { @@ -37,7 +38,7 @@ func Recover() { } } -// align returns the smallest y >= x such that y % a == 0. +// Align returns the smallest y >= x such that y % a == 0. func Align(x, a uintptr) uintptr { if a < 1 { return x @@ -114,10 +115,10 @@ var oneBitCount = [256]uint8{ 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8} -//mSpanList 支持并发的操作 +// mSpanList 支持并发的操作 type mSpanList struct { first *xSpan // first span in list, or nil if none - //last *xSpan // last span in list, or nil if none + // last *xSpan // last span in list, or nil if none lock sync.Mutex } @@ -135,7 +136,7 @@ func (list *mSpanList) insert(span *xSpan) { if first == nil && atomic.CompareAndSwapPointer(addr, nil, unsafe.Pointer(span)) { return } - //先将新插入的赋值first，这时候会断裂为两个链。然后再赋值。 + // 先将新插入的赋值first，这时候会断裂为两个链。然后再赋值。 if first != nil && atomic.CompareAndSwapPointer(addr, first, unsafe.Pointer(span)) { span.next = (*xSpan)(first) return @@ -171,7 +172,7 @@ func (list *mSpanList) move(span *xSpan) { } pre = node } - //并发cas + // 并发cas var addr *unsafe.Pointer if pre != nil { addr = (*unsafe.Pointer)(unsafe.Pointer(&pre.next)) diff --git a/common_test.go b/common_test.go index 75686fe..61e8a00 100644 --- a/common_test.go +++ b/common_test.go @@ -48,7 +48,7 @@ func TestSpanList(t *testing.T) { if i != waitCnt { t.Fatal(u, "的数目不对 cnt:", i) } - //fmt.Println(u, i) + // fmt.Println(u, i) } if cnt != waitCnt*loopCnt { t.Fatal("总数不对") diff --git a/docs/README.md b/docs/README.md index cc98572..d7550fe 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,13 +12,13 @@ XMM 测试程序快速预览下载使用： 3. [XMM 使用 - 链表](https://github.com/heiyeluren/XMM/blob/main/example/xmm-test02.go) 4. [XMM 使用 - 哈希表](https://github.com/heiyeluren/XMM/blob/main/example/xmm-test03.go) -

## XMM 实现原理介绍 ### 1. [XMM 的核心设计与实现流程](https://github.com/heiyeluren/XMM/blob/main/docs/XMM-DesignImplementation.md) + ### 2. [XMM 设计实现技术调研参考](https://github.com/heiyeluren/XMM/blob/main/docs/XMM-InvestigateResearch.md)
diff --git a/docs/XMM-DesignImplementation.md b/docs/XMM-DesignImplementation.md index 262a3af..490497b 100644 --- a/docs/XMM-DesignImplementation.md +++ b/docs/XMM-DesignImplementation.md @@ -1,19 +1,18 @@ - # XMM 的核心设计与实现流程 - [XMM 的核心设计与实现流程](#xmm-的核心设计与实现流程) - - [设计思考与要求](#设计思考与要求) - - [1、模块设计图](#1模块设计图) - - [2、数据结构](#2数据结构) - - [3、流程图](#3流程图) - - [3.1、启动分配 Start](#31启动分配-start) - - [3.2、申请内存 Alloc](#32申请内存-alloc) - - [3.3、申请 span 流程 Alloc span](#33申请-span-流程-alloc-span) + - [设计思考与要求](#设计思考与要求) + - [1、模块设计图](#1模块设计图) + - [2、数据结构](#2数据结构) + - [3、流程图](#3流程图) + - [3.1、启动分配 Start](#31启动分配-start) + - [3.2、申请内存 Alloc](#32申请内存-alloc) + - [3.3、申请 span 流程 Alloc span](#33申请-span-流程-alloc-span) ## 设计思考与要求 -XMM - X(eXtensible) Memory Manager（完全自主研发的第三方 Go 内存分配管理器） -XMM 是一个在 Go 语言环境中完全自主实现的第三方内存管理库，不依赖于 Go 本身的任何内存管理能力，纯自主实现的 Go 内存管理库；能够应对各种场景下大小内存的 分配/释放/管理 等工作，能够帮助适用于任何复杂数据结构的构建（链表/数组/树/hash 等场景），能够良好完美的逃逸掉 Go 内置的 GC 机制，保证程序的超高性能，是构建高性能程序基础设施。 +XMM - X(eXtensible) Memory Manager（完全自主研发的第三方 Go 内存分配管理器） XMM 是一个在 Go 语言环境中完全自主实现的第三方内存管理库，不依赖于 Go 本身的任何内存管理能力，纯自主实现的 Go +内存管理库；能够应对各种场景下大小内存的 分配/释放/管理 等工作，能够帮助适用于任何复杂数据结构的构建（链表/数组/树/hash 等场景），能够良好完美的逃逸掉 Go 内置的 GC 机制，保证程序的超高性能，是构建高性能程序基础设施。 XMM 主要特点： @@ -24,8 +23,8 @@ XMM 主要特点：
-为了达成以上的目标，进行了很多内存分配器的调研学习，通过 Golang malloc/tcmalloc 的学习，发现 Golang 有高性能对象分配方式，但是需要对大对象 GC 买单：超大对象的 GC 会带来长时间的 STP。面对我们大数据量的 LocalCache 显然不是那么友好，不能满足我们需求，所以，我们需要设计一个不参与 GC 的高性能内存分配器。 -（更多实现细节建议阅读源码） +为了达成以上的目标，进行了很多内存分配器的调研学习，通过 Golang malloc/tcmalloc 的学习，发现 Golang 有高性能对象分配方式，但是需要对大对象 GC 买单：超大对象的 GC 会带来长时间的 +STP。面对我们大数据量的 LocalCache 显然不是那么友好，不能满足我们需求，所以，我们需要设计一个不参与 GC 的高性能内存分配器。 （更多实现细节建议阅读源码）

@@ -41,47 +40,47 @@ XMM 主要特点： ```go // 核心堆结构 type xHeap struct { - lock sync.mutex +lock sync.mutex - freeChunks *FreeChunkTree // 红黑树 - rawMemorys rawMemory +freeChunks *FreeChunkTree // 红黑树 +rawMemorys rawMemory - addrMap []*[]*rawMemory //addr -> page -> rawMemory 关系 - allchunk []*chunk +addrMap []*[]*rawMemory // addr -> page -> rawMemory 关系 +allchunk []*chunk } ​ -​// Span 池 +​ // Span 池 type spanPool struct{ - spans [classSize]*span - heap *xHeap +spans [classSize]*span +heap *xHeap } ​ // 写无锁 atomic、扩容必须得全局锁 type span struct{ - lock sync.mutex +lock sync.mutex - classIndex uint // class 的索引 - classSize uint // classSpan 的长度 +classIndex uint // class 的索引 +classSize uint // classSpan 的长度 - startAddr uintptr - npages uintptr - freeIndex uintptr - fact float32 // 扩容负载因子 +startAddr uintptr +npages uintptr +freeIndex uintptr +fact float32 // 扩容负载因子 } ​ // 连续 page 的管理 type chunk struct { - startAddr uintptr - npages uintptr +startAddr uintptr +npages uintptr } ​ // 用来管理 mmap 申请的内存，用于实际存放地址的元数据 type rawMemory struct { - addr uintptr - data []byte - down bool - next *rawMemory - chunks [pagesPerArena]*chunk +addr uintptr +data []byte +down bool +next *rawMemory +chunks [pagesPerArena]*chunk } ``` diff --git a/docs/XMM-INTRO-EN.md b/docs/XMM-INTRO-EN.md index 1a68145..1695f9b 100644 --- a/docs/XMM-INTRO-EN.md +++ b/docs/XMM-INTRO-EN.md @@ -1,27 +1,20 @@ - - - - # XMM (eXtensible) Memory Manager - High performance Go memory manager - [XMM (eXtensible) Memory Manager - fully-autonomous-third-party-go-memory-allocation-manager] - - What is [XMM?] (#xmm-what-is-it) - - [XMM has the following main features] - - [Why design XMM?] - - [Why design an autonomous memory manager?] - - [Why not use built-in data structures like map/slice?] - - [Why not use other open source memory pools?] - - [What is the final design conclusion of XMM?] - - [What are the goals of the XMM design?] - - [XMM Quick Start] - - [☆ XMM Use Cases ☆] - - [XMM Implementation Principles] - - [XMM Technology Exchange] - - - + - What is [XMM?] (#xmm-what-is-it) + - [XMM has the following main features] + - [Why design XMM?] + - [Why design an autonomous memory manager?] + - [Why not use built-in data structures like map/slice?] + - [Why not use other open source memory pools?] + - [What is the final design conclusion of XMM?] + - [What are the goals of the XMM design?] + - [XMM Quick Start] + - [☆ XMM Use Cases ☆] + - [XMM Implementation Principles] + - [XMM Technology Exchange]
@@ -29,23 +22,36 @@ XMM - X(eXtensible) Memory Manager (high performance third party Go memory allocation manager) -XMM is a third-party memory management library implemented in the Go language environment, which does not rely on any memory management capabilities of Go itself, and is a purely independent Go memory management library; it can handle the allocation/release/management of large and small memory in various scenarios, and can help with the construction of any complex data structures (chains/arrays/trees/hashes, etc.), and can escape Go's built-in GC mechanism well and perfectly to ensure the ultra-high performance of programs, which is the infrastructure for building high-performance programs. +XMM is a third-party memory management library implemented in the Go language environment, which does not rely on any +memory management capabilities of Go itself, and is a purely independent Go memory management library; it can handle the +allocation/release/management of large and small memory in various scenarios, and can help with the construction of any +complex data structures (chains/arrays/trees/hashes, etc.), and can escape Go's built-in GC mechanism well and perfectly +to ensure the ultra-high performance of programs, which is the infrastructure for building high-performance programs.
### XMM Key Features -1. XMM is a third-party memory management library implemented in the Go language environment, which does not rely on any memory management capabilities of Go itself, and is implemented in 6000 lines of pure Go code. - -2. XMM can handle the allocation/release/management of memory in various scenarios, and can help build complex data structures such as chained tables/arrays/trees/hash tables, etc. XMM allows you to use system memory as easily and conveniently as C/C++, without worrying about performance. +1. XMM is a third-party memory management library implemented in the Go language environment, which does not rely on any + memory management capabilities of Go itself, and is implemented in 6000 lines of pure Go code. -3. XMM is a good and perfect way to escape Go's built-in GC mechanism to ensure high performance and is the infrastructure for building high performance programs; however, unlike sync. XMM is more like a memory allocator such as TcMalloc. 4. +2. XMM can handle the allocation/release/management of memory in various scenarios, and can help build complex data + structures such as chained tables/arrays/trees/hash tables, etc. XMM allows you to use system memory as easily and + conveniently as C/C++, without worrying about performance. -4. XMM is process-safe and has very high allocation performance, currently reaching 350w alloc/s on a normal Linux server, which means that it can perform 3.5 million memory allocation operations per second without lagging, making it ideal for scenarios where you want to manage memory autonomously and with high performance. +3. XMM is a good and perfect way to escape Go's built-in GC mechanism to ensure high performance and is the + infrastructure for building high performance programs; however, unlike sync. XMM is more like a memory allocator such + as TcMalloc. 4. -5. The XMM memory library has a simple interface and is compatible with Go 1.8 and above, so it is easy to get started (go 1.12+ is recommended) and can be used to reconstruct all the high-performance data structures you want on top of XMM, such as map/slice and so on. (The examples section can be used as a reference for some data structure implementations) +4. XMM is process-safe and has very high allocation performance, currently reaching 350w alloc/s on a normal Linux + server, which means that it can perform 3.5 million memory allocation operations per second without lagging, making + it ideal for scenarios where you want to manage memory autonomously and with high performance. +5. The XMM memory library has a simple interface and is compatible with Go 1.8 and above, so it is easy to get started ( + go 1.12+ is recommended) and can be used to reconstruct all the high-performance data structures you want on top of + XMM, such as map/slice and so on. (The examples section can be used as a reference for some data structure + implementations)

@@ -56,44 +62,68 @@ XMM is a third-party memory management library implemented in the Go language en ### Why design a third party memory manager? -In order to cope with a variety of memory management scenarios, it may be necessary to have some scenarios where memory is used autonomously in addition to the built-in data structures, such as building complex high performance data structures, minimising Go's GC mechanism in the case of large memory footprint, or a very large number of small memory blocks, and ensuring stable service performance without jitter due to GC. +In order to cope with a variety of memory management scenarios, it may be necessary to have some scenarios where memory +is used autonomously in addition to the built-in data structures, such as building complex high performance data +structures, minimising Go's GC mechanism in the case of large memory footprint, or a very large number of small memory +blocks, and ensuring stable service performance without jitter due to GC.
- ### Why not use Go's built-in data structures like map/slice? -Golang itself, for the sake of performance and memory control, is completely closed to the public, and has its own GC mechanism, so it is more troublesome to manage memory independently; the GC mechanism that comes with Go has been iterated over many versions, and its performance is already very good so far, but in a large-scale fragmented memory block, GC will still have some loss, and in extreme high-performance scenarios, GC will In extreme high performance scenarios, GC can cause the entire background application service to underperform (or occasionally stall). So in a nutshell, Go itself is affected by GC in terms of performance of pointers and so on, which always leads to poor performance of the service. +Golang itself, for the sake of performance and memory control, is completely closed to the public, and has its own GC +mechanism, so it is more troublesome to manage memory independently; the GC mechanism that comes with Go has been +iterated over many versions, and its performance is already very good so far, but in a large-scale fragmented memory +block, GC will still have some loss, and in extreme high-performance scenarios, GC will In extreme high performance +scenarios, GC can cause the entire background application service to underperform (or occasionally stall). So in a +nutshell, Go itself is affected by GC in terms of performance of pointers and so on, which always leads to poor +performance of the service.
### Why not use other open source memory pools? -1. in addition to Go's own memory module, research understands that most of the existing third-party object pools/memory pools/byte pools and other scenarios that require a certain piece of autonomous memory operations are basically in the form of Map/sync. +1. in addition to Go's own memory module, research understands that most of the existing third-party object pools/memory + pools/byte pools and other scenarios that require a certain piece of autonomous memory operations are basically in + the form of Map/sync. -2. Pool is suitable for storing various types of data, but has a high probability of GC; sync.Pool is suitable for storing reused temporary objects, and can also be used for various data bodies, which can reduce GC appropriately (it cannot avoid GC); Bytes[] is suitable for storing byte data, and can only store byte data, which can avoid GC scanning as much as possible through certain processing; (compare with [Go language's byte pool based on channel-based implementation of concurrency-safe byte pooling](https://zhuanlan.zhihu.com/p/265790840)) +2. Pool is suitable for storing various types of data, but has a high probability of GC; sync.Pool is suitable for + storing reused temporary objects, and can also be used for various data bodies, which can reduce GC appropriately (it + cannot avoid GC); Bytes[] is suitable for storing byte data, and can only store byte data, which can avoid GC + scanning as much as possible through certain processing; (compare + with [Go language's byte pool based on channel-based implementation of concurrency-safe byte pooling](https://zhuanlan.zhihu.com/p/265790840)) -3. existing open source libraries include: mcache [gopkg/mcache.go](https://github.com/bytedance/gopkg/blob/main/lang/mcache/mcache.go), which relies on sync.Pool, for example, and [Bytes[]], which uses Bytes[] Pool [bpool minio/bpool.go](https://github.com/minio/minio/blob/master/internal/bpool/bpool.go), for example, and [bpool minio/bpool.go](https://github.com/minio/minio/blob/master/internal/bpool/bpool.go) for MinIO. +3. existing open source libraries include: + mcache [gopkg/mcache.go](https://github.com/bytedance/gopkg/blob/main/lang/mcache/mcache.go), which relies on + sync.Pool, for example, and [Bytes[]], which uses Bytes[] + Pool [bpool minio/bpool.go](https://github.com/minio/minio/blob/master/internal/bpool/bpool.go), for example, + and [bpool minio/bpool.go](https://github.com/minio/minio/blob/master/internal/bpool/bpool.go) for MinIO. -4. Conclusion: XMM is completely different from their implementation mechanism, XMM is closer to Go's built-in memory allocation mechanism principle +4. Conclusion: XMM is completely different from their implementation mechanism, XMM is closer to Go's built-in memory + allocation mechanism principle
### What is the final design conclusion of the XMM? -The XMM module has been implemented from scratch in order to escape Golang's GC mechanism and to have a fully autonomous memory management operation, without any jitter caused by Go's own GC mechanism in the case of thousands of small objects. +The XMM module has been implemented from scratch in order to escape Golang's GC mechanism and to have a fully autonomous +memory management operation, without any jitter caused by Go's own GC mechanism in the case of thousands of small +objects.

### What are the XMM design goals? -To ensure high performance, the XMM was designed with three core objectives in mind. -1. a single machine (6-core KVM or physical machine) memory allocation performance of 350w+ alloc/s; (memory allocation speed per second). +To ensure high performance, the XMM was designed with three core objectives in mind. -2. support for calling the user to manually force a block of memory to be free, or support for XMM itself to automatically GC some memory banks that are not manually free; (autonomous implementation of GC function) +1. a single machine (6-core KVM or physical machine) memory allocation performance of 350w+ alloc/s; (memory allocation + speed per second). -3. no memory leaks, and memory management is not crude, but granular, fully comparable to the industry's mainstream memory management allocators. +2. support for calling the user to manually force a block of memory to be free, or support for XMM itself to + automatically GC some memory banks that are not manually free; (autonomous implementation of GC function) +3. no memory leaks, and memory management is not crude, but granular, fully comparable to the industry's mainstream + memory management allocators.

@@ -113,25 +143,28 @@ Description: A quick preview of the XMM test program to download and use

- ## Introduction to the principle of XMM implementation 1. [Core Design and Implementation Process of XMM](https://github.com/heiyeluren/XMM/blob/main/docs/XMM-DesignImplementation.md) 1. [XMM design and implementation technology research reference](https://github.com/heiyeluren/XMM/blob/main/docs/XMM-InvestigateResearch.md) -

### XMM Technology Exchange Community -XMM is currently an early version, the overall performance is relatively good, and is also currently used in another self-developed XMap module, of course, there are also some problems and bugs, welcome everyone to create together, you can submit issues and PR, etc.. +XMM is currently an early version, the overall performance is relatively good, and is also currently used in another +self-developed XMap module, of course, there are also some problems and bugs, welcome everyone to create together, you +can submit issues and PR, etc.. -You can also send emails to the author for communication, and if you are convenient to use WeChat, you can add the author's WeChat. +You can also send emails to the author for communication, and if you are convenient to use WeChat, you can add the +author's WeChat.
#### Author's email: heiyeluren@gmail.com / heiyeluren@qq.com + #### Author's WeChat: (swipe to add) +
diff --git a/docs/XMM-InvestigateResearch.md b/docs/XMM-InvestigateResearch.md index 9122c48..8d01528 100644 --- a/docs/XMM-InvestigateResearch.md +++ b/docs/XMM-InvestigateResearch.md @@ -1,20 +1,22 @@ # XMM 参考调研 - TCMalloc&Go 内存管理调研 - [XMM 参考调研 - TCMalloc&Go 内存管理调研](#xmm-参考调研---tcmallocgo-内存管理调研) - - [1、调研背景](#1调研背景) - - [2、TcMalloc 工作机制](#2tcmalloc-工作机制) - - [数据模型](#数据模型) - - [3、Go 内存分配机制](#3go-内存分配机制) - - [3.1、数据模型](#31数据模型) - - [3.2、内存初始化](#32内存初始化) - - [3.2、对象申请内存](#32对象申请内存) - - [相关参考文档](#相关参考文档) + - [1、调研背景](#1调研背景) + - [2、TcMalloc 工作机制](#2tcmalloc-工作机制) + - [数据模型](#数据模型) + - [3、Go 内存分配机制](#3go-内存分配机制) + - [3.1、数据模型](#31数据模型) + - [3.2、内存初始化](#32内存初始化) + - [3.2、对象申请内存](#32对象申请内存) + - [相关参考文档](#相关参考文档) ## 1、调研背景 -为了解决 Golang 的大内存 GC 问题，需要深入了解 Golang 的内存分配原理，Golang 的内存分配器思想来源于 TCMalloc，他继承了 TCMalloc 的高性能、高内存利用率等优点。实际上与 TCMalloc 有区别的，同时融入了自己的定制化内容。现在我们先了解下 TCMalloc 的实现原理。 +为了解决 Golang 的大内存 GC 问题，需要深入了解 Golang 的内存分配原理，Golang 的内存分配器思想来源于 TCMalloc，他继承了 TCMalloc 的高性能、高内存利用率等优点。实际上与 TCMalloc +有区别的，同时融入了自己的定制化内容。现在我们先了解下 TCMalloc 的实现原理。 -说明：TcMalloc - Multi-threaded memory allocate（Goolge 开发的内存分配器）[github.com/google/tcmalloc](https://github.com/google/tcmalloc) +说明：TcMalloc - Multi-threaded memory allocate（Goolge +开发的内存分配器）[github.com/google/tcmalloc](https://github.com/google/tcmalloc)
@@ -24,7 +26,8 @@
-在多线程环境下，TCMalloc 可以极大减少锁资源的争夺。针对 small object，TCMalloc 几乎就是 lock free 的。针对 large object，TCMalloc 采用高效的细粒度的自旋锁。同时内存也做了更好管理更加精细化，较少了内存空洞。 +在多线程环境下，TCMalloc 可以极大减少锁资源的争夺。针对 small object，TCMalloc 几乎就是 lock free 的。针对 large object，TCMalloc +采用高效的细粒度的自旋锁。同时内存也做了更好管理更加精细化，较少了内存空洞。
@@ -64,7 +67,6 @@
- ### 3.2、对象申请内存 ![这是图片](https://raw.githubusercontent.com/heiyeluren/XMM/main/docs/img/ir04.png) diff --git a/docs/XMM-Usage-EN.md b/docs/XMM-Usage-EN.md index 13607a6..daa8e41 100644 --- a/docs/XMM-Usage-EN.md +++ b/docs/XMM-Usage-EN.md @@ -1,17 +1,15 @@ - -## How to use XMM （XMM Usage） +## How to use XMM （XMM Usage） - [XMM How to Use Case Illustrations] - - [Example 1: XMM Quick Start] - - [example two: how to use XMM in structures] - - [Example three: using XMM to build a chain table] - - [Example IV: Using XMM to build a hash table] + - [Example 1: XMM Quick Start] + - [example two: how to use XMM in structures] + - [Example three: using XMM to build a chain table] + - [Example IV: Using XMM to build a hash table] - [XMM problem feedback] -
+

- ### Example 1: A quick introduction to using XMM XMM is very simple and easy to use, so let's look at it directly in code. @@ -24,7 +22,8 @@ Example 1: See a quick start example of a simple common variable stored using XM ### Example 2: How to use XMM in a structure -Illustration: A slightly more complex application of how to use XMM in a structure for requesting and releasing memory operations. +Illustration: A slightly more complex application of how to use XMM in a structure for requesting and releasing memory +operations. - [XMM Usage - Structs](https://github.com/heiyeluren/XMM/blob/main/example/xmm-test01.go) @@ -40,7 +39,9 @@ Description: Build some complex data structure applications with XMM, constructi ### Example 4: Building a hash table with XMM -Description: Build some complex data structure applications with XMM, build a hash table data structure to use, you can carefully refer to see if it helps you to use XMM +Description: Build some complex data structure applications with XMM, build a hash table data structure to use, you can +carefully refer to see if it helps you to use XMM + - [XMM Usage - Hash Tables](https://github.com/heiyeluren/XMM/blob/main/example/xmm-test03.go)
@@ -49,14 +50,19 @@ Description: Build some complex data structure applications with XMM, build a ha ### XMM Technology Exchange Community -XMM is currently an early version, the overall performance is relatively good, and is also currently used in another self-developed XMap module, of course, there are also some problems and bugs, welcome everyone to create together, you can submit issues and PR, etc.. +XMM is currently an early version, the overall performance is relatively good, and is also currently used in another +self-developed XMap module, of course, there are also some problems and bugs, welcome everyone to create together, you +can submit issues and PR, etc.. -You can also send emails to the author for communication, and if you are convenient to use WeChat, you can add the author's WeChat. +You can also send emails to the author for communication, and if you are convenient to use WeChat, you can add the +author's WeChat.
#### Author's email: heiyeluren@gmail.com / heiyeluren@qq.com + #### Author's WeChat: (swipe to add) +
diff --git a/docs/XMM-Usage.md b/docs/XMM-Usage.md index 870aff9..7484385 100644 --- a/docs/XMM-Usage.md +++ b/docs/XMM-Usage.md @@ -1,14 +1,13 @@ - ## XMM 如何使用案例说明 - [XMM 如何使用案例说明](#xmm-如何使用案例说明) - - [示例一：XMM 快速使用入门](#示例一xmm-快速使用入门) - - [示例二：如何在结构体中使用 XMM](#示例二如何在结构体中使用-xmm) - - [示例三：使用 XMM 构建一个链表](#示例三使用-xmm-构建一个链表) - - [示例四：使用 XMM 构建一个哈希表](#示例四使用-xmm-构建一个哈希表) + - [示例一：XMM 快速使用入门](#示例一xmm-快速使用入门) + - [示例二：如何在结构体中使用 XMM](#示例二如何在结构体中使用-xmm) + - [示例三：使用 XMM 构建一个链表](#示例三使用-xmm-构建一个链表) + - [示例四：使用 XMM 构建一个哈希表](#示例四使用-xmm-构建一个哈希表) - [XMM问题反馈](#xmm问题反馈) -
-
+
+

@@ -19,7 +18,6 @@ 3. [XMM 使用 - 链表](https://github.com/heiyeluren/XMM/blob/main/example/xmm-test02.go) 4. [XMM 使用 - 哈希表](https://github.com/heiyeluren/XMM/blob/main/example/xmm-test03.go) -
### 示例一：XMM 快速使用入门 @@ -30,60 +28,60 @@ XMM 的使用非常简单方便，我们直接通过代码来看。 ```go /* - XMM 示例00 + XMM 示例00 - 目标：如何简单快速使用XMM - 说明：就是示例如何快速简单使用XMM内存库 + 目标：如何简单快速使用XMM + 说明：就是示例如何快速简单使用XMM内存库 */ package main import ( - //xmm "xmm/src" - xmm "github.com/heiyeluren/xmm" - "fmt" - "unsafe" + // xmm "xmm/src" + xmm "github.com/heiyeluren/xmm" + "fmt" + "unsafe" ) func main() { - - //初始化XMM对象 - f := &xmm.Factory{} - - //从操作系统申请一个内存块 - //如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值），0.6这个数值可以自主配置 - mm, err := f.CreateMemory(0.6) - if err != nil { - panic("CreateMemory fail ") - } - - //操作int类型，申请内存后赋值 - var tmpNum int = 9527 - p, err := mm.Alloc(unsafe.Sizeof(tmpNum)) - if err != nil { - panic("Alloc fail ") - } - Id := (*int)(p) - //把设定好的数字赋值写入到XMM内存中 - *Id = tmpNum - - //操作字符串类型，XMM提供了From()接口，可以直接获得一个指针，字符串会存储在XMM中 - Name, err := mm.From("heiyeluren") - if err != nil { - panic("Alloc fail ") - } - - //从XMM内存中输出变量和内存地址 - fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 00 ======\n") - fmt.Println("\n-- Memory data status --\n") - fmt.Println(" Id :", *Id, "\t\t( Id ptr addr: ", Id, ")" ) - fmt.Println(" Name:", Name, "\t( Name ptr addr: ", &Name, ")") - - fmt.Println("\n===== Example test success ======\n") - - //释放Id,Name内存块 - mm.Free(uintptr(p)) - mm.FreeString(Name) + + // 初始化XMM对象 + f := &xmm.Factory{} + + // 从操作系统申请一个内存块 + // 如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值），0.6这个数值可以自主配置 + mm, err := f.CreateMemory(0.6) + if err != nil { + panic("CreateMemory fail ") + } + + // 操作int类型，申请内存后赋值 + var tmpNum int = 9527 + p, err := mm.Alloc(unsafe.Sizeof(tmpNum)) + if err != nil { + panic("Alloc fail ") + } + Id := (*int)(p) + // 把设定好的数字赋值写入到XMM内存中 + *Id = tmpNum + + // 操作字符串类型，XMM提供了From()接口，可以直接获得一个指针，字符串会存储在XMM中 + Name, err := mm.From("heiyeluren") + if err != nil { + panic("Alloc fail ") + } + + // 从XMM内存中输出变量和内存地址 + fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 00 ======\n") + fmt.Println("\n-- Memory data status --\n") + fmt.Println(" Id :", *Id, "\t\t( Id ptr addr: ", Id, ")") + fmt.Println(" Name:", Name, "\t( Name ptr addr: ", &Name, ")") + + fmt.Println("\n===== Example test success ======\n") + + // 释放Id,Name内存块 + mm.Free(uintptr(p)) + mm.FreeString(Name) } @@ -94,12 +92,10 @@ func main() {

- ### 示例二：如何在结构体中使用 XMM 说明：稍微复杂一点的应用，如何在结构体中使用 XMM，进行申请和释放内存操作： - ```go /* @@ -111,72 +107,71 @@ func main() { package main import ( - //xmm "xmm/src" - xmm "github.com/heiyeluren/xmm" - "fmt" - "unsafe" + // xmm "xmm/src" + xmm "github.com/heiyeluren/xmm" + "fmt" + "unsafe" ) func main() { - - //定义一个类型(结构体) - type User struct { - Id uint - Name string - Age uint - Email string - Salary float32 - } - - //初始化XMM对象 - f := &xmm.Factory{} - //从操作系统申请一个内存块 - //如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值），0.6这个数值可以自主配置 - mm, err := f.CreateMemory(0.6) - if err != nil { - panic("CreateMemory fail ") - } - - //自己从内存块中申请一小段自己想用的内存 - size := unsafe.Sizeof(User{}) - p, err := mm.Alloc(size) - if err != nil { - panic("Alloc fail ") - } - - //使用该内存块，进行结构体元素赋值 - user := (*User)(p) - user.Id = 1 - user.Age = 18 - user.Name = "heiyeluren" - user.Email = "heiyeluren@example.com" - - //输出变量，打印整个结构体等 - fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 01 ======\n") - - fmt.Println("\n-- Memory data status --\n") - fmt.Println("User ptr addr: \t", p) - fmt.Println("User data: \t", user) - - //释放内存块（实际是做mark标记操作） - mm.Free(uintptr(p)) - - //Free()后再看看变量值，只是针对这个内存块进行mark标记动作，并未彻底从内存中释放（XMM设计机制，降低实际gc回收空闲时间） - //XMM内部会有触发gc的机制，主要是内存容量，参数TotalGCFactor=0.0004，目前如果要配置，需要自己修改这个常量，一般不用管它，Free()操作中有万分之4的概率会命中触发gc~ - //GC触发策略：待释放内存 > 总内存 * 万分之4 会触发gc动作 - fmt.Println("\n-- Memory data status after XMM.Free() --\n") - fmt.Println("memory ptr addr:\t", p) - fmt.Println("User data:\t\t", user) - - //结束 - fmt.Println("\n===== Example test success ======\n") + + // 定义一个类型(结构体) + type User struct { + Id uint + Name string + Age uint + Email string + Salary float32 + } + + // 初始化XMM对象 + f := &xmm.Factory{} + // 从操作系统申请一个内存块 + // 如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值），0.6这个数值可以自主配置 + mm, err := f.CreateMemory(0.6) + if err != nil { + panic("CreateMemory fail ") + } + + // 自己从内存块中申请一小段自己想用的内存 + size := unsafe.Sizeof(User{}) + p, err := mm.Alloc(size) + if err != nil { + panic("Alloc fail ") + } + + // 使用该内存块，进行结构体元素赋值 + user := (*User)(p) + user.Id = 1 + user.Age = 18 + user.Name = "heiyeluren" + user.Email = "heiyeluren@example.com" + + // 输出变量，打印整个结构体等 + fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 01 ======\n") + + fmt.Println("\n-- Memory data status --\n") + fmt.Println("User ptr addr: \t", p) + fmt.Println("User data: \t", user) + + // 释放内存块（实际是做mark标记操作） + mm.Free(uintptr(p)) + + // Free()后再看看变量值，只是针对这个内存块进行mark标记动作，并未彻底从内存中释放（XMM设计机制，降低实际gc回收空闲时间） + // XMM内部会有触发gc的机制，主要是内存容量，参数TotalGCFactor=0.0004，目前如果要配置，需要自己修改这个常量，一般不用管它，Free()操作中有万分之4的概率会命中触发gc~ + // GC触发策略：待释放内存 > 总内存 * 万分之4 会触发gc动作 + fmt.Println("\n-- Memory data status after XMM.Free() --\n") + fmt.Println("memory ptr addr:\t", p) + fmt.Println("User data:\t\t", user) + + // 结束 + fmt.Println("\n===== Example test success ======\n") } ``` -


@@ -187,7 +182,6 @@ func main() {
- ```go /* XMM 简单示例 - 02 @@ -198,130 +192,128 @@ func main() { package main import ( - //xmm "xmm/src" - xmm "github.com/heiyeluren/xmm" - "fmt" - "unsafe" + // xmm "xmm/src" + xmm "github.com/heiyeluren/xmm" + "fmt" + "unsafe" ) -//定义一个链表的节点结构 +// 定义一个链表的节点结构 type XListNode struct { - Val int - Next *XListNode + Val int + Next *XListNode } -//单链表主结构 +// 单链表主结构 type XList struct { - Head *XListNode - //Tail *XListNode + Head *XListNode + // Tail *XListNode } - -//初始化链表 -func (l *XList) Init( mm xmm.XMemory ) { - Node,err := mm.Alloc(unsafe.Sizeof(XListNode{})) - if err != nil { - panic("Alloc fail") - } - //head := &XListNode{Val:list[0]} - head := (*XListNode)(Node) - l.Head = head - fmt.Println("Init Xlist done") +// 初始化链表 +func (l *XList) Init(mm xmm.XMemory) { + Node, err := mm.Alloc(unsafe.Sizeof(XListNode{})) + if err != nil { + panic("Alloc fail") + } + // head := &XListNode{Val:list[0]} + head := (*XListNode)(Node) + l.Head = head + fmt.Println("Init Xlist done") } -//链表增加节点 +// 链表增加节点 func (l *XList) Append(i int, mm xmm.XMemory) { - h := l.Head - for h.Next != nil { - h = h.Next - } - p, err := mm.Alloc(unsafe.Sizeof(XListNode{})) - if err != nil { - panic("Alloc fail") - } - Node := (*XListNode)(p) - Node.Val = i - Node.Next = nil - h.Next = Node - - fmt.Println("Append item:", Node.Val) + h := l.Head + for h.Next != nil { + h = h.Next + } + p, err := mm.Alloc(unsafe.Sizeof(XListNode{})) + if err != nil { + panic("Alloc fail") + } + Node := (*XListNode)(p) + Node.Val = i + Node.Next = nil + h.Next = Node + + fmt.Println("Append item:", Node.Val) } -//遍历所有链表节点并打印 +// 遍历所有链表节点并打印 func (l *XList) Show() { - h := l.Head - //fmt.Println(h.Val) - for h.Next != nil { - h = h.Next - fmt.Println("Show item:", h.Val) - } + h := l.Head + // fmt.Println(h.Val) + for h.Next != nil { + h = h.Next + fmt.Println("Show item:", h.Val) + } } -//释放整个链表结构 +// 释放整个链表结构 func (l *XList) Destroy(mm xmm.XMemory) { - cnt := 0 - h := l.Head - //fmt.Println(h.Val) - - //统计需要释放总数 - for h.Next != nil { - h = h.Next - cnt++ - //fmt.Println(h.Val) - } - //fmt.Println("item count:", cnt) - - //循环释放所有内存 - for i := 0; i <= cnt; i++ { - h := l.Head - pre := l.Head - for h.Next != nil { - pre = h - h = h.Next - } - fmt.Println("Free item:", h.Val) - mm.Free(uintptr(unsafe.Pointer(h))) - pre.Next = nil - } + cnt := 0 + h := l.Head + // fmt.Println(h.Val) + + // 统计需要释放总数 + for h.Next != nil { + h = h.Next + cnt++ + // fmt.Println(h.Val) + } + // fmt.Println("item count:", cnt) + + // 循环释放所有内存 + for i := 0; i <= cnt; i++ { + h := l.Head + pre := l.Head + for h.Next != nil { + pre = h + h = h.Next + } + fmt.Println("Free item:", h.Val) + mm.Free(uintptr(unsafe.Pointer(h))) + pre.Next = nil + } } -//主函数 +// 主函数 func main() { - - //初始化XMM对象 - f := &xmm.Factory{} - //从操作系统申请一个内存块，如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值） - mm, err := f.CreateMemory(0.6) - if err != nil { - panic("CreateMemory fail ") - } - - //要生成的数字列表 - list := []int{ 2, 4, 3} - - fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 02 - LinkedList ======\n") - - - //初始化链表 - l := &XList{} - l.Init(mm) - fmt.Println("") - - //把元素压入链表 - for i := 0; i < len(list); i++ { - l.Append(list[i], mm) - } - fmt.Println("") - - //遍历所有链表数据 - l.Show() - fmt.Println("") - - //释放所有链表内存 - l.Destroy(mm) - - //结束 - fmt.Println("\n===== Example test success ======\n") + + // 初始化XMM对象 + f := &xmm.Factory{} + // 从操作系统申请一个内存块，如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值） + mm, err := f.CreateMemory(0.6) + if err != nil { + panic("CreateMemory fail ") + } + + // 要生成的数字列表 + list := []int{2, 4, 3} + + fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 02 - LinkedList ======\n") + + // 初始化链表 + l := &XList{} + l.Init(mm) + fmt.Println("") + + // 把元素压入链表 + for i := 0; i < len(list); i++ { + l.Append(list[i], mm) + } + fmt.Println("") + + // 遍历所有链表数据 + l.Show() + fmt.Println("") + + // 释放所有链表内存 + l.Destroy(mm) + + // 结束 + fmt.Println("\n===== Example test success ======\n") } @@ -329,8 +321,6 @@ func main() { ``` - -


@@ -351,346 +341,339 @@ func main() { package main import ( - //xmm "xmm/src" - xmm "github.com/heiyeluren/xmm" - "strconv" - "strings" - "fmt" - "unsafe" - "encoding/json" + // xmm "xmm/src" + xmm "github.com/heiyeluren/xmm" + "strconv" + "strings" + "fmt" + "unsafe" + "encoding/json" ) -//定义哈希表桶数量 +// 定义哈希表桶数量 const HashTableBucketMax = 1024 -//定义哈希表存储实际KV节点存储数据 +// 定义哈希表存储实际KV节点存储数据 type XEntity struct { - Key string //Key，必须是字符串 - Value string //Value值，是字符串，把interface{}转json + Key string // Key，必须是字符串 + Value string // Value值，是字符串，把interface{}转json } -//定义哈希表中单个桶结构（开拉链法） +// 定义哈希表中单个桶结构（开拉链法） type XBucket struct { - Data *XEntity //当前元素KV - Next *XBucket //如果冲突情况开拉链法下一个节点的Next指针 + Data *XEntity // 当前元素KV + Next *XBucket // 如果冲突情况开拉链法下一个节点的Next指针 } -//哈希表入口主结构HashTable +// 哈希表入口主结构HashTable type XHashTable struct { - Table []*XBucket //哈希表所有桶存储池 - Size uint64 //哈希表已存总元素数量 - mm xmm.XMemory //XMM内存管理对象 + Table []*XBucket // 哈希表所有桶存储池 + Size uint64 // 哈希表已存总元素数量 + mm xmm.XMemory // XMM内存管理对象 } - -//初始化哈希表 -func (h *XHashTable) Init( mm xmm.XMemory ) { - //设置需要申请多大的连续数组内存空间，如果是动态扩容情况建议设置为16，目前是按照常量桶大小 - cap := HashTableBucketMax - initCap := uintptr(cap) - //申请按照设定总桶数量大小的内存块 - p, err := mm.AllocSlice(unsafe.Sizeof(&XBucket{}), initCap, initCap) - if err != nil { - panic("Alloc fail") - } - //把申请的内存库给哈希表总存储池，初始化数量和XMM内存对象 - h.Table = *(*[]*XBucket)(p) - h.Size = 0 - h.mm = mm - - fmt.Println("Init XHashTable done") +// 初始化哈希表 +func (h *XHashTable) Init(mm xmm.XMemory) { + // 设置需要申请多大的连续数组内存空间，如果是动态扩容情况建议设置为16，目前是按照常量桶大小 + cap := HashTableBucketMax + initCap := uintptr(cap) + // 申请按照设定总桶数量大小的内存块 + p, err := mm.AllocSlice(unsafe.Sizeof(&XBucket{}), initCap, initCap) + if err != nil { + panic("Alloc fail") + } + // 把申请的内存库给哈希表总存储池，初始化数量和XMM内存对象 + h.Table = *(*[]*XBucket)(p) + h.Size = 0 + h.mm = mm + + fmt.Println("Init XHashTable done") } -//释放整个哈希表 +// 释放整个哈希表 func (h *XHashTable) Destroy() (error) { - return nil + return nil } -//哈希表 Set数据 -func (h *XHashTable) Set(Key string, Value interface{}) (error) { - - //-------------- - // 构造Entity - //-------------- - //Value进行序列化 - jdata, err := json.Marshal(Value) - if err != nil { - fmt.Println("Set() op Value [", Value, "] json encode fail") - return err - } - sValue := string(jdata) - - //申请三个内存，操作字符串类型，XMM提供了From()接口，可以直接获得一个指针，字符串会存储在XMM中 - //size := unsafe.Sizeof(User{}) - p, err := h.mm.Alloc(unsafe.Sizeof(XEntity{})) - if err != nil { - panic("Alloc fail ") - } - pKey, err := h.mm.From(Key) - if err != nil { - panic("Alloc fail ") - } - pVal, err := h.mm.From(sValue) - if err != nil { - panic("Alloc fail ") - } - - //拼装Entity - pEntity := (*XEntity)(p) - pEntity.Key = pKey - pEntity.Value = pVal - - //--------------- - // 挂接到Bucket - //--------------- - bucketIdx := getBucketSlot(Key) - //bucketSize := unsafe.Sizeof(&XBucket{}) - bucket := h.Table[bucketIdx] - - //构造bucket - pb, err := h.mm.Alloc(unsafe.Sizeof(XBucket{})) - if err != nil { - panic("Alloc fail ") - } - pBucket := (*XBucket)(pb) - pBucket.Data = pEntity - pBucket.Next = nil - - //如果槽没有被占用则压入数据后结束 - if bucket == nil { - h.Table[bucketIdx] = pBucket - h.Size = h.Size + 1 - return nil - } - - //使用开拉链法把KV放入到冲突的槽中 - var k string - for bucket != nil { - k = bucket.Data.Key - //如果发现有重名key，则直接替换Value - if strings.Compare(strings.ToLower(Key), strings.ToLower(k)) == 0 { - //释放原Value内存，挂接新Value - //pv := bucket.Data.Value - //mm.Free(pv) - pValNew, err := h.mm.From(sValue) - if err != nil { - panic("Alloc fail ") - } - bucket.Data.Value = pValNew - return nil - } - //如果是最后一个拉链的节点，则把当前KV挂上去 - if bucket.Next == nil { - bucket.Next = pBucket - h.Size = h.Size + 1 - return nil - } - //没找到则继续循环 - bucket = bucket.Next - } - return nil +// 哈希表 Set数据 +func (h *XHashTable) Set(Key string, Value interface{}) (error) { + + // -------------- + // 构造Entity + // -------------- + // Value进行序列化 + jdata, err := json.Marshal(Value) + if err != nil { + fmt.Println("Set() op Value [", Value, "] json encode fail") + return err + } + sValue := string(jdata) + + // 申请三个内存，操作字符串类型，XMM提供了From()接口，可以直接获得一个指针，字符串会存储在XMM中 + // size := unsafe.Sizeof(User{}) + p, err := h.mm.Alloc(unsafe.Sizeof(XEntity{})) + if err != nil { + panic("Alloc fail ") + } + pKey, err := h.mm.From(Key) + if err != nil { + panic("Alloc fail ") + } + pVal, err := h.mm.From(sValue) + if err != nil { + panic("Alloc fail ") + } + + // 拼装Entity + pEntity := (*XEntity)(p) + pEntity.Key = pKey + pEntity.Value = pVal + + // --------------- + // 挂接到Bucket + // --------------- + bucketIdx := getBucketSlot(Key) + // bucketSize := unsafe.Sizeof(&XBucket{}) + bucket := h.Table[bucketIdx] + + // 构造bucket + pb, err := h.mm.Alloc(unsafe.Sizeof(XBucket{})) + if err != nil { + panic("Alloc fail ") + } + pBucket := (*XBucket)(pb) + pBucket.Data = pEntity + pBucket.Next = nil + + // 如果槽没有被占用则压入数据后结束 + if bucket == nil { + h.Table[bucketIdx] = pBucket + h.Size = h.Size + 1 + return nil + } + + // 使用开拉链法把KV放入到冲突的槽中 + var k string + for bucket != nil { + k = bucket.Data.Key + // 如果发现有重名key，则直接替换Value + if strings.Compare(strings.ToLower(Key), strings.ToLower(k)) == 0 { + // 释放原Value内存，挂接新Value + // pv := bucket.Data.Value + // mm.Free(pv) + pValNew, err := h.mm.From(sValue) + if err != nil { + panic("Alloc fail ") + } + bucket.Data.Value = pValNew + return nil + } + // 如果是最后一个拉链的节点，则把当前KV挂上去 + if bucket.Next == nil { + bucket.Next = pBucket + h.Size = h.Size + 1 + return nil + } + // 没找到则继续循环 + bucket = bucket.Next + } + return nil } -//哈希表 Get数据 -func (h *XHashTable) Get(Key string) (interface{}, error) { - var k string - var val interface{} - bucketIdx := getBucketSlot(Key) - //bucketSize := unsafe.Sizeof(&XBucket{}) - bucket := h.Table[bucketIdx] - for bucket != nil { - k = bucket.Data.Key - //如果查找到相同Key开始返回数据 - if strings.Compare(strings.ToLower(Key), strings.ToLower(k)) == 0 { - //bTmp := []byte(k) - err := json.Unmarshal([]byte(bucket.Data.Value), &val) - if err != nil { - //fmt.Println("Get() op Value [", Value, "] json decode fail") - return nil, err - } - return val, nil - } - //没找到则继续向后查找 - if bucket.Next != nil { - bucket = bucket.Next - continue - } - } - return nil, nil +// 哈希表 Get数据 +func (h *XHashTable) Get(Key string) (interface{}, error) { + var k string + var val interface{} + bucketIdx := getBucketSlot(Key) + // bucketSize := unsafe.Sizeof(&XBucket{}) + bucket := h.Table[bucketIdx] + for bucket != nil { + k = bucket.Data.Key + // 如果查找到相同Key开始返回数据 + if strings.Compare(strings.ToLower(Key), strings.ToLower(k)) == 0 { + // bTmp := []byte(k) + err := json.Unmarshal([]byte(bucket.Data.Value), &val) + if err != nil { + // fmt.Println("Get() op Value [", Value, "] json decode fail") + return nil, err + } + return val, nil + } + // 没找到则继续向后查找 + if bucket.Next != nil { + bucket = bucket.Next + continue + } + } + return nil, nil } -//Delete数据 -func (h *XHashTable) Remove(Key string)(error) { - var k string - bucketIdx := getBucketSlot(Key) - bucket := h.Table[bucketIdx] - - //如果节点不存在直接返回 - if bucket == nil { - return nil - } - - //进行节点判断处理 - tmpBucketPre := bucket - linkDepthSize := 0 //存储当前开拉链第几层 - for bucket != nil { - linkDepthSize = linkDepthSize + 1 - tmpBucketPre = bucket //把当前节点保存下来 - k = bucket.Data.Key - //如果查找了相同的Key进行删除操作 - if strings.Compare(strings.ToLower(Key), strings.ToLower(k)) == 0 { - //如果是深度第一层的拉链，把下一层拉链替换顶层拉链后返回 - if linkDepthSize == 1 { - //如果是终点了直接当前桶置为nil - if bucket.Next == nil { - h.Table[bucketIdx] = nil - } else { //如果还有其他下一级开拉链元素则替代本级元素 - h.Table[bucketIdx] = bucket.Next - } - } else { //如果查到的可以不是第一级元素，则进行前后替换 - tmpBucketPre.Next = bucket.Next - } - //释放内存 - //p := bucket.Data.Key - //h.mm.Free(p) - h.mm.FreeString(bucket.Data.Key) - h.mm.FreeString(bucket.Data.Value) - h.mm.Free(uintptr(unsafe.Pointer(bucket.Data))) - //h.mm.Free(bucket) - h.Size = h.Size - 1 - return nil - } - //如果还没找到，继续遍历把下一节点升级为当前节点 - if bucket.Next != nil { - bucket = bucket.Next - continue - } - } - return nil +// Delete数据 +func (h *XHashTable) Remove(Key string) (error) { + var k string + bucketIdx := getBucketSlot(Key) + bucket := h.Table[bucketIdx] + + // 如果节点不存在直接返回 + if bucket == nil { + return nil + } + + // 进行节点判断处理 + tmpBucketPre := bucket + linkDepthSize := 0 // 存储当前开拉链第几层 + for bucket != nil { + linkDepthSize = linkDepthSize + 1 + tmpBucketPre = bucket // 把当前节点保存下来 + k = bucket.Data.Key + // 如果查找了相同的Key进行删除操作 + if strings.Compare(strings.ToLower(Key), strings.ToLower(k)) == 0 { + // 如果是深度第一层的拉链，把下一层拉链替换顶层拉链后返回 + if linkDepthSize == 1 { + // 如果是终点了直接当前桶置为nil + if bucket.Next == nil { + h.Table[bucketIdx] = nil + } else { // 如果还有其他下一级开拉链元素则替代本级元素 + h.Table[bucketIdx] = bucket.Next + } + } else { // 如果查到的可以不是第一级元素，则进行前后替换 + tmpBucketPre.Next = bucket.Next + } + // 释放内存 + // p := bucket.Data.Key + // h.mm.Free(p) + h.mm.FreeString(bucket.Data.Key) + h.mm.FreeString(bucket.Data.Value) + h.mm.Free(uintptr(unsafe.Pointer(bucket.Data))) + // h.mm.Free(bucket) + h.Size = h.Size - 1 + return nil + } + // 如果还没找到，继续遍历把下一节点升级为当前节点 + if bucket.Next != nil { + bucket = bucket.Next + continue + } + } + return nil } -//获取目前总元素数量 +// 获取目前总元素数量 func (h *XHashTable) getSize() (uint64) { - return h.Size + return h.Size } -//获取槽的计算位置 +// 获取槽的计算位置 func getBucketSlot(key string) uint64 { - hash := BKDRHash(key) - return hash % HashTableBucketMax + hash := BKDRHash(key) + return hash % HashTableBucketMax } -//哈希函数（采用冲突率低性能高的 BKDR Hash算法，也可采用 MurmurHash） +// 哈希函数（采用冲突率低性能高的 BKDR Hash算法，也可采用 MurmurHash） func BKDRHash(key string) uint64 { - var str []byte = []byte(key) // string transfer format to []byte - seed := uint64(131) // 31 131 1313 13131 131313 etc.. - hash := uint64(0) - for i := 0; i < len(str); i++ { - hash = (hash * seed) + uint64(str[i]) - } - return hash ^ (hash>>16)&0x7FFFFFFF + var str []byte = []byte(key) // string transfer format to []byte + seed := uint64(131) // 31 131 1313 13131 131313 etc.. + hash := uint64(0) + for i := 0; i < len(str); i++ { + hash = (hash * seed) + uint64(str[i]) + } + return hash ^ (hash>>16)&0x7FFFFFFF } - - -//主函数 +// 主函数 func main() { - - //初始化XMM对象 - f := &xmm.Factory{} - //从操作系统申请一个内存块，如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值） - mm, err := f.CreateMemory(0.6) - if err != nil { - panic("CreateMemory fail ") - } - - fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 03 - HashTable ======\n") - - - //初始化哈希表 - h := &XHashTable{} - h.Init(mm) - - //简单数据类型压入哈希表 - fmt.Println("\n---- Simple data type hash Set/Get -----") - - //压入一批数据 - for i := 0; i < 5; i++ { - fmt.Println("Hash Set: ", strconv.Itoa(i), strconv.Itoa(i*10)) - h.Set(strconv.Itoa(i), strconv.Itoa(i*10)) - } - //读取数据 - for i := 0; i < 5; i++ { - fmt.Print("Hash Get: ", i, " ") - fmt.Println(h.Get(strconv.Itoa(i))) - } - fmt.Println("Hash Table Size: ", h.getSize()) - - - - //存取复合型数据结构到哈希表 - fmt.Println("\n---- Mixed data type hash Set/Get -----") - - //构造测试数据 - testKV := make(map[string]interface{}) - testKV["map01"] = map[string]string{"name":"heiyeluren", "email":"heiyeluren@example.com"} - testKV["array01"] = [...]uint{9527, 2022, 8} - testKV["slice01"] = make([]int, 3, 5) - //fmt.Println(testKV) - - //压入数据到哈希表 - for k, v := range testKV { - fmt.Print("Hash Set: ", k, " \n") - h.Set(k, v) - } - //读取哈希表 - for k, _ := range testKV { - fmt.Print("Hash Get: ", k, " ") - fmt.Println(h.Get(k)) - } - fmt.Println("Hash Table Size: ", h.getSize()) - - //覆盖同样Key数据 - fmt.Println("\n---- Overwrite data hash Set/Get -----") - for k, _ := range testKV { - fmt.Print("Cover Hash Set: ", k, " \n") - h.Set(k, "Overwrite data") - } - for k, _ := range testKV { - fmt.Print("Cover Hash Get: ", k, " ") - fmt.Println(h.Get(k)) - } - fmt.Println("Hash Table Size: ", h.getSize()) - - //删除Key - fmt.Println("\n---- Delete data Remove op -----") - - k1 := "test01" - v1 := "value01" - fmt.Println("Hash Set: ", k1, " ", v1, " ", h.Set(k1, v1)) - fmt.Print("Hash Get: ", k1, " ") - fmt.Println(h.Get(k1)) - - fmt.Println("Hash Table Size: ", h.getSize()) - - fmt.Print("Remove Key: ", k1) - fmt.Println(h.Remove(k1)) - fmt.Print("Hash Get: ", k1, " ") - fmt.Println(h.Get(k1)) - - //读取老的key看看有没有受影响 - for k, _ := range testKV { - fmt.Print("Hash Get: ", k, " ") - fmt.Println(h.Get(k)) - } - fmt.Println("Hash Table Size: ", h.getSize()) - - - //释放所有哈希表 - h.Destroy() - - //结束 - fmt.Println("\n===== Example test success ======\n") + + // 初始化XMM对象 + f := &xmm.Factory{} + // 从操作系统申请一个内存块，如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值） + mm, err := f.CreateMemory(0.6) + if err != nil { + panic("CreateMemory fail ") + } + + fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 03 - HashTable ======\n") + + // 初始化哈希表 + h := &XHashTable{} + h.Init(mm) + + // 简单数据类型压入哈希表 + fmt.Println("\n---- Simple data type hash Set/Get -----") + + // 压入一批数据 + for i := 0; i < 5; i++ { + fmt.Println("Hash Set: ", strconv.Itoa(i), strconv.Itoa(i*10)) + h.Set(strconv.Itoa(i), strconv.Itoa(i*10)) + } + // 读取数据 + for i := 0; i < 5; i++ { + fmt.Print("Hash Get: ", i, " ") + fmt.Println(h.Get(strconv.Itoa(i))) + } + fmt.Println("Hash Table Size: ", h.getSize()) + + // 存取复合型数据结构到哈希表 + fmt.Println("\n---- Mixed data type hash Set/Get -----") + + // 构造测试数据 + testKV := make(map[string]interface{}) + testKV["map01"] = map[string]string{"name": "heiyeluren", "email": "heiyeluren@example.com"} + testKV["array01"] = [...]uint{9527, 2022, 8} + testKV["slice01"] = make([]int, 3, 5) + // fmt.Println(testKV) + + // 压入数据到哈希表 + for k, v := range testKV { + fmt.Print("Hash Set: ", k, " \n") + h.Set(k, v) + } + // 读取哈希表 + for k, _ := range testKV { + fmt.Print("Hash Get: ", k, " ") + fmt.Println(h.Get(k)) + } + fmt.Println("Hash Table Size: ", h.getSize()) + + // 覆盖同样Key数据 + fmt.Println("\n---- Overwrite data hash Set/Get -----") + for k, _ := range testKV { + fmt.Print("Cover Hash Set: ", k, " \n") + h.Set(k, "Overwrite data") + } + for k, _ := range testKV { + fmt.Print("Cover Hash Get: ", k, " ") + fmt.Println(h.Get(k)) + } + fmt.Println("Hash Table Size: ", h.getSize()) + + // 删除Key + fmt.Println("\n---- Delete data Remove op -----") + + k1 := "test01" + v1 := "value01" + fmt.Println("Hash Set: ", k1, " ", v1, " ", h.Set(k1, v1)) + fmt.Print("Hash Get: ", k1, " ") + fmt.Println(h.Get(k1)) + + fmt.Println("Hash Table Size: ", h.getSize()) + + fmt.Print("Remove Key: ", k1) + fmt.Println(h.Remove(k1)) + fmt.Print("Hash Get: ", k1, " ") + fmt.Println(h.Get(k1)) + + // 读取老的key看看有没有受影响 + for k, _ := range testKV { + fmt.Print("Hash Get: ", k, " ") + fmt.Println(h.Get(k)) + } + fmt.Println("Hash Table Size: ", h.getSize()) + + // 释放所有哈希表 + h.Destroy() + + // 结束 + fmt.Println("\n===== Example test success ======\n") } @@ -700,17 +683,14 @@ func main() { ``` -


- ## XMM问题反馈 XMM 目前是早期版本，总体性能比较好，目前也在另外一个自研的XMap模块中使用，当然也少不了一些问题和bug，欢迎大家一起共创，或者直接提交PR等等。 欢迎加入XMM技术交流微信群，要加群，可以先添加如下微信让对方拉入群： - ![image](https://raw.githubusercontent.com/heiyeluren/docs/master/imgs/koala_wx.png) diff --git a/entry.go b/entry.go index 193f570..d430105 100644 --- a/entry.go +++ b/entry.go @@ -15,6 +15,7 @@ language governing permissions and limitations under the License. */ +// Package xmm // Package redblacktree provides a pure Golang implementation // of a red-black tree as described by Thomas H. Cormen's et al. // in their seminal Algorithms book (3rd ed). This data structure @@ -43,10 +44,9 @@ type NodeEntry struct { right *NodeEntry left *NodeEntry parent *NodeEntry - color Color //比二叉查找树要多出一个颜色属性 + color Color // 比二叉查找树要多出一个颜色属性 } - type encodedNodeEntry struct { TotalLen uint64 KeyLen uint64 @@ -57,9 +57,9 @@ type encodedNodeEntry struct { right *NodeEntry left *NodeEntry parent *NodeEntry - color Color //比二叉查找树要多出一个颜色属性 + color Color // 比二叉查找树要多出一个颜色属性 - //追加string key、value 要不然就内存泄露了 + // 追加string key、value 要不然就内存泄露了 } @@ -81,6 +81,7 @@ func (n *NodeEntry) encode() []byte { }*/ +// BytesAscSort is a helper function for sorting a slice of byte slices var BytesAscSort Comparator = func(o1, o2 interface{}) int { key1, key2 := o1.([]byte), o2.([]byte) return bytes.Compare(key1, key2) @@ -102,6 +103,7 @@ func (c Color) String() string { } } +// String returns string func (d Direction) String() string { switch d { case LEFT: @@ -160,15 +162,16 @@ type Visitor interface { Visit(*NodeEntry) } -// A redblack tree is `Visitable` by a `Visitor`. +// Visitable A redblack tree is `Visitable` by a `Visitor`. type Visitable interface { Walk(Visitor) } -// Keys must be comparable. It's mandatory to provide a Comparator, +// Comparator Keys must be comparable. It's mandatory to provide a Comparator, // which returns zero if o1 == o2, -1 if o1 < o2, 1 if o1 > o2 type Comparator func(o1, o2 interface{}) int +// IntComparator inits a Comparator for ints // Default comparator expects keys to be of type `int`. // Warning: if either one of `o1` or `o2` cannot be asserted to `int`, it panics. func IntComparator(o1, o2 interface{}) int { @@ -184,7 +187,7 @@ func IntComparator(o1, o2 interface{}) int { } } -// Keys of type `string`. +// StringComparator Keys of type `string`. // Warning: if either one of `o1` or `o2` cannot be asserted to `string`, it panics. func StringComparator(o1, o2 interface{}) int { s1 := o1.(string) @@ -244,7 +247,7 @@ func (t *Tree) SetComparator(c Comparator) { // Return value in 1st position indicates whether any payload was found. func (t *Tree) Get(key []byte) (bool, []byte) { if err := mustBeValidKey(key); err != nil { - //logger.Printf("Get was prematurely aborted: %s\n", err.Error()) + // logger.Printf("Get was prematurely aborted: %s\n", err.Error()) return false, nil } ok, node := t.getNode(key) @@ -292,7 +295,7 @@ func (t *Tree) getMinimum(x *NodeEntry) *NodeEntry { // GetParent looks for the node with supplied key and returns the parent node. func (t *Tree) GetParent(key interface{}) (found bool, parent *NodeEntry, dir Direction) { if err := mustBeValidKey(key); err != nil { - //logger.Printf("GetParent was prematurely aborted: %s\n", err.Error()) + // logger.Printf("GetParent was prematurely aborted: %s\n", err.Error()) return false, nil, NODIR } @@ -319,17 +322,17 @@ func (t *Tree) internalLookup(parent *NodeEntry, this *NodeEntry, key interface{ } } -// Reverses actions of RotateLeft +// RotateRight Reverses actions of RotateLeft func (t *Tree) RotateRight(y *NodeEntry) { if y == nil { - //logger.Printf("RotateRight: nil arg cannot be rotated. Noop\n") + // logger.Printf("RotateRight: nil arg cannot be rotated. Noop\n") return } if y.left == nil { - //logger.Printf("RotateRight: y has nil left subtree. Noop\n") + // logger.Printf("RotateRight: y has nil left subtree. Noop\n") return } - //logger.Printf("\t\t\trotate right of %s\n", y) + // logger.Printf("\t\t\trotate right of %s\n", y) x := y.left y.left = x.right if x.right != nil { @@ -349,17 +352,17 @@ func (t *Tree) RotateRight(y *NodeEntry) { y.parent = x } -// Side-effect: red-black tree properties is maintained. +// RotateLeft Side-effect: red-black tree properties is maintained. func (t *Tree) RotateLeft(x *NodeEntry) { if x == nil { - //logger.Printf("RotateLeft: nil arg cannot be rotated. Noop\n") + // logger.Printf("RotateLeft: nil arg cannot be rotated. Noop\n") return } if x.right == nil { - //logger.Printf("RotateLeft: x has nil right subtree. Noop\n") + // logger.Printf("RotateLeft: x has nil right subtree. Noop\n") return } - //logger.Printf("\t\t\trotate left of %s\n", x) + // logger.Printf("\t\t\trotate left of %s\n", x) y := x.right x.right = y.left @@ -387,7 +390,7 @@ func (t *Tree) Put(node *NodeEntry) error { node.parent, node.left, node.Next, node.right = nil, nil, nil, nil key, data := node.Key, node.Value if err := mustBeValidKey(key); err != nil { - //logger.Printf("Put was prematurely aborted: %s\n", err.Error()) + // logger.Printf("Put was prematurely aborted: %s\n", err.Error()) return err } t.lock.Lock() @@ -395,17 +398,17 @@ func (t *Tree) Put(node *NodeEntry) error { if t.root == nil { node.color = BLACK t.root = node - //logger.Printf("Added %s as root node\n", t.root.String()) + // logger.Printf("Added %s as root node\n", t.root.String()) return nil } found, parent, dir := t.internalLookup(nil, t.root, key, NODIR) if found { if parent == nil { - //logger.Printf("Put: parent=nil & found. Overwrite ROOT node\n") + // logger.Printf("Put: parent=nil & found. Overwrite ROOT node\n") t.root.Value = data } else { - //logger.Printf("Put: parent!=nil & found. Overwriting\n") + // logger.Printf("Put: parent!=nil & found. Overwriting\n") switch dir { case LEFT: parent.left.Value = data @@ -424,7 +427,7 @@ func (t *Tree) Put(node *NodeEntry) error { case RIGHT: parent.right = newNode } - //logger.Printf("Added %s to %s node of parent %s\n", newNode.String(), dir, parent.String()) + // logger.Printf("Added %s to %s node of parent %s\n", newNode.String(), dir, parent.String()) t.fixupPut(newNode) } } @@ -450,10 +453,10 @@ func isRed(n *NodeEntry) bool { // // @param z - the newly added Node to the tree. func (t *Tree) fixupPut(z *NodeEntry) { - //logger.Printf("\tfixup new node z %s\n", z.String()) + // logger.Printf("\tfixup new node z %s\n", z.String()) loop: for { - //logger.Printf("\tcurrent z %s\n", z.String()) + // logger.Printf("\tcurrent z %s\n", z.String()) switch { case z.parent == nil: fallthrough @@ -461,18 +464,18 @@ loop: fallthrough default: // When the loop terminates, it does so because p[z] is black. - //logger.Printf("\t\t=> bye\n") + // logger.Printf("\t\t=> bye\n") break loop case z.parent.color == RED: grandparent := z.parent.parent - //logger.Printf("\t\tgrandparent is nil %t addr:%d\n", grandparent == nil, unsafe.Pointer(t)) + // logger.Printf("\t\tgrandparent is nil %t addr:%d\n", grandparent == nil, unsafe.Pointer(t)) if z.parent == grandparent.left { - //logger.Printf("\t\t%s is the left child of %s\n", z.parent, grandparent) + // logger.Printf("\t\t%s is the left child of %s\n", z.parent, grandparent) y := grandparent.right - ////logger.Printf("\t\ty (right) %s\n", y) + // //logger.Printf("\t\ty (right) %s\n", y) if isRed(y) { // case 1 - y is RED - //logger.Printf("\t\t(*) case 1\n") + // logger.Printf("\t\t(*) case 1\n") z.parent.color = BLACK y.color = BLACK grandparent.color = RED @@ -481,40 +484,40 @@ loop: } else { if z == z.parent.right { // case 2 - //logger.Printf("\t\t(*) case 2\n") + // logger.Printf("\t\t(*) case 2\n") z = z.parent t.RotateLeft(z) } // case 3 - //logger.Printf("\t\t(*) case 3\n") + // logger.Printf("\t\t(*) case 3\n") z.parent.color = BLACK grandparent.color = RED t.RotateRight(grandparent) } } else { - //logger.Printf("\t\t%s is the right child of %s\n", z.parent, grandparent) + // logger.Printf("\t\t%s is the right child of %s\n", z.parent, grandparent) y := grandparent.left - //logger.Printf("\t\ty (left) %s\n", y) + // logger.Printf("\t\ty (left) %s\n", y) if isRed(y) { // case 1 - y is RED - //logger.Printf("\t\t..(*) case 1\n") + // logger.Printf("\t\t..(*) case 1\n") z.parent.color = BLACK y.color = BLACK grandparent.color = RED z = grandparent } else { - //logger.Printf("\t\t## %s\n", z.parent.left) + // logger.Printf("\t\t## %s\n", z.parent.left) if z == z.parent.left { // case 2 - //logger.Printf("\t\t..(*) case 2\n") + // logger.Printf("\t\t..(*) case 2\n") z = z.parent t.RotateRight(z) } // case 3 - //logger.Printf("\t\t..(*) case 3\n") + // logger.Printf("\t\t..(*) case 3\n") z.parent.color = BLACK grandparent.color = RED t.RotateLeft(grandparent) @@ -535,7 +538,7 @@ func (t *Tree) Size() uint64 { // Has checks for existence of a item identified by supplied key. func (t *Tree) Has(key interface{}) bool { if err := mustBeValidKey(key); err != nil { - //logger.Printf("Has was prematurely aborted: %s\n", err.Error()) + // logger.Printf("Has was prematurely aborted: %s\n", err.Error()) return false } found, _, _ := t.internalLookup(nil, t.root, key, NODIR) @@ -559,7 +562,7 @@ func (t *Tree) transplant(u *NodeEntry, v *NodeEntry) { // Delete is a noop if the supplied key doesn't exist. func (t *Tree) Delete(key []byte) *NodeEntry { if !t.Has(key) { - //logger.Printf("Delete: bail as no node exists for key %d\n", key) + // logger.Printf("Delete: bail as no node exists for key %d\n", key) return nil } _, z := t.getNode(key) @@ -569,26 +572,26 @@ func (t *Tree) Delete(key []byte) *NodeEntry { if z.left == nil { // one child (RIGHT) - //logger.Printf("\t\tDelete: case (a)\n") + // logger.Printf("\t\tDelete: case (a)\n") x = z.right - //logger.Printf("\t\t\t--- x is right of z") + // logger.Printf("\t\t\t--- x is right of z") t.transplant(z, z.right) } else if z.right == nil { // one child (LEFT) - //logger.Printf("\t\tDelete: case (b)\n") + // logger.Printf("\t\tDelete: case (b)\n") x = z.left - //logger.Printf("\t\t\t--- x is left of z") + // logger.Printf("\t\t\t--- x is left of z") t.transplant(z, z.left) } else { // two children - //logger.Printf("\t\tDelete: case (c) & (d)\n") + // logger.Printf("\t\tDelete: case (c) & (d)\n") y = t.getMinimum(z.right) - //logger.Printf("\t\t\tminimum of z.right is %s (color=%s)\n", y, y.color) + // logger.Printf("\t\t\tminimum of z.right is %s (color=%s)\n", y, y.color) yOriginalColor = y.color x = y.right - //logger.Printf("\t\t\t--- x is right of minimum") + // logger.Printf("\t\t\t--- x is right of minimum") if y.parent == z { if x != nil { @@ -611,7 +614,7 @@ func (t *Tree) Delete(key []byte) *NodeEntry { } func (t *Tree) fixupDelete(x *NodeEntry) { - //logger.Printf("\t\t\tfixupDelete of node %s\n", x) + // logger.Printf("\t\t\tfixupDelete of node %s\n", x) if x == nil { return } @@ -619,17 +622,17 @@ loop: for { switch { case x == t.root: - //logger.Printf("\t\t\t=> bye .. is root\n") + // logger.Printf("\t\t\t=> bye .. is root\n") break loop case x.color == RED: - //logger.Printf("\t\t\t=> bye .. RED\n") + // logger.Printf("\t\t\t=> bye .. RED\n") break loop case x == x.parent.right: - //logger.Printf("\t\tBRANCH: x is right child of parent\n") + // logger.Printf("\t\tBRANCH: x is right child of parent\n") w := x.parent.left // is nillable if isRed(w) { // Convert case 1 into case 2, 3, or 4 - //logger.Printf("\t\t\tR> case 1\n") + // logger.Printf("\t\t\tR> case 1\n") w.color = BLACK x.parent.color = RED t.RotateRight(x.parent) @@ -639,13 +642,13 @@ loop: switch { case !isRed(w.left) && !isRed(w.right): // case 2 - both children of w are BLACK - //logger.Printf("\t\t\tR> case 2\n") + // logger.Printf("\t\t\tR> case 2\n") w.color = RED x = x.parent // recurse up tree case isRed(w.right) && !isRed(w.left): // case 3 - right child RED & left child BLACK // convert to case 4 - //logger.Printf("\t\t\tR> case 3\n") + // logger.Printf("\t\t\tR> case 3\n") w.right.color = BLACK w.color = RED t.RotateLeft(w) @@ -653,7 +656,7 @@ loop: } if isRed(w.left) { // case 4 - left child is RED - //logger.Printf("\t\t\tR> case 4\n") + // logger.Printf("\t\t\tR> case 4\n") w.color = x.parent.color x.parent.color = BLACK w.left.color = BLACK @@ -662,11 +665,11 @@ loop: } } case x == x.parent.left: - //logger.Printf("\t\tBRANCH: x is left child of parent\n") + // logger.Printf("\t\tBRANCH: x is left child of parent\n") w := x.parent.right // is nillable if isRed(w) { // Convert case 1 into case 2, 3, or 4 - //logger.Printf("\t\t\tL> case 1\n") + // logger.Printf("\t\t\tL> case 1\n") w.color = BLACK x.parent.color = RED t.RotateLeft(x.parent) @@ -676,13 +679,13 @@ loop: switch { case !isRed(w.left) && !isRed(w.right): // case 2 - both children of w are BLACK - //logger.Printf("\t\t\tL> case 2\n") + // logger.Printf("\t\t\tL> case 2\n") w.color = RED x = x.parent // recurse up tree case isRed(w.left) && !isRed(w.right): // case 3 - left child RED & right child BLACK // convert to case 4 - //logger.Printf("\t\t\tL> case 3\n") + // logger.Printf("\t\t\tL> case 3\n") w.left.color = BLACK w.color = RED t.RotateRight(w) @@ -690,7 +693,7 @@ loop: } if isRed(w.right) { // case 4 - right child is RED - //logger.Printf("\t\t\tL> case 4\n") + // logger.Printf("\t\t\tL> case 4\n") w.color = x.parent.color x.parent.color = BLACK w.right.color = BLACK @@ -758,7 +761,7 @@ func (v *InorderVisitor) Visit(node *NodeEntry) { v.buffer.Write([]byte("(")) v.Visit(node.left) v.buffer.Write([]byte(fmt.Sprintf("%s", node.Key))) // @TODO - //v.buffer.Write([]byte(fmt.Sprintf("%d{%s}", node.Key, v.trim(node.color.String())))) + // v.buffer.Write([]byte(fmt.Sprintf("%d{%s}", node.Key, v.trim(node.color.String())))) v.Visit(node.right) v.buffer.Write([]byte(")")) } @@ -790,19 +793,19 @@ func mustBeValidKey(key interface{}) error { } /*keyValue := reflect.ValueOf(key) - switch keyValue.Kind() { - case reflect.Chan: - fallthrough - case reflect.Func: - fallthrough - case reflect.Interface: - fallthrough - case reflect.Map: - fallthrough - case reflect.Ptr: - return ErrorKeyDisallowed - default: - return nil - }*/ + switch keyValue.Kind() { + case reflect.Chan: + fallthrough + case reflect.Func: + fallthrough + case reflect.Interface: + fallthrough + case reflect.Map: + fallthrough + case reflect.Ptr: + return ErrorKeyDisallowed + default: + return nil + }*/ return nil } diff --git a/example/xmm-test00.go b/example/xmm-test00.go index 11ec646..2975045 100644 --- a/example/xmm-test00.go +++ b/example/xmm-test00.go @@ -1,41 +1,41 @@ /* - XMM Example 00 + XMM Example 00 - Goal: How to use XMM quickly and easily - Description: An example of how to use the XMM memory library quickly and easily. - - XMM 示例00 + Goal: How to use XMM quickly and easily + Description: An example of how to use the XMM memory library quickly and easily. - 目标：如何简单快速使用XMM - 说明：就是示例如何快速简单使用XMM内存库 + XMM 示例00 + + 目标：如何简单快速使用XMM + 说明：就是示例如何快速简单使用XMM内存库 */ package main import ( - //xmm "xmm/src" - xmm "github.com/heiyeluren/xmm" "fmt" "unsafe" + + "github.com/heiyeluren/xmm" ) func main() { - //Initialising XMM objects - //初始化XMM对象 + // Initialising XMM objects + // 初始化XMM对象 f := &xmm.Factory{} // Request a memory block from the operating system - //If memory usage reaches 60%, asynchronous automatic expansion is performed, each time 256MB of memory is expanded asynchronously (fixed value), the value of 0.6 can be configured independently - //从操作系统申请一个内存块 - //如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值），0.6这个数值可以自主配置 + // If memory usage reaches 60%, asynchronous automatic expansion is performed, each time 256MB of memory is expanded asynchronously (fixed value), the value of 0.6 can be configured independently + // 从操作系统申请一个内存块 + // 如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值），0.6这个数值可以自主配置 mm, err := f.CreateMemory(0.6) if err != nil { panic("CreateMemory fail ") } // manipulate int types, request memory and assign a value - //操作int类型，申请内存后赋值 + // 操作int类型，申请内存后赋值 var tmpNum int = 9527 p, err := mm.Alloc(unsafe.Sizeof(tmpNum)) if err != nil { @@ -43,28 +43,27 @@ func main() { } Id := (*int)(p) // Write the set number assignment to the XMM memory - //把设定好的数字赋值写入到XMM内存中 + // 把设定好的数字赋值写入到XMM内存中 *Id = tmpNum // To manipulate string types, XMM provides the From() interface to get a pointer directly and the string will be stored in XMM - //操作字符串类型，XMM提供了From()接口，可以直接获得一个指针，字符串会存储在XMM中 + // 操作字符串类型，XMM提供了From()接口，可以直接获得一个指针，字符串会存储在XMM中 Name, err := mm.From("heiyeluren") if err != nil { - panic("Alloc fail ") + panic("Alloc fail ") } // Output variables and memory addresses from XMM memory - //从XMM内存中输出变量和内存地址 + // 从XMM内存中输出变量和内存地址 fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 00 ======\n") fmt.Println("\n-- Memory data status --\n") - fmt.Println(" Id :", *Id, "\t\t( Id ptr addr: ", Id, ")" ) + fmt.Println(" Id :", *Id, "\t\t( Id ptr addr: ", Id, ")") fmt.Println(" Name:", Name, "\t( Name ptr addr: ", &Name, ")") fmt.Println("\n===== Example test success ======\n") - //Free the Id,Name memory block - //释放Id,Name内存块 + // Free the Id,Name memory block + // 释放Id,Name内存块 mm.Free(uintptr(p)) mm.FreeString(Name) } - diff --git a/example/xmm-test01.go b/example/xmm-test01.go index a4f3a4f..5e48a27 100644 --- a/example/xmm-test01.go +++ b/example/xmm-test01.go @@ -4,22 +4,22 @@ 目标：如何在结构体中使用XMM Objective: How to use XMM in structures - - 说明：示例如何结构体类场景如何使用XMM内存库 - Description: Example of how to use the XMM memory library in a structure class scenario + + 说明：示例如何结构体类场景如何使用XMM内存库 + Description: Example of how to use the XMM memory library in a structure class scenario */ package main import ( - //xmm "xmm/src" - xmm "github.com/heiyeluren/xmm" "fmt" "unsafe" + + "github.com/heiyeluren/xmm" ) func main() { - //定义一个类型(结构体) + // 定义一个类型(结构体) // Define a type (structure) type User struct { Id uint @@ -29,21 +29,21 @@ func main() { Salary float32 } - //初始化XMM对象 + // 初始化XMM对象 // Initialise the XMM object f := &xmm.Factory{} - - //从操作系统申请一个内存块 - //如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值），0.6这个数值可以自主配置 + + // 从操作系统申请一个内存块 + // 如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值），0.6这个数值可以自主配置 // Request a memory block from the operating system - //If memory usage reaches 60%, asynchronous automatic expansion is performed, each time 256MB of memory is expanded asynchronously (fixed value), the value of 0.6 can be configured independently - + // If memory usage reaches 60%, asynchronous automatic expansion is performed, each time 256MB of memory is expanded asynchronously (fixed value), the value of 0.6 can be configured independently + mm, err := f.CreateMemory(0.6) if err != nil { panic("CreateMemory fail ") } - //自己从内存块中申请一小段自己想用的内存 + // 自己从内存块中申请一小段自己想用的内存 // Request a small section of memory from the memory block yourself that you want to use size := unsafe.Sizeof(User{}) p, err := mm.Alloc(size) @@ -51,39 +51,38 @@ func main() { panic("Alloc fail ") } - //使用该内存块，进行结构体元素赋值 + // 使用该内存块，进行结构体元素赋值 // Use this memory block for structure element assignment user := (*User)(p) - user.Id = 1 - user.Age = 18 - user.Name = "heiyeluren" - user.Email = "heiyeluren@example.com" + user.Id = 1 + user.Age = 18 + user.Name = "heiyeluren" + user.Email = "heiyeluren@example.com" - //输出变量，打印整个结构体等 + // 输出变量，打印整个结构体等 // Output variables, print entire structures, etc. fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 01 ======\n") fmt.Println("\n-- Memory data status --\n") fmt.Println("User ptr addr: \t", p) - fmt.Println("User data: \t", user) + fmt.Println("User data: \t", user) - //释放内存块（实际是做mark标记操作） - //Free memory blocks (actually doing a mark mark operation) + // 释放内存块（实际是做mark标记操作） + // Free memory blocks (actually doing a mark mark operation) mm.Free(uintptr(p)) - //Free()后再看看变量值，只是针对这个内存块进行mark标记动作，并未彻底从内存中释放（XMM设计机制，降低实际gc回收空闲时间） - //XMM内部会有触发gc的机制，主要是内存容量，参数TotalGCFactor=0.0004，目前如果要配置，需要自己修改这个常量，一般不用管它，Free()操作中有万分之4的概率会命中触发gc~ - //GC触发策略：待释放内存 > 总内存 * 万分之4 会触发gc动作 - //After Free() and then look at the variable value, only for this memory block to mark mark action, not completely released from memory (XMM design mechanism to reduce the actual gc recovery idle time) - //XMM will have an internal mechanism to trigger gc, mainly memory capacity, parameter TotalGCFactor=0.0004, currently if you want to configure, you need to modify this constant yourself, generally do not bother with it, Free() operation has a 4 in 10,000 probability of hitting the trigger gc ~ - //GC trigger policy: memory to be freed > total memory * 4 in 10,000 will trigger gc action - + // Free()后再看看变量值，只是针对这个内存块进行mark标记动作，并未彻底从内存中释放（XMM设计机制，降低实际gc回收空闲时间） + // XMM内部会有触发gc的机制，主要是内存容量，参数TotalGCFactor=0.0004，目前如果要配置，需要自己修改这个常量，一般不用管它，Free()操作中有万分之4的概率会命中触发gc~ + // GC触发策略：待释放内存 > 总内存 * 万分之4 会触发gc动作 + // After Free() and then look at the variable value, only for this memory block to mark mark action, not completely released from memory (XMM design mechanism to reduce the actual gc recovery idle time) + // XMM will have an internal mechanism to trigger gc, mainly memory capacity, parameter TotalGCFactor=0.0004, currently if you want to configure, you need to modify this constant yourself, generally do not bother with it, Free() operation has a 4 in 10,000 probability of hitting the trigger gc ~ + // GC trigger policy: memory to be freed > total memory * 4 in 10,000 will trigger gc action + fmt.Println("\n-- Memory data status after XMM.Free() --\n") fmt.Println("memory ptr addr:\t", p) - fmt.Println("User data:\t\t", user) + fmt.Println("User data:\t\t", user) - //结束 + // 结束 // End fmt.Println("\n===== Example test success ======\n") } - diff --git a/example/xmm-test02.go b/example/xmm-test02.go index dde2181..91dabeb 100644 --- a/example/xmm-test02.go +++ b/example/xmm-test02.go @@ -4,49 +4,49 @@ 目标：使用XMM构建一个单链表程序 Objective: To build a single linked table program using XMM - + 说明：示例复杂场景中使用XMM内存库 Description: Example complex scenario using the XMM memory library */ package main import ( - //xmm "xmm/src" - xmm "github.com/heiyeluren/xmm" "fmt" "unsafe" + + "github.com/heiyeluren/xmm" ) -//定义一个链表的节点结构 -// Define the node structure of a chain table +// XListNode Define the node structure of a chain table +// 定义一个链表的节点结构 type XListNode struct { - Val int + Val int Next *XListNode } -//单链表主结构 -// Single linked table main structure +// XList Single linked table main structure +// 单链表主结构 type XList struct { Head *XListNode - //Tail *XListNode + // Tail *XListNode } - -//初始化链表 -//Initialize the chain table -func (l *XList) Init( mm xmm.XMemory ) { - Node,err := mm.Alloc(unsafe.Sizeof(XListNode{})) +// Init Initialize the chain table +// 初始化链表 +// Initialize the chain table +func (l *XList) Init(mm xmm.XMemory) { + Node, err := mm.Alloc(unsafe.Sizeof(XListNode{})) if err != nil { panic("Alloc fail") } - //head := &XListNode{Val:list[0]} + // head := &XListNode{Val:list[0]} head := (*XListNode)(Node) l.Head = head fmt.Println("Init Xlist done") } -//链表增加节点 -//Link table adds nodes +// Append Link table adds nodes +// 链表增加节点 func (l *XList) Append(i int, mm xmm.XMemory) { h := l.Head for h.Next != nil { @@ -64,34 +64,34 @@ func (l *XList) Append(i int, mm xmm.XMemory) { fmt.Println("Append item:", Node.Val) } -//遍历所有链表节点并打印 -// iterate through all the chain table nodes and print +// Show iterate through all the chain table nodes and print +// 遍历所有链表节点并打印 func (l *XList) Show() { h := l.Head - //fmt.Println(h.Val) + // fmt.Println(h.Val) for h.Next != nil { h = h.Next fmt.Println("Show item:", h.Val) } } -//释放整个链表结构 -//Releasing the entire linked table structure +// Destroy Releasing the entire linked table structure +// 释放整个链表结构 func (l *XList) Destroy(mm xmm.XMemory) { cnt := 0 h := l.Head - //fmt.Println(h.Val) + // fmt.Println(h.Val) - //统计需要释放总数 + // 统计需要释放总数 // Count the total number of releases required for h.Next != nil { h = h.Next cnt++ - //fmt.Println(h.Val) + // fmt.Println(h.Val) } - //fmt.Println("item count:", cnt) + // fmt.Println("item count:", cnt) - //循环释放所有内存 + // 循环释放所有内存 // Loop to free all memory for i := 0; i <= cnt; i++ { h := l.Head @@ -106,35 +106,34 @@ func (l *XList) Destroy(mm xmm.XMemory) { } } -//主函数 +// 主函数 // Main functions func main() { - //初始化XMM对象 + // 初始化XMM对象 // Initialise the XMM object f := &xmm.Factory{} - - //从操作系统申请一个内存块，如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值） + + // 从操作系统申请一个内存块，如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值） // Request a block of memory from the OS and perform asynchronous auto-expansion if memory usage reaches 60%, 256MB of memory per asynchronous expansion (fixed value) mm, err := f.CreateMemory(0.6) if err != nil { panic("CreateMemory fail ") } - //要生成的数字列表 + // 要生成的数字列表 // List of numbers to be generated - list := []int{ 2, 4, 3} + list := []int{2, 4, 3} fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 02 - LinkedList ======\n") - - //初始化链表 - //Initialize LinkedList + // 初始化链表 + // Initialize LinkedList l := &XList{} l.Init(mm) fmt.Println("") - //把元素压入链表 + // 把元素压入链表 // Pressing elements into a LinkedList for i := 0; i < len(list); i++ { l.Append(list[i], mm) @@ -145,12 +144,10 @@ func main() { l.Show() fmt.Println("") - //Free all LinkedList memory + // Free all LinkedList memory l.Destroy(mm) - //结束 - //End + // 结束 + // End fmt.Println("\n===== Example test success ======\n") } - - diff --git a/example/xmm-test03.go b/example/xmm-test03.go index b607b56..dab202f 100644 --- a/example/xmm-test03.go +++ b/example/xmm-test03.go @@ -1,70 +1,69 @@ - /* XMM 简单示例 - 03 目标：使用XMM构建一个哈希表程序 说明：示例复杂场景中使用XMM内存库 - + XMM Simple Example - 03 Objective: To build a hash table program using XMM - Description: Example complex scenario using the XMM memory library + Description: Example complex scenario using the XMM memory library */ package main import ( - //xmm "xmm/src" - xmm "github.com/heiyeluren/xmm" + "encoding/json" + "fmt" "strconv" "strings" - "fmt" "unsafe" - "encoding/json" + + // xmm "xmm/src" + "github.com/heiyeluren/xmm" ) -//定义哈希表桶数量 -// Define the number of hash table buckets +// HashTableBucketMax Define the number of hash table buckets +// 定义哈希表桶数量 const HashTableBucketMax = 1024 -//定义哈希表存储实际KV节点存储数据 -// Define hash tables to store actual KV node storage data +// XEntity Define hash tables to store actual KV node storage data +// 定义哈希表存储实际KV节点存储数据 type XEntity struct { - Key string //Key必须是字符串 Key, must be a string - Value string //Value是字符串，从interface转成json格式 Value value, a string, convert interface{} to json + Key string // Key必须是字符串 Key, must be a string + Value string // Value是字符串，从interface转成json格式 Value value, a string, convert interface{} to json } -//定义哈希表中单个桶结构（开拉链法） -// Define a single bucket structure in a hash table (open zip method) +// XBucket Define a single bucket structure in a hash table (open zip method) +// 定义哈希表中单个桶结构（开拉链法） type XBucket struct { - Data *XEntity //当前元素KV Current element KV - Next *XBucket //如果冲突情况开拉链法下一个节点的Next指针 Next pointer to the next node of the open zip method if a conflict situation arises + Data *XEntity // 当前元素KV Current element KV + Next *XBucket // 如果冲突情况开拉链法下一个节点的Next指针 Next pointer to the next node of the open zip method if a conflict situation arises } -//哈希表入口主结构HashTable -// HashTable entry main structure HashTable +// XHashTable HashTable entry the main structure HashTable +// 哈希表入口主结构HashTable type XHashTable struct { - Table []*XBucket //哈希表所有桶存储池 Hash table all bucket storage pool - Size uint64 //哈希表已存总元素数量 Total number of elements stored in the hash table - mm xmm.XMemory //XMM内存管理对象 XMM Memory Management Objects + Table []*XBucket // 哈希表所有桶存储池 Hash table all bucket storage pool + Size uint64 // 哈希表已存总元素数量 Total number of elements stored in the hash table + mm xmm.XMemory // XMM内存管理对象 XMM Memory Management Objects } - -//初始化哈希表 -// Initialize the hash table -func (h *XHashTable) Init( mm xmm.XMemory ) { - //设置需要申请多大的连续数组内存空间，如果是动态扩容情况建议设置为16，目前是按照常量桶大小 +// Init Initialize the hash table +// 初始化哈希表 +func (h *XHashTable) Init(mm xmm.XMemory) { + // 设置需要申请多大的连续数组内存空间，如果是动态扩容情况建议设置为16，目前是按照常量桶大小 // Set how much contiguous array memory space to request, 16 is recommended for dynamic expansion cases, currently it is based on the constant bucket size cap := HashTableBucketMax initCap := uintptr(cap) - - //申请按照设定总桶数量大小的内存块 + + // 申请按照设定总桶数量大小的内存块 // Request a block of memory of the set total bucket size p, err := mm.AllocSlice(unsafe.Sizeof(&XBucket{}), initCap, initCap) if err != nil { panic("Alloc fail") } - - //把申请的内存库给哈希表总存储池，初始化数量和XMM内存对象 + + // 把申请的内存库给哈希表总存储池，初始化数量和XMM内存对象 // give the requested memory bank to the total hash table storage pool, initialize the number and XMM memory objects h.Table = *(*[]*XBucket)(p) h.Size = 0 @@ -73,23 +72,23 @@ func (h *XHashTable) Init( mm xmm.XMemory ) { fmt.Println("Init XHashTable done") } -//释放整个哈希表 -//Free the entire hash table -func (h *XHashTable) Destroy() (error) { +// Destroy Free the entire hash table +// 释放整个哈希表 +func (h *XHashTable) Destroy() error { return nil } -//哈希表 Set数据 -///hash tables Set data -func (h *XHashTable) Set(Key string, Value interface{}) (error) { +// Set hash tables Set data +// 哈希表 Set数据 +func (h *XHashTable) Set(Key string, Value interface{}) error { - //-------------- - // 构造Entity + // -------------- + // 构造Entity // Constructing Entity - //-------------- - - //Value进行序列化 - //Value for serialisation + // -------------- + + // Value进行序列化 + // Value for serialisation jdata, err := json.Marshal(Value) if err != nil { fmt.Println("Set() op Value [", Value, "] json encode fail") @@ -97,9 +96,9 @@ func (h *XHashTable) Set(Key string, Value interface{}) (error) { } sValue := string(jdata) - //申请三个内存，操作字符串类型，XMM提供了From()接口，可以直接获得一个指针，字符串会存储在XMM中 + // 申请三个内存，操作字符串类型，XMM提供了From()接口，可以直接获得一个指针，字符串会存储在XMM中 // request three memory, manipulate string types, XMM provides a From() interface to get a pointer directly, the string will be stored in XMM - //size := unsafe.Sizeof(User{}) + // size := unsafe.Sizeof(User{}) p, err := h.mm.Alloc(unsafe.Sizeof(XEntity{})) if err != nil { panic("Alloc fail ") @@ -113,21 +112,21 @@ func (h *XHashTable) Set(Key string, Value interface{}) (error) { panic("Alloc fail ") } - //拼装Entity + // 拼装Entity // Assembling Entity pEntity := (*XEntity)(p) pEntity.Key = pKey pEntity.Value = pVal - - //--------------- + + // --------------- // 挂接到Bucket // Hook up to Bucket - //--------------- + // --------------- bucketIdx := getBucketSlot(Key) - //bucketSize := unsafe.Sizeof(&XBucket{}) + // bucketSize := unsafe.Sizeof(&XBucket{}) bucket := h.Table[bucketIdx] - //构造bucket + // 构造bucket // Constructing a bucket pb, err := h.mm.Alloc(unsafe.Sizeof(XBucket{})) if err != nil { @@ -137,7 +136,7 @@ func (h *XHashTable) Set(Key string, Value interface{}) (error) { pBucket.Data = pEntity pBucket.Next = nil - //如果槽没有被占用则压入数据后结束 + // 如果槽没有被占用则压入数据后结束 // end after pressing in data if the slot is not occupied if bucket == nil { h.Table[bucketIdx] = pBucket @@ -145,15 +144,15 @@ func (h *XHashTable) Set(Key string, Value interface{}) (error) { return nil } - //使用开拉链法把KV放入到冲突的槽中 + // 使用开拉链法把KV放入到冲突的槽中 // Use the unzipping method to place the KV in the conflicting slot var k string for bucket != nil { k = bucket.Data.Key - //如果发现有重名key，则直接替换Value + // 如果发现有重名key，则直接替换Value // If a duplicate key is found, the Value is replaced directly if strings.Compare(strings.ToLower(Key), strings.ToLower(k)) == 0 { - //挂接新Value + // 挂接新Value // mount the new Value pValNew, err := h.mm.From(sValue) if err != nil { @@ -162,42 +161,42 @@ func (h *XHashTable) Set(Key string, Value interface{}) (error) { bucket.Data.Value = pValNew return nil } - //如果是最后一个拉链的节点，则把当前KV挂上去 + // 如果是最后一个拉链的节点，则把当前KV挂上去 // If it is the last node to zip, hook up the current KV if bucket.Next == nil { bucket.Next = pBucket h.Size = h.Size + 1 return nil } - //没找到则继续循环 + // 没找到则继续循环 // continue the cycle if not found bucket = bucket.Next } return nil } -//哈希表 Get数据 -// Hash Table Get Data -func (h *XHashTable) Get(Key string) (interface{}, error) { +// Get Hash Table Get Data +// 哈希表 Get数据 +func (h *XHashTable) Get(Key string) (interface{}, error) { var k string var val interface{} bucketIdx := getBucketSlot(Key) - //bucketSize := unsafe.Sizeof(&XBucket{}) + // bucketSize := unsafe.Sizeof(&XBucket{}) bucket := h.Table[bucketIdx] for bucket != nil { k = bucket.Data.Key - //如果查找到相同Key开始返回数据 + // 如果查找到相同Key开始返回数据 // If the same Key is found start returning data if strings.Compare(strings.ToLower(Key), strings.ToLower(k)) == 0 { - //bTmp := []byte(k) + // bTmp := []byte(k) err := json.Unmarshal([]byte(bucket.Data.Value), &val) if err != nil { - //fmt.Println("Get() op Value [", Value, "] json decode fail") + // fmt.Println("Get() op Value [", Value, "] json decode fail") return nil, err } return val, nil } - //没找到则继续向后查找 + // 没找到则继续向后查找 // continue to search backwards if not found if bucket.Next != nil { bucket = bucket.Next @@ -207,57 +206,57 @@ func (h *XHashTable) Get(Key string) (interface{}, error) { return nil, nil } -//哈希表 Delete数据 -// Hash Table Delete Data -func (h *XHashTable) Remove(Key string)(error) { +// Remove Hash Table Delete Data +// 哈希表 Delete数据 +func (h *XHashTable) Remove(Key string) error { var k string bucketIdx := getBucketSlot(Key) bucket := h.Table[bucketIdx] - //如果节点不存在直接返回 + // 如果节点不存在直接返回 // return directly if node does not exist if bucket == nil { return nil } - //进行节点判断处理 + // 进行节点判断处理 // Perform node judgement processing tmpBucketPre := bucket - linkDepthSize := 0 //存储当前开拉链第几层 Store the current layer of open zips + linkDepthSize := 0 // 存储当前开拉链第几层 Store the current layer of open zips for bucket != nil { linkDepthSize = linkDepthSize + 1 - tmpBucketPre = bucket //把当前节点保存下来 Save the current node + tmpBucketPre = bucket // 把当前节点保存下来 Save the current node k = bucket.Data.Key - - //如果查找了相同的Key进行删除操作 + + // 如果查找了相同的Key进行删除操作 // If the same Key is found for the delete operation if strings.Compare(strings.ToLower(Key), strings.ToLower(k)) == 0 { - - //如果是深度第一层的拉链，把下一层拉链替换顶层拉链后返回 + + // 如果是深度第一层的拉链，把下一层拉链替换顶层拉链后返回 // In the case of a deep first layer of zips, replace the next layer of zips with the top layer and return if linkDepthSize == 1 { - //如果是终点了直接当前桶置为nil - //If it's the end of the line set the current bucket to nil + // 如果是终点了直接当前桶置为nil + // If it's the end of the line set the current bucket to nil if bucket.Next == nil { h.Table[bucketIdx] = nil - } else { //如果还有其他下一级开拉链元素则替代本级元素 If there are other open zip elements at the next level, they replace the element at this level + } else { // 如果还有其他下一级开拉链元素则替代本级元素 If there are other open zip elements at the next level, they replace the element at this level h.Table[bucketIdx] = bucket.Next } - } else { //如果查到的可以不是第一级元素，则进行前后替换 If the element found can be other than the first level, then a back and forth substitution is made + } else { // 如果查到的可以不是第一级元素，则进行前后替换 If the element found can be other than the first level, then a back and forth substitution is made tmpBucketPre.Next = bucket.Next } - //释放内存 - //Free memory - //p := bucket.Data.Key - //h.mm.Free(p) + // 释放内存 + // Free memory + // p := bucket.Data.Key + // h.mm.Free(p) h.mm.FreeString(bucket.Data.Key) h.mm.FreeString(bucket.Data.Value) h.mm.Free(uintptr(unsafe.Pointer(bucket.Data))) - //h.mm.Free(bucket) + // h.mm.Free(bucket) h.Size = h.Size - 1 return nil } - //如果还没找到，继续遍历把下一节点升级为当前节点 + // 如果还没找到，继续遍历把下一节点升级为当前节点 // If not found, continue traversing to upgrade the next node to the current one if bucket.Next != nil { bucket = bucket.Next @@ -267,24 +266,24 @@ func (h *XHashTable) Remove(Key string)(error) { return nil } -//获取哈希表元素总数量 +// 获取哈希表元素总数量 // Get the total number of hash table elements -func (h *XHashTable) getSize() (uint64) { +func (h *XHashTable) getSize() uint64 { return h.Size } -//获取指定Key在哈希表中槽的位置 +// 获取指定Key在哈希表中槽的位置 // Get the position of the specified Key in the hash table slot func getBucketSlot(key string) uint64 { hash := BKDRHash(key) return hash % HashTableBucketMax } -//哈希函数（采用冲突率低性能高的 BKDR Hash算法，也可采用 MurmurHash） -// Hash function (using BKDR Hash algorithm with low conflict rate and high performance, MurmurHash can also be used) +// BKDRHash Hash function (using BKDR Hash algorithm with low conflict rate and high performance, MurmurHash can also be used) +// 哈希函数（采用冲突率低性能高的 BKDR Hash算法，也可采用 MurmurHash） func BKDRHash(key string) uint64 { - var str []byte = []byte(key) // string transfer format to []byte - seed := uint64(131) // 31 131 1313 13131 131313 etc.. + var str []byte = []byte(key) // string transfer format to []byte + seed := uint64(131) // 31 131 1313 13131 131313 etc.. hash := uint64(0) for i := 0; i < len(str); i++ { hash = (hash * seed) + uint64(str[i]) @@ -292,17 +291,15 @@ func BKDRHash(key string) uint64 { return hash ^ (hash>>16)&0x7FFFFFFF } - - -//主函数 +// 主函数 // Main functions func main() { - //初始化XMM对象 + // 初始化XMM对象 // Initialise the XMM object f := &xmm.Factory{} - - //从操作系统申请一个内存块，如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值） + + // 从操作系统申请一个内存块，如果内存使用达到60%，就进行异步自动扩容，每次异步扩容256MB内存（固定值） // Request a block of memory from the OS and perform asynchronous auto-expansion if memory usage reaches 60%, 256MB of memory per asynchronous expansion (fixed value) mm, err := f.CreateMemory(0.6) if err != nil { @@ -311,23 +308,22 @@ func main() { fmt.Println("\n===== XMM X(eXtensible) Memory Manager example 03 - HashTable ======\n") - - //初始化哈希表 + // 初始化哈希表 // Initialize the hash table h := &XHashTable{} h.Init(mm) - //简单数据类型压入哈希表 + // 简单数据类型压入哈希表 // Simple data types pressed into a hash table fmt.Println("\n---- Simple data type hash Set/Get -----") - //压入一批数据 + // 压入一批数据 // Press in a batch of data for i := 0; i < 5; i++ { fmt.Println("Hash Set: ", strconv.Itoa(i), strconv.Itoa(i*10)) h.Set(strconv.Itoa(i), strconv.Itoa(i*10)) } - //读取数据 + // 读取数据 // Read data for i := 0; i < 5; i++ { fmt.Print("Hash Get: ", i, " ") @@ -335,27 +331,25 @@ func main() { } fmt.Println("Hash Table Size: ", h.getSize()) - - - //存取复合型数据结构到哈希表 + // 存取复合型数据结构到哈希表 // Accessing compound data structures to hash tables fmt.Println("\n---- Mixed data type hash Set/Get -----") - //构造测试数据 + // 构造测试数据 // Constructing test data testKV := make(map[string]interface{}) - testKV["map01"] = map[string]string{"name":"heiyeluren", "email":"heiyeluren@example.com"} + testKV["map01"] = map[string]string{"name": "heiyeluren", "email": "heiyeluren@example.com"} testKV["array01"] = [...]uint{9527, 2022, 8} testKV["slice01"] = make([]int, 3, 5) - //fmt.Println(testKV) + // fmt.Println(testKV) - //压入数据到哈希表 + // 压入数据到哈希表 // Pressing data into a hash table for k, v := range testKV { fmt.Print("Hash Set: ", k, " \n") h.Set(k, v) } - //读取哈希表 + // 读取哈希表 // Read the hash table for k, _ := range testKV { fmt.Print("Hash Get: ", k, " ") @@ -363,7 +357,7 @@ func main() { } fmt.Println("Hash Table Size: ", h.getSize()) - //覆盖同样Key数据 + // 覆盖同样Key数据 // Overwrite the same Key data fmt.Println("\n---- Overwrite data hash Set/Get -----") for k, _ := range testKV { @@ -376,7 +370,7 @@ func main() { } fmt.Println("Hash Table Size: ", h.getSize()) - //删除Key + // 删除Key // Delete Key fmt.Println("\n---- Delete data Remove op -----") @@ -393,7 +387,7 @@ func main() { fmt.Print("Hash Get: ", k1, " ") fmt.Println(h.Get(k1)) - //读取老的key看看有没有受影响 + // 读取老的key看看有没有受影响 // Read the old key to see if it is affected for k, _ := range testKV { fmt.Print("Hash Get: ", k, " ") @@ -401,14 +395,11 @@ func main() { } fmt.Println("Hash Table Size: ", h.getSize()) - - //释放所有哈希表 + // 释放所有哈希表 // Free all hash tables h.Destroy() - //结束 - //End + // 结束 + // End fmt.Println("\n===== Example test success ======\n") } - - diff --git a/go.mod b/go.mod index 3534f2e..7124c84 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,5 @@ module github.com/heiyeluren/xmm -// go 1.12 +go 1.16 -require ( - github.com/spf13/cast v1.4.1 // indirect -) +require github.com/spf13/cast v1.4.1 diff --git a/go.sum b/go.sum index 8f6d9b8..74bf81f 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,8 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/heap.go b/heap.go index d84c08b..1925087 100644 --- a/heap.go +++ b/heap.go @@ -36,8 +36,8 @@ type xHeap struct { rawLinearMemoryAlloc linearAlloc - //默认RawMemoryL1Bits=0,退化为以为数据。RawMemoryL2Bits = 20个 - addrMap [1 << RawMemoryL1Bits]*[1 << RawMemoryL2Bits]*xRawLinearMemory //addr -> page -> rawMemory 关系 + // 默认RawMemoryL1Bits=0,退化为以为数据。RawMemoryL2Bits = 20个 + addrMap [1 << RawMemoryL1Bits]*[1 << RawMemoryL2Bits]*xRawLinearMemory // addr -> page -> rawMemory 关系 allChunk []*xChunk @@ -110,7 +110,7 @@ func (xh *xHeap) addFreeCapacity(size int64) { func (xh *xHeap) addChunks(xChunks []*xChunk) error { if xh.allChunk == nil || len(xChunks)+len(xh.allChunk) > cap(xh.allChunk) { cap := cap(xh.allChunk) << 1 - //log.Printf("xHeap.addChunks grow cap:%d\n", cap) + // log.Printf("xHeap.addChunks grow cap:%d\n", cap) err := xh.allChunkAllocator.grow(uintptr(cap), func(newPtr unsafe.Pointer) error { sl := reflect.SliceHeader{ Data: uintptr(newPtr), @@ -256,13 +256,13 @@ func (xh *xHeap) freeChunk(pageNum uintptr) (ptr *xChunk, err error) { return &xChunk{startAddr: startAddr, npages: pageNum}, nil } -//todo 释放：地址中保存len、保存空闲地址、要么直接复用，要么合并page再复用 +// todo 释放：地址中保存len、保存空闲地址、要么直接复用，要么合并page再复用 func (xh *xHeap) free(addr uintptr) error { - //todo 标记完成，接下来触发清理 - //panic("todo 释放：地址中保存len、保存空闲地址、要么直接复用，要么合并page再复用,存放到树状结构") - //todo 还给 span，比较高效。 - //key:开始地址 value:结束地址 存放到红黑树中。 - //key找key最相近的，找到判断value。有则更新，没有则插入。 + // todo 标记完成，接下来触发清理 + // panic("todo 释放：地址中保存len、保存空闲地址、要么直接复用，要么合并page再复用,存放到树状结构") + // todo 还给 span，比较高效。 + // key:开始地址 value:结束地址 存放到红黑树中。 + // key找key最相近的，找到判断value。有则更新，没有则插入。 if err := xh.mark(addr); err != nil { return err } @@ -281,7 +281,7 @@ func (xh *xHeap) needSweep() bool { } sweepCtl := atomic.LoadInt32(&xh.sweepCtl) if sweepCtl < 0 { - //扩容线程+1 + // 扩容线程+1 if atomic.CompareAndSwapInt32(&xh.sweepCtl, sweepCtl, sweepCtl+1) { return true } @@ -299,7 +299,7 @@ var logg bool // todo classSpan中并发支持 func (xh *xHeap) sweep() { - //统计判断 + // 统计判断 if !xh.needSweep() { return } @@ -310,7 +310,7 @@ func (xh *xHeap) sweep() { continue } classSpan := xh.classSpan[sweepIndex] - //todo 环循环 + // todo 环循环 for span := classSpan.full.first; span != nil; span = span.next { if logg { fmt.Println("sweep", sweepIndex, uintptr(unsafe.Pointer(span))) @@ -329,7 +329,7 @@ func (xh *xHeap) sweep() { if total < 1 { return } - //logg = true + // logg = true xh.addFreeCapacity(0 - int64(total)) for { sweepCtl := atomic.LoadInt32(&xh.sweepCtl) @@ -340,7 +340,7 @@ func (xh *xHeap) sweep() { if sweepCtl != sweepCtlStatus { return } - //sweep执行结束 + // sweep执行结束 atomic.StoreInt32(&xh.sweepCtl, 0) atomic.StoreUint32(&xh.sweepIndex, 0) return @@ -355,12 +355,12 @@ func (xh *xHeap) ChunkInsert(chunk *xChunk) error { // 清理span（span级别锁） func (xh *xHeap) sweepFullSpan(span *xSpan) (sweep bool, size uint, err error) { - //fmt.Println("======================") + // fmt.Println("======================") if span.classIndex > 0 { // 所有还给classspan return xh.classSpan[span.classIndex].freeSpan(span) } else if span.classIndex == 0 && span.nelems > 0 { - //大对象释放 + // 大对象释放 span.lock.Lock() defer span.lock.Unlock() if span.nelems != span.countGcMarkBits() { @@ -424,12 +424,12 @@ func (xh *xHeap) grow2(pageNum uintptr) error { } offset := uintptr(p) for i := 0; i < int(Align(size, heapRawMemoryBytes)/heapRawMemoryBytes); i++ { - //页地址到RawMemory保存 + // 页地址到RawMemory保存 rawLinearMemoryPtr, err := xh.rawLinearMemoryAllocator.alloc() if err != nil { return err } - //addrMap 初始化xRawLinearMemory + // addrMap 初始化xRawLinearMemory rlm := (*xRawLinearMemory)(rawLinearMemoryPtr) index := RawMemoryIndex(offset) if addrs := xh.addrMap[index.l1()]; addrs == nil { @@ -469,7 +469,7 @@ func (xh *xHeap) grow() error { if err := xh.addChunks([]*xChunk{chunk}); err != nil { return err } - //页地址到RawMemory保存 + // 页地址到RawMemory保存 rawLinearMemoryPtr, err := xh.rawLinearMemoryAllocator.alloc() if err != nil { return err @@ -489,6 +489,7 @@ func (xh *xHeap) grow() error { return nil } +// RawMemoryIndex . func RawMemoryIndex(p uintptr) RawMemoryIdx { return RawMemoryIdx((p + RawMemoryBaseOffset) / heapRawMemoryBytes) } @@ -499,6 +500,7 @@ func RawMemoryBase(i RawMemoryIdx) uintptr { return uintptr(i)*heapRawMemoryBytes - RawMemoryBaseOffset } +// RawMemoryIdx . type RawMemoryIdx uint func (i RawMemoryIdx) l1() uint { @@ -506,15 +508,13 @@ func (i RawMemoryIdx) l1() uint { // Let the compiler optimize this away if there's no // L1 map. return 0 - } else { - return uint(i) >> RawMemoryL1Shift } + return uint(i) >> RawMemoryL1Shift } func (i RawMemoryIdx) l2() uint { if RawMemoryL1Bits == 0 { return uint(i) - } else { - return uint(i) & (1< 0; retry--*/ { oldVal := atomic.LoadUintptr(&xa.nchunk) @@ -118,7 +118,7 @@ func (xa *xFixedAllocator) casNchunk() (old uintptr) { } func (xa *xFixedAllocator) casChunk() (old uintptr) { - size := xa.size //不变的size + size := xa.size // 不变的size var swapped bool for /*retry := 3; retry > 0; retry--*/ { oldVal := atomic.LoadUintptr(&xa.chunk) diff --git a/raw_linear_memory.go b/raw_linear_memory.go index 1c451fb..823882f 100644 --- a/raw_linear_memory.go +++ b/raw_linear_memory.go @@ -19,7 +19,7 @@ package xmm type xRawLinearMemory struct { - //bitmap [heapArenaBitmapBytes]byte + // bitmap [heapArenaBitmapBytes]byte spans [pagesPerRawMemory]*xSpan pageInUse [pagesPerRawMemory / 8]uint8 pageMarks [pagesPerRawMemory / 8]uint8 diff --git a/raw_memory.go b/raw_memory.go index 2b5d8b9..f9ac5cf 100644 --- a/raw_memory.go +++ b/raw_memory.go @@ -26,7 +26,7 @@ import ( "unsafe" ) -//用来管理mmap申请的内存，用于实际存放地址的元数据 +// 用来管理mmap申请的内存，用于实际存放地址的元数据 type xRawMemory struct { addr uintptr down bool @@ -40,7 +40,7 @@ func newXRawMemory(byteNum int) (*xRawMemory, error) { if err != nil { return nil, err } - //log.Printf(" newXRawMemory(mmap) byteNum:%d byte \n", byteNum) + // log.Printf(" newXRawMemory(mmap) byteNum:%d byte \n", byteNum) ptr := unsafe.Pointer(&mem[0]) xrm := &xRawMemory{addr: uintptr(ptr), mem: mem} return xrm, nil @@ -64,9 +64,9 @@ func init() { // type xRawMemoryPool struct { - //空闲free + // 空闲free frees []*block - //申请内存，头插法 + // 申请内存，头插法 xrm *xRawMemory index uintptr lock sync.RWMutex @@ -82,7 +82,7 @@ func (xrmp *xRawMemoryPool) alloc(byteSize uintptr) (ptr unsafe.Pointer, err err return nil, err } if offset == 0 { - //扩容 + // 扩容 xrm, err := newXRawMemory(metadataRawMemoryBytes) if err != nil { return nil, err @@ -114,7 +114,7 @@ func (xrmp *xRawMemoryPool) grow() error { return nil } -//通过cas获取空闲的offset +// 通过cas获取空闲的offset func (xrmp *xRawMemoryPool) freeOffset(size uintptr) (uintptr, error) { if size > metadataRawMemoryBytes { return 0, errors.New("size is over") diff --git a/raw_memory_test.go b/raw_memory_test.go index 98a1128..93a37a5 100644 --- a/raw_memory_test.go +++ b/raw_memory_test.go @@ -49,18 +49,18 @@ func TestSet(t *testing.T) { func Test_Syscall6(t *testing.T) { /*arenaSizes := []uintptr{ - 512 << 20, - 256 << 20, - 128 << 20, - } - for _, arenaSize := range arenaSizes { - a, size := sysReserveAligned(unsafe.Pointer(p), arenaSize, heapArenaBytes) - if a != nil { - mheap_.arena.init(uintptr(a), size) - p = uintptr(a) + size // For hint below - break - } - }*/ + 512 << 20, + 256 << 20, + 128 << 20, + } + for _, arenaSize := range arenaSizes { + a, size := sysReserveAligned(unsafe.Pointer(p), arenaSize, heapArenaBytes) + if a != nil { + mheap_.arena.init(uintptr(a), size) + p = uintptr(a) + size // For hint below + break + } + }*/ size := int(round(23232, 4096)) addr := uintptr(sysReserve(size)) // 必须预留的多才可以 arena linearAlloc学习这个 for i := 0; i < 8; i++ { @@ -74,7 +74,7 @@ func Test_Syscall6(t *testing.T) { if err != nil { panic(err) } - //fmt.Println(ret, err) + // fmt.Println(ret, err) user := (*User)(unsafe.Pointer(ret)) user.Age = 11 user.Name = 13 @@ -102,9 +102,9 @@ func Test_LineAlloc(t *testing.T) { user := (*User)(unsafe.Pointer(ret)) user.Age = 11 user.Name = 13 - //addr += uintptr(length) + // addr += uintptr(length) ret += userSize - //fmt.Printf("user:%+v addr:%d\n", (*User)(unsafe.Pointer(ret)), ret) + // fmt.Printf("user:%+v addr:%d\n", (*User)(unsafe.Pointer(ret)), ret) } } @@ -133,9 +133,9 @@ func Test_MMapAlloc(t *testing.T) { user := (*User)(unsafe.Pointer(ret)) user.Age = 11 user.Name = 13 - //addr += uintptr(length) + // addr += uintptr(length) ret += userSize - //fmt.Printf("user:%+v addr:%d\n", (*User)(unsafe.Pointer(ret)), ret) + // fmt.Printf("user:%+v addr:%d\n", (*User)(unsafe.Pointer(ret)), ret) } } diff --git a/span.go b/span.go index a0f4583..dc94c93 100644 --- a/span.go +++ b/span.go @@ -29,22 +29,22 @@ type xSpan struct { lock sync.Mutex classIndex uint // class的索引 classSize uintptr // classSpan的长度 - startAddr uintptr //bit索引 + startAddr uintptr // bit索引 npages uintptr freeIndex uintptr extensionPoint uintptr // 扩容负载因子 - //span中可以分配的个数 + // span中可以分配的个数 nelems uintptr - //bitmap 每个bit标识地址是否被分配(用来分配计算空闲地址) - //1111111111111111111111111111111111111111111111111111111100000000 从右往左，遇到第一个1开始可以分配。 + // bitmap 每个bit标识地址是否被分配(用来分配计算空闲地址) + // 1111111111111111111111111111111111111111111111111111111100000000 从右往左，遇到第一个1开始可以分配。 allocCache uint64 - //allocCache 保存的指针 + // allocCache 保存的指针 allocBits *gcBits - //gc标记的bitmap（1为不需要的） + // gc标记的bitmap（1为不需要的） gcmarkBits *gcBits // gc @@ -56,7 +56,7 @@ type xSpan struct { allocCount uintptr next *xSpan - //pre *xSpan + // pre *xSpan heap *xHeap } @@ -117,7 +117,7 @@ func (s *xSpan) isFull() bool { return (val+1)*s.classSize >= _PageSize*s.npages } -//freeIndex增加： +// freeIndex增加： // 1、首先使用自旋CAS获取空闲索引 // 2、CAS失败，则锁加重为排他锁 // 3、必须内存对齐 @@ -163,7 +163,7 @@ func (s *xSpan) markBitsForIndex(objIndex uintptr) markBits { func (s *xSpan) setMarkBitsForIndex(objIndex uintptr) { uint32p, mask := s.gcmarkBits.bitp(objIndex) - //fmt.Printf("xSpan:%d size:%d\n", uintptr(unsafe.Pointer(s)), s.classSize) + // fmt.Printf("xSpan:%d size:%d\n", uintptr(unsafe.Pointer(s)), s.classSize) s.heap.addFreeCapacity(int64(s.classSize)) markBits{uint32p, mask, objIndex}.setMarked() } @@ -172,7 +172,7 @@ func (s *xSpan) markBitsForBase() markBits { return markBits{(*uint32)(s.gcmarkBits), uint32(1), 0} } -//当前的allocCache中的最后一个bit空位和free位置。 +// 当前的allocCache中的最后一个bit空位和free位置。 func (s *xSpan) nextFreeFast() uintptr { if s.freeIndex >= s.nelems { return 0 @@ -210,7 +210,7 @@ func (s *xSpan) nextFree() (v uintptr, has bool) { return } -//当前allocCache没该内容 +// 当前allocCache没该内容 func (s *xSpan) nextFreeIndex() (uintptr, bool) { sfreeindex := s.freeIndex snelems := s.nelems @@ -261,7 +261,7 @@ func (s *xSpan) refillAllocCache(whichByte uintptr) { s.allocCache = aCache } -//得到GC标记数目 +// 得到GC标记数目 func (s *xSpan) countGcMarkBits() uintptr { count := uintptr(0) maxIndex := s.nelems / 32 diff --git a/span_pool.go b/span_pool.go index eceabf9..b0f4426 100644 --- a/span_pool.go +++ b/span_pool.go @@ -41,11 +41,11 @@ type xSpanPool struct { inuse [_NumSizeClasses]uint64 debug bool spanGen [_NumSizeClasses]int32 // 小于0 正在扩容 - spans [_NumSizeClasses]*[]*xSpan //预分配,spans很短，不存在引用超长，第一个为当前正在使用的，第二个为预先分配的span + spans [_NumSizeClasses]*[]*xSpan // 预分配,spans很短，不存在引用超长，第一个为当前正在使用的，第二个为预先分配的span heap *xHeap spanFact float32 specialPageNumCoefficient [_NumSizeClasses]uint8 - //1750 + 950 + // 1750 + 950 classSpan [_NumSizeClasses]*xClassSpan } @@ -106,8 +106,8 @@ func (sp *xSpanPool) AllocSlice(eleSize uintptr, cap, len uintptr) (p unsafe.Poi var is bool -//通过增加key、value长度使得分配到不同span -//todo 擦除数据 +// 通过增加key、value长度使得分配到不同span +// todo 擦除数据 func (sp *xSpanPool) Alloc(size uintptr) (p unsafe.Pointer, err error) { if size > _MaxSmallSize { pageNum := Align(size, _PageSize) / _PageSize @@ -156,14 +156,14 @@ func (sp *xSpanPool) Alloc(size uintptr) (p unsafe.Pointer, err error) { return unsafe.Pointer(ptr), nil } if idex > 0 && has { - //idx前已经使用完了,删除前面满了的。 + // idx前已经使用完了,删除前面满了的。 sp.addInuse(sizeclass) sp.growSpan(sizeclass, RemoveHead, spanGen) return unsafe.Pointer(ptr), nil } // 同步扩容 if !has { - //所有的span都没有空位，需要阻塞同步分配 产生新的class span，释放所有的span + // 所有的span都没有空位，需要阻塞同步分配 产生新的class span，释放所有的span if err := sp.growSpan(sizeclass, ExpendSync, spanGen); err != nil { return nil, err } @@ -323,6 +323,6 @@ func newXConcurrentHashMapSpanPool(heap *xHeap, spanFact float32, pageNumCoeffic return sp, nil } -//启动利用class_to_allocnpages 预先分配span。 -//alloc超过阈值，异步预分配 -//alloc没有空闲时候，同步分配（防止分配太多）。 +// 启动利用class_to_allocnpages 预先分配span。 +// alloc超过阈值，异步预分配 +// alloc没有空闲时候，同步分配（防止分配太多）。 diff --git a/span_pool_test.go b/span_pool_test.go index ed4f1e3..8913931 100644 --- a/span_pool_test.go +++ b/span_pool_test.go @@ -78,7 +78,7 @@ func TestSpanLock(t *testing.T) { } wait.Wait() fmt.Printf("%+v \nfreeIndex:%d %d %d \n", cs, cs.freeIndex, allcCnt*4, allcCnt) - //todo 问题,好像自己实现的锁有问题，应该是CAS 和 悲观锁冲突了。只能选一个。cas和锁必须互斥 + // todo 问题,好像自己实现的锁有问题，应该是CAS 和 悲观锁冲突了。只能选一个。cas和锁必须互斥 // 锁里面有CAS，是否考虑用锁呢？ <-time.After(time.Second) if val := atomic.LoadUintptr(&cs.freeIndex); val != allcCnt*4 { diff --git a/string_alloc.go b/string_alloc.go index 33a16f2..6afa752 100644 --- a/string_alloc.go +++ b/string_alloc.go @@ -51,7 +51,7 @@ func (sa *xStringAllocator) alloc(size uintptr) (sh *reflect.StringHeader, err e return sh, nil } -//todo 这里有bug，StringHeader没有计算进去，应该使用[]byte +// todo 这里有bug，StringHeader没有计算进去，应该使用[]byte func (sa *xStringAllocator) FromInAddr(addr uintptr, contents ...string) (p []*string, err error) { offset := addr p = make([]*string, len(contents)) @@ -69,7 +69,7 @@ func (sa *xStringAllocator) FromInAddr(addr uintptr, contents ...string) (p []*s source := reflect.SliceHeader{Data: sourcePtr.Data, Len: sourcePtr.Len, Cap: sourcePtr.Len} srcByte := *(*[]byte)(unsafe.Pointer(&source)) if len := copy(dstBytes, srcByte); len != sh.Len { - return nil, fmt.Errorf("copy length is err: len = %d\n", len) + return nil, fmt.Errorf("copy length is err: len = %d", len) } p[i] = (*string)(unsafe.Pointer(sh)) } @@ -89,7 +89,7 @@ func (sa *xStringAllocator) From2(item1 string, item2 string) (newItem1 string, dst := reflect.SliceHeader{Data: uintptr(dataPtr), Len: rawSize, Cap: rawSize} dstBytes := *(*[]byte)(unsafe.Pointer(&dst)) if len := copy(dstBytes, raw); len != rawSize { - return "", "", fmt.Errorf("copy length is err: len = %d\n", len) + return "", "", fmt.Errorf("copy length is err: len = %d", len) } dst.Len = item1Size str1 := *(*string)(unsafe.Pointer(&dst)) @@ -109,14 +109,12 @@ func (sa *xStringAllocator) From(content string) (p string, err error) { dst := reflect.SliceHeader{Data: uintptr(dataPtr), Len: length, Cap: length} dstBytes := *(*[]byte)(unsafe.Pointer(&dst)) if len := copy(dstBytes, content); len != length { - return "", fmt.Errorf("copy length is err: len = %d\n", len) + return "", fmt.Errorf("copy length is err: len = %d", len) } return *(*string)(unsafe.Pointer(&dst)), err } - -func (sa *xStringAllocator) FreeString(content string)error{ +func (sa *xStringAllocator) FreeString(content string) error { sh := (*reflect.StringHeader)(unsafe.Pointer(&content)) return sa.sp.Free(sh.Data) } - diff --git a/trees.go b/trees.go index c1f985b..6950c44 100644 --- a/trees.go +++ b/trees.go @@ -332,7 +332,7 @@ func (root *xTreap) removeNode(t *treapNode) error { root.treap = nil } // Return the found treapNode's span after freeing the treapNode. - //mheap_.treapalloc.free(unsafe.Pointer(t)) + // mheap_.treapalloc.free(unsafe.Pointer(t)) return nil } diff --git a/xmm.go b/xmm.go index 992557d..22de482 100644 --- a/xmm.go +++ b/xmm.go @@ -27,30 +27,30 @@ import ( var NilError = errors.New("params is illegal") type spanPool interface { - //Alloc 分配一般对象 + // Alloc 分配一般对象 Alloc(byteSize uintptr) (p unsafe.Pointer, err error) - //AllocSlice 分配slice + // AllocSlice 分配slice AllocSlice(eleSize uintptr, cap, len uintptr) (p unsafe.Pointer, err error) - //Free 释放内存 + // Free 释放内存 Free(addr uintptr) error - //Copy2 byte内存拷贝(拷贝两个) item1-> newItem1 item2-> newItem2 + // Copy2 byte内存拷贝(拷贝两个) item1-> newItem1 item2-> newItem2 Copy2(item1 []byte, item2 []byte) (newItem1 []byte, newItem2 []byte, err error) } type stringAllocator interface { - //From 分配string xmm内存，并拷贝到xmm内存中 + // From 分配string xmm内存，并拷贝到xmm内存中 From(content string) (p string, err error) - //From2 分配2个string xmm内存，并拷贝到xmm内存中 + // From2 分配2个string xmm内存，并拷贝到xmm内存中 From2(item1 string, item2 string) (newItem1 string, newItem2 string, err error) - //FromInAddr 将contents拷贝到addr内存地址中 + // FromInAddr 将contents拷贝到addr内存地址中 FromInAddr(addr uintptr, contents ...string) (p []*string, err error) - //FreeString 释放字符串 + // FreeString 释放字符串 FreeString(content string) error } @@ -63,10 +63,10 @@ type XMemory interface { spanPool stringAllocator - //RawAlloc 分配原始内存 + // RawAlloc 分配原始内存 RawAlloc(pageNum uintptr) (p *Chunk, err error) - //GetPageSize 得到页大小 + // GetPageSize 得到页大小 GetPageSize() uintptr } diff --git a/xmm_test.go b/xmm_test.go index 42dd323..8deba8b 100644 --- a/xmm_test.go +++ b/xmm_test.go @@ -20,7 +20,6 @@ package xmm import ( "fmt" - "github.com/spf13/cast" "log" "math/rand" "net/http" @@ -33,6 +32,8 @@ import ( "testing" "time" "unsafe" + + "github.com/spf13/cast" ) func TestName(t *testing.T) { @@ -302,8 +303,8 @@ func BenchmarkRBTree_ClearxSpanPool(b *testing.B) { }) } -//数组、string(序列化后内容)、slice需要知道其长度。别的都不需要知道，固定的。 -//释放内容的时候需要传入大小和offset +// 数组、string(序列化后内容)、slice需要知道其长度。别的都不需要知道，固定的。 +// 释放内容的时候需要传入大小和offset func TestString(t *testing.T) { h, err := newXHeap() if err != nil { @@ -320,12 +321,12 @@ func TestString(t *testing.T) { t.Fatal(err) } /*if pre > 0 && uintptr(p)-pre != 128 { - t.Log("重新分配了一个span地址") - } else { - t.Log("同一个span中") - } - pre = uintptr(unsafe.Pointer(p)) - v := *(p)*/ + t.Log("重新分配了一个span地址") + } else { + t.Log("同一个span中") + } + pre = uintptr(unsafe.Pointer(p)) + v := *(p)*/ fmt.Println(p) } } @@ -517,7 +518,7 @@ func TestMm_Free(t *testing.T) { size := unsafe.Sizeof(User{}) for i := 0; i < 1000; i++ { if i%85 == 0 && i > 0 { - //panic清空前85个 + // panic清空前85个 for j := 0; j < 85; j++ { if err := sp.Free(uintptr(us[j])); err != nil { t.Fatal(err) @@ -549,7 +550,7 @@ func TestMm_Free2(t *testing.T) { size := unsafe.Sizeof(User{}) for i := 0; i < 20000; i++ { if i == 12000 { - //删除前600个偶数对象 + // 删除前600个偶数对象 for j := 0; j < 6000; j += 2 { if err := sp.Free(uintptr(us[j])); err != nil { t.Fatal(err) @@ -585,7 +586,7 @@ func TestMm2(t *testing.T) { } uSize := unsafe.Sizeof(User{}) - //t1 := time.Now() + // t1 := time.Now() us := make(chan uintptr, 1000000) var wait sync.WaitGroup wait.Add(10) @@ -716,5 +717,5 @@ func TestAlloc_Benchmark(t *testing.T) { }(j) } wait.Wait() - fmt.Println(time.Now().Sub(now), 10*1000000/1000000, "百万") //300w ops + fmt.Println(time.Now().Sub(now), 10*1000000/1000000, "百万") // 300w ops } From f00ec59d7e0d797b536c04484cf355c13407add6 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 26 Feb 2022 13:13:34 +0800 Subject: [PATCH 2/2] remove .idea --- .gitignore | 2 ++ .idea/.gitignore | 8 -------- .idea/dictionaries | 6 ------ .idea/encodings.xml | 4 ---- .idea/markdown.xml | 9 --------- .idea/misc.xml | 14 -------------- .idea/modules.xml | 8 -------- .idea/vcs.xml | 6 ------ .idea/watcherTasks.xml | 10 ---------- .idea/xmm.iml | 9 --------- 10 files changed, 2 insertions(+), 74 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/dictionaries delete mode 100644 .idea/encodings.xml delete mode 100644 .idea/markdown.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/watcherTasks.xml delete mode 100644 .idea/xmm.iml diff --git a/.gitignore b/.gitignore index 66fd13c..bbfd2cb 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ # Dependency directories (remove the comment below to include it) # vendor/ + +.idea \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b8..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/dictionaries b/.idea/dictionaries deleted file mode 100644 index a88a91f..0000000 --- a/.idea/dictionaries +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml deleted file mode 100644 index 72dd4ff..0000000 --- a/.idea/encodings.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/markdown.xml b/.idea/markdown.xml deleted file mode 100644 index 1e34094..0000000 --- a/.idea/markdown.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index c49ff23..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 50a1fe7..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/watcherTasks.xml b/.idea/watcherTasks.xml deleted file mode 100644 index bd32330..0000000 --- a/.idea/watcherTasks.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/.idea/xmm.iml b/.idea/xmm.iml deleted file mode 100644 index 5e764c4..0000000 --- a/.idea/xmm.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file