runtime: convert page allocator bitmap to sparse array

Currently the page allocator bitmap is implemented as a single giant
memory mapping which is reserved at init time and committed as needed.
This causes problems on systems that don't handle large uncommitted
mappings well, or institute low virtual address space defaults as a
memory limiting mechanism.

This change modifies the implementation of the page allocator bitmap
away from a directly-mapped set of bytes to a sparse array in same vein
as mheap.arenas. This will hurt performance a little but the biggest
gains are from the lockless allocation possible with the page allocator,
so the impact of this extra layer of indirection should be minimal.

In fact, this is exactly what we see:
    https://perf.golang.org/search?q=upload:20191125.5

This reduces the amount of mapped (PROT_NONE) memory needed on systems
with 48-bit address spaces to ~600 MiB down from almost 9 GiB. The bulk
of this remaining memory is used by the summaries.

Go processes with 32-bit address spaces now always commit to 128 KiB of
memory for the bitmap. Previously it would only commit the pages in the
bitmap which represented the range of addresses (lowest address to
highest address, even if there are unused regions in that range) used by
the heap.

Updates #35568.
Updates #35451.

Change-Id: I0ff10380156568642b80c366001eefd0a4e6c762
Reviewed-on: https://go-review.googlesource.com/c/go/+/207497
Run-TryBot: Michael Knyszek <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Austin Clements <[email protected]>
Reviewed-by: Cherry Zhang <[email protected]>

src/runtime/export_test.go

@@ -355,7 +355,7 @@ func ReadMemStatsSlow() (base, slow MemStats) {
 		}
 
 		for i := mheap_.pages.start; i < mheap_.pages.end; i++ {
-			pg := mheap_.pages.chunks[i].scavenged.popcntRange(0, pallocChunkPages)
+			pg := mheap_.pages.chunkOf(i).scavenged.popcntRange(0, pallocChunkPages)
 			slow.HeapReleased += uint64(pg) * pageSize
 		}
 		for _, p := range allp {
@@ -726,16 +726,23 @@ func (p *PageAlloc) Free(base, npages uintptr) {
 func (p *PageAlloc) Bounds() (ChunkIdx, ChunkIdx) {
 	return ChunkIdx((*pageAlloc)(p).start), ChunkIdx((*pageAlloc)(p).end)
 }
-func (p *PageAlloc) PallocData(i ChunkIdx) *PallocData {
-	return (*PallocData)(&((*pageAlloc)(p).chunks[i]))
-}
 func (p *PageAlloc) Scavenge(nbytes uintptr, locked bool) (r uintptr) {
 	systemstack(func() {
 		r = (*pageAlloc)(p).scavenge(nbytes, locked)
 	})
 	return
 }
 
+// Returns nil if the PallocData's L2 is missing.
+func (p *PageAlloc) PallocData(i ChunkIdx) *PallocData {
+	ci := chunkIdx(i)
+	l2 := (*pageAlloc)(p).chunks[ci.l1()]
+	if l2 == nil {
+		return nil
+	}
+	return (*PallocData)(&l2[ci.l2()])
+}
+
 // BitRange represents a range over a bitmap.
 type BitRange struct {
 	I, N uint // bit index and length in bits
@@ -769,7 +776,7 @@ func NewPageAlloc(chunks, scav map[ChunkIdx][]BitRange) *PageAlloc {
 		p.grow(addr, pallocChunkBytes)
 
 		// Initialize the bitmap and update pageAlloc metadata.
-		chunk := &p.chunks[chunkIndex(addr)]
+		chunk := p.chunkOf(chunkIndex(addr))
 
 		// Clear all the scavenged bits which grow set.
 		chunk.scavenged.clearRange(0, pallocChunkPages)
@@ -823,8 +830,13 @@ func FreePageAlloc(pp *PageAlloc) {
 	}
 
 	// Free the mapped space for chunks.
-	chunksLen := uintptr(cap(p.chunks)) * unsafe.Sizeof(p.chunks[0])
-	sysFree(unsafe.Pointer(&p.chunks[0]), alignUp(chunksLen, physPageSize), nil)
+	for i := range p.chunks {
+		if x := p.chunks[i]; x != nil {
+			p.chunks[i] = nil
+			// This memory comes from sysAlloc and will always be page-aligned.
+			sysFree(unsafe.Pointer(x), unsafe.Sizeof(*p.chunks[0]), nil)
+		}
+	}
 }
 
 // BaseChunkIdx is a convenient chunkIdx value which works on both
@@ -861,7 +873,7 @@ func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) {
 		lock(&mheap_.lock)
 	chunkLoop:
 		for i := mheap_.pages.start; i < mheap_.pages.end; i++ {
-			chunk := &mheap_.pages.chunks[i]
+			chunk := mheap_.pages.chunkOf(i)
 			for j := 0; j < pallocChunkPages/64; j++ {
 				// Run over each 64-bit bitmap section and ensure
 				// scavenged is being cleared properly on allocation.

src/runtime/mgcscavenge.go

@@ -420,7 +420,7 @@ func (s *pageAlloc) scavengeOne(max uintptr, locked bool) uintptr {
 		// continue if the summary says we can because that's how
 		// we can tell if parts of the address space are unused.
 		// See the comment on s.chunks in mpagealloc.go.
-		base, npages := s.chunks[ci].findScavengeCandidate(chunkPageIndex(s.scavAddr), minPages, maxPages)
+		base, npages := s.chunkOf(ci).findScavengeCandidate(chunkPageIndex(s.scavAddr), minPages, maxPages)
 
 		// If we found something, scavenge it and return!
 		if npages != 0 {
@@ -450,16 +450,20 @@ func (s *pageAlloc) scavengeOne(max uintptr, locked bool) uintptr {
 
 		// Run over the chunk looking harder for a candidate. Again, we could
 		// race with a lot of different pieces of code, but we're just being
-		// optimistic.
-		if !s.chunks[i].hasScavengeCandidate(minPages) {
+		// optimistic. Make sure we load the l2 pointer atomically though, to
+		// avoid races with heap growth. It may or may not be possible to also
+		// see a nil pointer in this case if we do race with heap growth, but
+		// just defensively ignore the nils. This operation is optimistic anyway.
+		l2 := (*[1 << pallocChunksL2Bits]pallocData)(atomic.Loadp(unsafe.Pointer(&s.chunks[i.l1()])))
+		if l2 == nil || !l2[i.l2()].hasScavengeCandidate(minPages) {
 			continue
 		}
 
 		// We found a candidate, so let's lock and verify it.
 		lockHeap()
 
 		// Find, verify, and scavenge if we can.
-		chunk := &s.chunks[i]
+		chunk := s.chunkOf(i)
 		base, npages := chunk.findScavengeCandidate(pallocChunkPages-1, minPages, maxPages)
 		if npages > 0 {
 			// We found memory to scavenge! Mark the bits and report that up.
@@ -488,7 +492,7 @@ func (s *pageAlloc) scavengeOne(max uintptr, locked bool) uintptr {
 //
 // s.mheapLock must be held.
 func (s *pageAlloc) scavengeRangeLocked(ci chunkIdx, base, npages uint) {
-	s.chunks[ci].scavenged.setRange(base, npages)
+	s.chunkOf(ci).scavenged.setRange(base, npages)
 
 	// Compute the full address for the start of the range.
 	addr := chunkBase(ci) + uintptr(base)*pageSize