From 951dbb1e5b264a253fe99fe5543c6e626ea33f96 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Oct 04 2019 16:54:50 +0000 Subject: [release-branch.go1.13] runtime: grow the heap incrementally Currently, we map and grow the heap a whole arena (64MB) at a time. Unfortunately, in order to fix #32828, we need to switch from scavenging inline with allocation back to scavenging on heap growth, but heap-growth scavenging happens in large jumps because we grow the heap in large jumps. In order to prepare for better heap-growth scavenging, this CL separates mapping more space for the heap from actually "growing" it (tracking the new space with spans). Instead, growing the heap keeps track of the "current arena" it's growing into. It track that with new spans as needed, and only maps more arena space when the current arena is inadequate. The effect to the user is the same, but this will let us scavenge on much smaller increments of heap growth. There are two slightly subtleties to this change: 1. If an allocation requires mapping a new arena and that new arena isn't contiguous with the current arena, we don't want to lose the unused space in the current arena, so we have to immediately track that with a span. 2. The mapped space must be accounted as released and idle, even though it isn't actually tracked in a span. For #32828, since this makes heap-growth scavenging far more effective, especially at small heap sizes. For example, this change is necessary for TestPhysicalMemoryUtilization to pass once we remove inline scavenging. Updates #34556 Change-Id: I300e74a0534062467e4ce91cdc3508e5ef9aa73a Reviewed-on: https://go-review.googlesource.com/c/go/+/189957 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall Reviewed-by: Michael Knyszek (cherry picked from commit f18109d7e30c8d1a6e1c87ba3458499ac7ab2a79) Reviewed-on: https://go-review.googlesource.com/c/go/+/198485 Run-TryBot: Andrew Bonventre Reviewed-by: Austin Clements --- diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index 6009932..e4a7faf 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -345,6 +345,9 @@ func ReadMemStatsSlow() (base, slow MemStats) { slow.HeapReleased += uint64(i.span().released()) } + // Unused space in the current arena also counts as released space. + slow.HeapReleased += uint64(mheap_.curArena.end - mheap_.curArena.base) + getg().m.mallocing-- }) diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index e835fc1..6721f3a 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -186,6 +186,12 @@ type mheap struct { // simply blocking GC (by disabling preemption). sweepArenas []arenaIdx + // curArena is the arena that the heap is currently growing + // into. This should always be physPageSize-aligned. + curArena struct { + base, end uintptr + } + _ uint32 // ensure 64-bit alignment of central // central free lists for small size classes. @@ -1250,29 +1256,74 @@ HaveSpan: // h must be locked. func (h *mheap) grow(npage uintptr) bool { ask := npage << _PageShift - v, size := h.sysAlloc(ask) - if v == nil { - print("runtime: out of memory: cannot allocate ", ask, "-byte block (", memstats.heap_sys, " in use)\n") - return false - } - // Create a fake "in use" span and free it, so that the - // right accounting and coalescing happens. + nBase := round(h.curArena.base+ask, physPageSize) + if nBase > h.curArena.end { + // Not enough room in the current arena. Allocate more + // arena space. This may not be contiguous with the + // current arena, so we have to request the full ask. + av, asize := h.sysAlloc(ask) + if av == nil { + print("runtime: out of memory: cannot allocate ", ask, "-byte block (", memstats.heap_sys, " in use)\n") + return false + } + + if uintptr(av) == h.curArena.end { + // The new space is contiguous with the old + // space, so just extend the current space. + h.curArena.end = uintptr(av) + asize + } else { + // The new space is discontiguous. Track what + // remains of the current space and switch to + // the new space. This should be rare. + if size := h.curArena.end - h.curArena.base; size != 0 { + h.growAddSpan(unsafe.Pointer(h.curArena.base), size) + } + // Switch to the new space. + h.curArena.base = uintptr(av) + h.curArena.end = uintptr(av) + asize + } + + // The memory just allocated counts as both released + // and idle, even though it's not yet backed by spans. + // + // The allocation is always aligned to the heap arena + // size which is always > physPageSize, so its safe to + // just add directly to heap_released. Coalescing, if + // possible, will also always be correct in terms of + // accounting, because s.base() must be a physical + // page boundary. + memstats.heap_released += uint64(asize) + memstats.heap_idle += uint64(asize) + + // Recalculate nBase + nBase = round(h.curArena.base+ask, physPageSize) + } + + // Grow into the current arena. + v := h.curArena.base + h.curArena.base = nBase + h.growAddSpan(unsafe.Pointer(v), nBase-v) + return true +} + +// growAddSpan adds a free span when the heap grows into [v, v+size). +// This memory must be in the Prepared state (not Ready). +// +// h must be locked. +func (h *mheap) growAddSpan(v unsafe.Pointer, size uintptr) { s := (*mspan)(h.spanalloc.alloc()) s.init(uintptr(v), size/pageSize) h.setSpans(s.base(), s.npages, s) s.state = mSpanFree - memstats.heap_idle += uint64(size) - // (*mheap).sysAlloc returns untouched/uncommitted memory. + // [v, v+size) is always in the Prepared state. The new span + // must be marked scavenged so the allocator transitions it to + // Ready when allocating from it. s.scavenged = true - // s is always aligned to the heap arena size which is always > physPageSize, - // so its totally safe to just add directly to heap_released. Coalescing, - // if possible, will also always be correct in terms of accounting, because - // s.base() must be a physical page boundary. - memstats.heap_released += uint64(size) + // This span is both released and idle, but grow already + // updated both memstats. h.coalesce(s) h.free.insert(s) - return true } // Free the span back into the heap.