Source file src/runtime/mem.go
1 // Copyright 2022 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import "unsafe" 8 9 // OS memory management abstraction layer 10 // 11 // Regions of the address space managed by the runtime may be in one of four 12 // states at any given time: 13 // 1) None - Unreserved and unmapped, the default state of any region. 14 // 2) Reserved - Owned by the runtime, but accessing it would cause a fault. 15 // Does not count against the process' memory footprint. 16 // 3) Prepared - Reserved, intended not to be backed by physical memory (though 17 // an OS may implement this lazily). Can transition efficiently to 18 // Ready. Accessing memory in such a region is undefined (may 19 // fault, may give back unexpected zeroes, etc.). 20 // 4) Ready - may be accessed safely. 21 // 22 // This set of states is more than is strictly necessary to support all the 23 // currently supported platforms. One could get by with just None, Reserved, and 24 // Ready. However, the Prepared state gives us flexibility for performance 25 // purposes. For example, on POSIX-y operating systems, Reserved is usually a 26 // private anonymous mmap'd region with PROT_NONE set, and to transition 27 // to Ready would require setting PROT_READ|PROT_WRITE. However the 28 // underspecification of Prepared lets us use just MADV_FREE to transition from 29 // Ready to Prepared. Thus with the Prepared state we can set the permission 30 // bits just once early on, we can efficiently tell the OS that it's free to 31 // take pages away from us when we don't strictly need them. 32 // 33 // This file defines a cross-OS interface for a common set of helpers 34 // that transition memory regions between these states. The helpers call into 35 // OS-specific implementations that handle errors, while the interface boundary 36 // implements cross-OS functionality, like updating runtime accounting. 37 38 // sysAlloc transitions an OS-chosen region of memory from None to Ready. 39 // More specifically, it obtains a large chunk of zeroed memory from the 40 // operating system, typically on the order of a hundred kilobytes 41 // or a megabyte. This memory is always immediately available for use. 42 // 43 // sysStat must be non-nil. 44 // 45 // Don't split the stack as this function may be invoked without a valid G, 46 // which prevents us from allocating more stack. 47 // 48 //go:nosplit 49 func sysAlloc(n uintptr, sysStat *sysMemStat, vmaName string) unsafe.Pointer { 50 sysStat.add(int64(n)) 51 gcController.mappedReady.Add(int64(n)) 52 p := sysAllocOS(n, vmaName) 53 54 // When using ASAN leak detection, we must tell ASAN about 55 // cases where we store pointers in mmapped memory. 56 if asanenabled { 57 lsanregisterrootregion(p, n) 58 } 59 60 return p 61 } 62 63 // sysUnused transitions a memory region from Ready to Prepared. It notifies the 64 // operating system that the physical pages backing this memory region are no 65 // longer needed and can be reused for other purposes. The contents of a 66 // sysUnused memory region are considered forfeit and the region must not be 67 // accessed again until sysUsed is called. 68 func sysUnused(v unsafe.Pointer, n uintptr) { 69 gcController.mappedReady.Add(-int64(n)) 70 sysUnusedOS(v, n) 71 } 72 73 // sysUsed transitions a memory region from Prepared to Ready. It notifies the 74 // operating system that the memory region is needed and ensures that the region 75 // may be safely accessed. This is typically a no-op on systems that don't have 76 // an explicit commit step and hard over-commit limits, but is critical on 77 // Windows, for example. 78 // 79 // This operation is idempotent for memory already in the Prepared state, so 80 // it is safe to refer, with v and n, to a range of memory that includes both 81 // Prepared and Ready memory. However, the caller must provide the exact amount 82 // of Prepared memory for accounting purposes. 83 func sysUsed(v unsafe.Pointer, n, prepared uintptr) { 84 gcController.mappedReady.Add(int64(prepared)) 85 sysUsedOS(v, n) 86 } 87 88 // sysHugePage does not transition memory regions, but instead provides a 89 // hint to the OS that it would be more efficient to back this memory region 90 // with pages of a larger size transparently. 91 func sysHugePage(v unsafe.Pointer, n uintptr) { 92 sysHugePageOS(v, n) 93 } 94 95 // sysNoHugePage does not transition memory regions, but instead provides a 96 // hint to the OS that it would be less efficient to back this memory region 97 // with pages of a larger size transparently. 98 func sysNoHugePage(v unsafe.Pointer, n uintptr) { 99 sysNoHugePageOS(v, n) 100 } 101 102 // sysHugePageCollapse attempts to immediately back the provided memory region 103 // with huge pages. It is best-effort and may fail silently. 104 func sysHugePageCollapse(v unsafe.Pointer, n uintptr) { 105 sysHugePageCollapseOS(v, n) 106 } 107 108 // sysFree transitions a memory region from any state to None. Therefore, it 109 // returns memory unconditionally. It is used if an out-of-memory error has been 110 // detected midway through an allocation or to carve out an aligned section of 111 // the address space. It is okay if sysFree is a no-op only if sysReserve always 112 // returns a memory region aligned to the heap allocator's alignment 113 // restrictions. 114 // 115 // sysStat must be non-nil. 116 // 117 // The size and start address must exactly match the size and returned address 118 // from the original sysAlloc/sysReserve/sysReserveAligned call. That is, 119 // sysFree cannot be used to free a subset of a memory region. 120 // 121 // Don't split the stack as this function may be invoked without a valid G, 122 // which prevents us from allocating more stack. 123 // 124 //go:nosplit 125 func sysFree(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) { 126 // When using ASAN leak detection, the memory being freed is known by 127 // the sanitizer. We need to unregister it so it's not accessed by it. 128 // 129 // lsanunregisterrootregion matches regions by start address and size, 130 // so it is not possible to unregister a subset of the region. This is 131 // why sysFree requires the full region from the initial allocation. 132 if asanenabled { 133 lsanunregisterrootregion(v, n) 134 } 135 136 sysStat.add(-int64(n)) 137 gcController.mappedReady.Add(-int64(n)) 138 sysFreeOS(v, n) 139 } 140 141 // sysFault transitions a memory region from Ready to Reserved. It 142 // marks a region such that it will always fault if accessed. Used only for 143 // debugging the runtime. 144 // 145 // TODO(mknyszek): Currently it's true that all uses of sysFault transition 146 // memory from Ready to Reserved, but this may not be true in the future 147 // since on every platform the operation is much more general than that. 148 // If a transition from Prepared is ever introduced, create a new function 149 // that elides the Ready state accounting. 150 func sysFault(v unsafe.Pointer, n uintptr) { 151 gcController.mappedReady.Add(-int64(n)) 152 sysFaultOS(v, n) 153 } 154 155 // sysReserve transitions a memory region from None to Reserved. It reserves 156 // address space in such a way that it would cause a fatal fault upon access 157 // (either via permissions or not committing the memory). Such a reservation is 158 // thus never backed by physical memory. 159 // 160 // If the pointer passed to it is non-nil, the caller wants the reservation 161 // there, but sysReserve can still choose another location if that one is 162 // unavailable. 163 // 164 // sysReserve returns OS-aligned memory. If a larger alignment is required, use 165 // sysReservedAligned. 166 func sysReserve(v unsafe.Pointer, n uintptr, vmaName string) unsafe.Pointer { 167 p := sysReserveOS(v, n, vmaName) 168 169 // When using ASAN leak detection, we must tell ASAN about 170 // cases where we store pointers in mmapped memory. 171 if asanenabled { 172 lsanregisterrootregion(p, n) 173 } 174 175 return p 176 } 177 178 // sysReserveAligned transitions a memory region from None to Reserved. 179 // 180 // Semantics are equivlent to sysReserve, but the returned pointer is aligned 181 // to align bytes. It may reserve either n or n+align bytes, so it returns the 182 // size that was reserved. 183 func sysReserveAligned(v unsafe.Pointer, size, align uintptr, vmaName string) (unsafe.Pointer, uintptr) { 184 if isSbrkPlatform { 185 if v != nil { 186 throw("unexpected heap arena hint on sbrk platform") 187 } 188 return sysReserveAlignedSbrk(size, align) 189 } 190 // Since the alignment is rather large in uses of this 191 // function, we're not likely to get it by chance, so we ask 192 // for a larger region and remove the parts we don't need. 193 retries := 0 194 retry: 195 p := uintptr(sysReserve(v, size+align, vmaName)) 196 switch { 197 case p == 0: 198 return nil, 0 199 case p&(align-1) == 0: 200 return unsafe.Pointer(p), size + align 201 case GOOS == "windows": 202 // On Windows we can't release pieces of a 203 // reservation, so we release the whole thing and 204 // re-reserve the aligned sub-region. This may race, 205 // so we may have to try again. 206 sysUnreserve(unsafe.Pointer(p), size+align) 207 p = alignUp(p, align) 208 p2 := sysReserve(unsafe.Pointer(p), size, vmaName) 209 if p != uintptr(p2) { 210 // Must have raced. Try again. 211 sysUnreserve(p2, size) 212 if retries++; retries == 100 { 213 throw("failed to allocate aligned heap memory; too many retries") 214 } 215 goto retry 216 } 217 // Success. 218 return p2, size 219 default: 220 // Trim off the unaligned parts. 221 pAligned := alignUp(p, align) 222 end := pAligned + size 223 endLen := (p + size + align) - end 224 225 // sysUnreserve does not allow unreserving a subset of the 226 // region because LSAN does not allow unregistering a subset. 227 // So we can't call sysUnreserve. Instead we simply unregister 228 // the entire region from LSAN and re-register with the smaller 229 // region before freeing the unecessary portions, which does 230 // allow subsets of the region. 231 if asanenabled { 232 lsanunregisterrootregion(unsafe.Pointer(p), size+align) 233 lsanregisterrootregion(unsafe.Pointer(pAligned), size) 234 } 235 sysFreeOS(unsafe.Pointer(p), pAligned-p) 236 if endLen > 0 { 237 sysFreeOS(unsafe.Pointer(end), endLen) 238 } 239 return unsafe.Pointer(pAligned), size 240 } 241 } 242 243 // sysUnreserve transitions a memory region from Reserved to None. 244 // 245 // The size and start address must exactly match the size and returned address 246 // from sysReserve/sysReserveAligned. That is, sysUnreserve cannot be used to 247 // unreserve a subset of a memory region. 248 // 249 // Don't split the stack as this function may be invoked without a valid G, 250 // which prevents us from allocating more stack. 251 // 252 //go:nosplit 253 func sysUnreserve(v unsafe.Pointer, n uintptr) { 254 // When using ASAN leak detection, the memory being freed is known by 255 // the sanitizer. We need to unregister it so it's not accessed by it. 256 // 257 // lsanunregisterrootregion matches regions by start address and size, 258 // so it is not possible to unregister a subset of the region. This is 259 // why sysUnreserve requires the full region from sysReserve. 260 if asanenabled { 261 lsanunregisterrootregion(v, n) 262 } 263 264 sysFreeOS(v, n) 265 } 266 267 // sysMap transitions a memory region from Reserved to Prepared. It ensures the 268 // memory region can be efficiently transitioned to Ready. 269 // 270 // sysStat must be non-nil. 271 func sysMap(v unsafe.Pointer, n uintptr, sysStat *sysMemStat, vmaName string) { 272 sysStat.add(int64(n)) 273 sysMapOS(v, n, vmaName) 274 } 275