1 /*
2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "logging/log.hpp"
27 #include "memory/resourceArea.hpp"
28 #include "memory/virtualspace.hpp"
29 #include "oops/compressedOops.hpp"
30 #include "oops/markWord.hpp"
31 #include "oops/oop.inline.hpp"
32 #include "runtime/globals_extension.hpp"
33 #include "runtime/java.hpp"
34 #include "runtime/os.hpp"
35 #include "services/memTracker.hpp"
36 #include "utilities/align.hpp"
37 #include "utilities/formatBuffer.hpp"
38 #include "utilities/powerOfTwo.hpp"
39
40 // ReservedSpace
41
42 // Dummy constructor
43 ReservedSpace::ReservedSpace() : _base(nullptr), _size(0), _noaccess_prefix(0),
44 _alignment(0), _special(false), _fd_for_heap(-1), _executable(false) {
45 }
46
47 ReservedSpace::ReservedSpace(size_t size) : _fd_for_heap(-1) {
48 // Want to use large pages where possible. If the size is
49 // not large page aligned the mapping will be a mix of
50 // large and normal pages.
51 size_t page_size = os::page_size_for_region_unaligned(size, 1);
52 size_t alignment = os::vm_allocation_granularity();
53 initialize(size, alignment, page_size, nullptr, false);
54 }
55
56 ReservedSpace::ReservedSpace(size_t size, size_t preferred_page_size) : _fd_for_heap(-1) {
57 // When a page size is given we don't want to mix large
58 // and normal pages. If the size is not a multiple of the
59 // page size it will be aligned up to achieve this.
60 size_t alignment = os::vm_allocation_granularity();;
61 if (preferred_page_size != os::vm_page_size()) {
62 alignment = MAX2(preferred_page_size, alignment);
63 size = align_up(size, alignment);
64 }
65 initialize(size, alignment, preferred_page_size, nullptr, false);
66 }
67
68 ReservedSpace::ReservedSpace(size_t size,
69 size_t alignment,
70 size_t page_size,
71 char* requested_address) : _fd_for_heap(-1) {
72 initialize(size, alignment, page_size, requested_address, false);
73 }
74
75 ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment, size_t page_size,
76 bool special, bool executable) : _fd_for_heap(-1) {
77 assert((size % os::vm_allocation_granularity()) == 0,
78 "size not allocation aligned");
79 initialize_members(base, size, alignment, page_size, special, executable);
80 }
81
82 // Helper method
83 static char* attempt_map_or_reserve_memory_at(char* base, size_t size, int fd, bool executable) {
84 if (fd != -1) {
85 return os::attempt_map_memory_to_file_at(base, size, fd);
86 }
87 return os::attempt_reserve_memory_at(base, size, executable);
88 }
89
90 // Helper method
91 static char* map_or_reserve_memory(size_t size, int fd, bool executable) {
92 if (fd != -1) {
93 return os::map_memory_to_file(size, fd);
94 }
95 return os::reserve_memory(size, executable);
96 }
97
98 // Helper method
99 static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int fd, bool executable) {
100 if (fd != -1) {
101 return os::map_memory_to_file_aligned(size, alignment, fd);
102 }
103 return os::reserve_memory_aligned(size, alignment, executable);
104 }
105
106 // Helper method
107 static void unmap_or_release_memory(char* base, size_t size, bool is_file_mapped) {
108 if (is_file_mapped) {
109 if (!os::unmap_memory(base, size)) {
110 fatal("os::unmap_memory failed");
111 }
112 } else if (!os::release_memory(base, size)) {
113 fatal("os::release_memory failed");
114 }
115 }
116
117 // Helper method
118 static bool failed_to_reserve_as_requested(char* base, char* requested_address) {
119 if (base == requested_address || requested_address == nullptr) {
120 return false; // did not fail
121 }
122
123 if (base != nullptr) {
124 // Different reserve address may be acceptable in other cases
125 // but for compressed oops heap should be at requested address.
126 assert(UseCompressedOops, "currently requested address used only for compressed oops");
127 log_debug(gc, heap, coops)("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, p2i(base), p2i(requested_address));
128 }
129 return true;
130 }
131
132 static bool use_explicit_large_pages(size_t page_size) {
133 return !os::can_commit_large_page_memory() &&
134 page_size != os::vm_page_size();
135 }
136
137 static bool large_pages_requested() {
138 return UseLargePages &&
139 (!FLAG_IS_DEFAULT(UseLargePages) || !FLAG_IS_DEFAULT(LargePageSizeInBytes));
140 }
141
142 static void log_on_large_pages_failure(char* req_addr, size_t bytes) {
143 if (large_pages_requested()) {
144 // Compressed oops logging.
145 log_debug(gc, heap, coops)("Reserve regular memory without large pages");
146 // JVM style warning that we did not succeed in using large pages.
147 char msg[128];
148 jio_snprintf(msg, sizeof(msg), "Failed to reserve and commit memory using large pages. "
149 "req_addr: " PTR_FORMAT " bytes: " SIZE_FORMAT,
150 req_addr, bytes);
151 warning("%s", msg);
152 }
153 }
154
155 static char* reserve_memory(char* requested_address, const size_t size,
156 const size_t alignment, int fd, bool exec) {
157 char* base;
158 // If the memory was requested at a particular address, use
159 // os::attempt_reserve_memory_at() to avoid mapping over something
160 // important. If the reservation fails, return null.
161 if (requested_address != 0) {
162 assert(is_aligned(requested_address, alignment),
163 "Requested address " PTR_FORMAT " must be aligned to " SIZE_FORMAT,
164 p2i(requested_address), alignment);
165 base = attempt_map_or_reserve_memory_at(requested_address, size, fd, exec);
166 } else {
167 // Optimistically assume that the OS returns an aligned base pointer.
168 // When reserving a large address range, most OSes seem to align to at
169 // least 64K.
170 base = map_or_reserve_memory(size, fd, exec);
171 // Check alignment constraints. This is only needed when there is
172 // no requested address.
173 if (!is_aligned(base, alignment)) {
174 // Base not aligned, retry.
175 unmap_or_release_memory(base, size, fd != -1 /*is_file_mapped*/);
176 // Map using the requested alignment.
177 base = map_or_reserve_memory_aligned(size, alignment, fd, exec);
178 }
179 }
180
181 return base;
182 }
183
184 static char* reserve_memory_special(char* requested_address, const size_t size,
185 const size_t alignment, const size_t page_size, bool exec) {
186
187 log_trace(pagesize)("Attempt special mapping: size: " SIZE_FORMAT "%s, "
188 "alignment: " SIZE_FORMAT "%s",
189 byte_size_in_exact_unit(size), exact_unit_for_byte_size(size),
190 byte_size_in_exact_unit(alignment), exact_unit_for_byte_size(alignment));
191
192 char* base = os::reserve_memory_special(size, alignment, page_size, requested_address, exec);
193 if (base != nullptr) {
194 // Check alignment constraints.
195 assert(is_aligned(base, alignment),
196 "reserve_memory_special() returned an unaligned address, base: " PTR_FORMAT
197 " alignment: " SIZE_FORMAT_X,
198 p2i(base), alignment);
199 }
200 return base;
201 }
202
203 void ReservedSpace::clear_members() {
204 initialize_members(nullptr, 0, 0, 0, false, false);
205 }
206
207 void ReservedSpace::initialize_members(char* base, size_t size, size_t alignment,
208 size_t page_size, bool special, bool executable) {
209 _base = base;
210 _size = size;
211 _alignment = alignment;
212 _page_size = page_size;
213 _special = special;
214 _executable = executable;
215 _noaccess_prefix = 0;
216 }
217
218 void ReservedSpace::reserve(size_t size,
219 size_t alignment,
220 size_t page_size,
221 char* requested_address,
222 bool executable) {
223 assert(is_aligned(size, alignment), "Size must be aligned to the requested alignment");
224
225 // There are basically three different cases that we need to handle below:
226 // 1. Mapping backed by a file
227 // 2. Mapping backed by explicit large pages
228 // 3. Mapping backed by normal pages or transparent huge pages
229 // The first two have restrictions that requires the whole mapping to be
230 // committed up front. To record this the ReservedSpace is marked 'special'.
231
232 // == Case 1 ==
233 if (_fd_for_heap != -1) {
234 // When there is a backing file directory for this space then whether
235 // large pages are allocated is up to the filesystem of the backing file.
236 // So UseLargePages is not taken into account for this reservation.
237 char* base = reserve_memory(requested_address, size, alignment, _fd_for_heap, executable);
238 if (base != nullptr) {
239 initialize_members(base, size, alignment, os::vm_page_size(), true, executable);
240 }
241 // Always return, not possible to fall back to reservation not using a file.
242 return;
243 }
244
245 // == Case 2 ==
246 if (use_explicit_large_pages(page_size)) {
247 // System can't commit large pages i.e. use transparent huge pages and
248 // the caller requested large pages. To satisfy this request we use
249 // explicit large pages and these have to be committed up front to ensure
250 // no reservations are lost.
251 do {
252 char* base = reserve_memory_special(requested_address, size, alignment, page_size, executable);
253 if (base != nullptr) {
254 // Successful reservation using large pages.
255 initialize_members(base, size, alignment, page_size, true, executable);
256 return;
257 }
258 page_size = os::page_sizes().next_smaller(page_size);
259 } while (page_size > os::vm_page_size());
260
261 // Failed to reserve explicit large pages, do proper logging.
262 log_on_large_pages_failure(requested_address, size);
263 // Now fall back to normal reservation.
264 assert(page_size == os::vm_page_size(), "inv");
265 }
266
267 // == Case 3 ==
268 char* base = reserve_memory(requested_address, size, alignment, -1, executable);
269 if (base != nullptr) {
270 // Successful mapping.
271 initialize_members(base, size, alignment, page_size, false, executable);
272 }
273 }
274
275 void ReservedSpace::initialize(size_t size,
276 size_t alignment,
277 size_t page_size,
278 char* requested_address,
279 bool executable) {
280 const size_t granularity = os::vm_allocation_granularity();
281 assert((size & (granularity - 1)) == 0,
282 "size not aligned to os::vm_allocation_granularity()");
283 assert((alignment & (granularity - 1)) == 0,
284 "alignment not aligned to os::vm_allocation_granularity()");
285 assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
286 "not a power of 2");
287 assert(page_size >= os::vm_page_size(), "Invalid page size");
288 assert(is_power_of_2(page_size), "Invalid page size");
289
290 clear_members();
291
292 if (size == 0) {
293 return;
294 }
295
296 // Adjust alignment to not be 0.
297 alignment = MAX2(alignment, os::vm_page_size());
298
299 // Reserve the memory.
300 reserve(size, alignment, page_size, requested_address, executable);
301
302 // Check that the requested address is used if given.
303 if (failed_to_reserve_as_requested(_base, requested_address)) {
304 // OS ignored the requested address, release the reservation.
305 release();
306 return;
307 }
308 }
309
310 ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment) {
311 assert(partition_size <= size(), "partition failed");
312 ReservedSpace result(base(), partition_size, alignment, page_size(), special(), executable());
313 return result;
314 }
315
316
317 ReservedSpace
318 ReservedSpace::last_part(size_t partition_size, size_t alignment) {
319 assert(partition_size <= size(), "partition failed");
320 ReservedSpace result(base() + partition_size, size() - partition_size,
321 alignment, page_size(), special(), executable());
322 return result;
323 }
324
325
326 size_t ReservedSpace::page_align_size_up(size_t size) {
327 return align_up(size, os::vm_page_size());
328 }
329
330
331 size_t ReservedSpace::page_align_size_down(size_t size) {
332 return align_down(size, os::vm_page_size());
333 }
334
335
336 size_t ReservedSpace::allocation_align_size_up(size_t size) {
337 return align_up(size, os::vm_allocation_granularity());
338 }
339
340 void ReservedSpace::release() {
341 if (is_reserved()) {
342 char *real_base = _base - _noaccess_prefix;
343 const size_t real_size = _size + _noaccess_prefix;
344 if (special()) {
345 if (_fd_for_heap != -1) {
346 os::unmap_memory(real_base, real_size);
347 } else {
348 os::release_memory_special(real_base, real_size);
349 }
350 } else{
351 os::release_memory(real_base, real_size);
352 }
353 clear_members();
354 }
355 }
356
357 static size_t noaccess_prefix_size(size_t alignment) {
358 return lcm(os::vm_page_size(), alignment);
359 }
360
361 void ReservedHeapSpace::establish_noaccess_prefix() {
362 assert(_alignment >= os::vm_page_size(), "must be at least page size big");
363 _noaccess_prefix = noaccess_prefix_size(_alignment);
364
365 if (base() && base() + _size > (char *)OopEncodingHeapMax) {
366 if (true
367 WIN64_ONLY(&& !UseLargePages)
368 AIX_ONLY(&& os::vm_page_size() != 64*K)) {
369 // Protect memory at the base of the allocated region.
370 // If special, the page was committed (only matters on windows)
371 if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, _special)) {
372 fatal("cannot protect protection page");
373 }
374 log_debug(gc, heap, coops)("Protected page at the reserved heap base: "
375 PTR_FORMAT " / " INTX_FORMAT " bytes",
376 p2i(_base),
377 _noaccess_prefix);
378 assert(CompressedOops::use_implicit_null_checks() == true, "not initialized?");
379 } else {
380 CompressedOops::set_use_implicit_null_checks(false);
381 }
382 }
383
384 _base += _noaccess_prefix;
385 _size -= _noaccess_prefix;
386 assert(((uintptr_t)_base % _alignment == 0), "must be exactly of required alignment");
387 }
388
389 // Tries to allocate memory of size 'size' at address requested_address with alignment 'alignment'.
390 // Does not check whether the reserved memory actually is at requested_address, as the memory returned
391 // might still fulfill the wishes of the caller.
392 // Assures the memory is aligned to 'alignment'.
393 // NOTE: If ReservedHeapSpace already points to some reserved memory this is freed, first.
394 void ReservedHeapSpace::try_reserve_heap(size_t size,
395 size_t alignment,
396 size_t page_size,
397 char* requested_address) {
398 if (_base != nullptr) {
399 // We tried before, but we didn't like the address delivered.
400 release();
401 }
402
403 // Try to reserve the memory for the heap.
404 log_trace(gc, heap, coops)("Trying to allocate at address " PTR_FORMAT
405 " heap of size " SIZE_FORMAT_X,
406 p2i(requested_address),
407 size);
408
409 reserve(size, alignment, page_size, requested_address, false);
410
411 // Check alignment constraints.
412 if (is_reserved() && !is_aligned(_base, _alignment)) {
413 // Base not aligned, retry.
414 release();
415 }
416 }
417
418 void ReservedHeapSpace::try_reserve_range(char *highest_start,
419 char *lowest_start,
420 size_t attach_point_alignment,
421 char *aligned_heap_base_min_address,
422 char *upper_bound,
423 size_t size,
424 size_t alignment,
425 size_t page_size) {
426 const size_t attach_range = highest_start - lowest_start;
427 // Cap num_attempts at possible number.
428 // At least one is possible even for 0 sized attach range.
429 const uint64_t num_attempts_possible = (attach_range / attach_point_alignment) + 1;
430 const uint64_t num_attempts_to_try = MIN2((uint64_t)HeapSearchSteps, num_attempts_possible);
431
432 const size_t stepsize = (attach_range == 0) ? // Only one try.
433 (size_t) highest_start : align_up(attach_range / num_attempts_to_try, attach_point_alignment);
434
435 // Try attach points from top to bottom.
436 char* attach_point = highest_start;
437 while (attach_point >= lowest_start &&
438 attach_point <= highest_start && // Avoid wrap around.
439 ((_base == nullptr) ||
440 (_base < aligned_heap_base_min_address || _base + size > upper_bound))) {
441 try_reserve_heap(size, alignment, page_size, attach_point);
442 attach_point -= stepsize;
443 }
444 }
445
446 #define SIZE_64K ((uint64_t) UCONST64( 0x10000))
447 #define SIZE_256M ((uint64_t) UCONST64( 0x10000000))
448 #define SIZE_32G ((uint64_t) UCONST64( 0x800000000))
449
450 // Helper for heap allocation. Returns an array with addresses
451 // (OS-specific) which are suited for disjoint base mode. Array is
452 // null terminated.
453 static char** get_attach_addresses_for_disjoint_mode() {
454 static uint64_t addresses[] = {
455 2 * SIZE_32G,
456 3 * SIZE_32G,
457 4 * SIZE_32G,
458 8 * SIZE_32G,
459 10 * SIZE_32G,
460 1 * SIZE_64K * SIZE_32G,
461 2 * SIZE_64K * SIZE_32G,
462 3 * SIZE_64K * SIZE_32G,
463 4 * SIZE_64K * SIZE_32G,
464 16 * SIZE_64K * SIZE_32G,
465 32 * SIZE_64K * SIZE_32G,
466 34 * SIZE_64K * SIZE_32G,
467 0
468 };
469
470 // Sort out addresses smaller than HeapBaseMinAddress. This assumes
471 // the array is sorted.
472 uint i = 0;
473 while (addresses[i] != 0 &&
474 (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) {
475 i++;
476 }
477 uint start = i;
478
479 // Avoid more steps than requested.
480 i = 0;
481 while (addresses[start+i] != 0) {
482 if (i == HeapSearchSteps) {
483 addresses[start+i] = 0;
484 break;
485 }
486 i++;
487 }
488
489 return (char**) &addresses[start];
490 }
491
492 void ReservedHeapSpace::initialize_compressed_heap(const size_t size, size_t alignment, size_t page_size) {
493 guarantee(size + noaccess_prefix_size(alignment) <= OopEncodingHeapMax,
494 "can not allocate compressed oop heap for this size");
495 guarantee(alignment == MAX2(alignment, os::vm_page_size()), "alignment too small");
496
497 const size_t granularity = os::vm_allocation_granularity();
498 assert((size & (granularity - 1)) == 0,
499 "size not aligned to os::vm_allocation_granularity()");
500 assert((alignment & (granularity - 1)) == 0,
501 "alignment not aligned to os::vm_allocation_granularity()");
502 assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
503 "not a power of 2");
504
505 // The necessary attach point alignment for generated wish addresses.
506 // This is needed to increase the chance of attaching for mmap and shmat.
507 const size_t os_attach_point_alignment =
508 AIX_ONLY(SIZE_256M) // Known shm boundary alignment.
509 NOT_AIX(os::vm_allocation_granularity());
510 const size_t attach_point_alignment = lcm(alignment, os_attach_point_alignment);
511
512 char *aligned_heap_base_min_address = (char *)align_up((void *)HeapBaseMinAddress, alignment);
513 size_t noaccess_prefix = ((aligned_heap_base_min_address + size) > (char*)OopEncodingHeapMax) ?
514 noaccess_prefix_size(alignment) : 0;
515
516 // Attempt to alloc at user-given address.
517 if (!FLAG_IS_DEFAULT(HeapBaseMinAddress)) {
518 try_reserve_heap(size + noaccess_prefix, alignment, page_size, aligned_heap_base_min_address);
519 if (_base != aligned_heap_base_min_address) { // Enforce this exact address.
520 release();
521 }
522 }
523
524 // Keep heap at HeapBaseMinAddress.
525 if (_base == nullptr) {
526
527 // Try to allocate the heap at addresses that allow efficient oop compression.
528 // Different schemes are tried, in order of decreasing optimization potential.
529 //
530 // For this, try_reserve_heap() is called with the desired heap base addresses.
531 // A call into the os layer to allocate at a given address can return memory
532 // at a different address than requested. Still, this might be memory at a useful
533 // address. try_reserve_heap() always returns this allocated memory, as only here
534 // the criteria for a good heap are checked.
535
536 // Attempt to allocate so that we can run without base and scale (32-Bit unscaled compressed oops).
537 // Give it several tries from top of range to bottom.
538 if (aligned_heap_base_min_address + size <= (char *)UnscaledOopHeapMax) {
539
540 // Calc address range within we try to attach (range of possible start addresses).
541 char* const highest_start = align_down((char *)UnscaledOopHeapMax - size, attach_point_alignment);
542 char* const lowest_start = align_up(aligned_heap_base_min_address, attach_point_alignment);
543 try_reserve_range(highest_start, lowest_start, attach_point_alignment,
544 aligned_heap_base_min_address, (char *)UnscaledOopHeapMax, size, alignment, page_size);
545 }
546
547 // zerobased: Attempt to allocate in the lower 32G.
548 // But leave room for the compressed class pointers, which is allocated above
549 // the heap.
550 // Note Lilliput: the advantages of this strategy were questionable before
551 // (since CDS=off + Compressed oops + heap large enough to suffocate us out of lower 32g
552 // is rare) and with Lilliput the encoding range drastically shrank. We may just do away
553 // with this altogether.
554 char *zerobased_max = (char *)OopEncodingHeapMax;
555 const size_t class_space = align_up(CompressedClassSpaceSize, alignment);
556 // For small heaps, save some space for compressed class pointer
557 // space so it can be decoded with no base.
558 if (UseCompressedClassPointers && !UseSharedSpaces &&
559 OopEncodingHeapMax <= KlassEncodingMetaspaceMax &&
560 (uint64_t)(aligned_heap_base_min_address + size + class_space) <= KlassEncodingMetaspaceMax) {
561 zerobased_max = (char *)OopEncodingHeapMax - class_space;
562 }
563
564 // Give it several tries from top of range to bottom.
565 if (aligned_heap_base_min_address + size <= zerobased_max && // Zerobased theoretical possible.
566 ((_base == nullptr) || // No previous try succeeded.
567 (_base + size > zerobased_max))) { // Unscaled delivered an arbitrary address.
568
569 // Calc address range within we try to attach (range of possible start addresses).
570 char *const highest_start = align_down(zerobased_max - size, attach_point_alignment);
571 // Need to be careful about size being guaranteed to be less
572 // than UnscaledOopHeapMax due to type constraints.
573 char *lowest_start = aligned_heap_base_min_address;
574 uint64_t unscaled_end = UnscaledOopHeapMax - size;
575 if (unscaled_end < UnscaledOopHeapMax) { // unscaled_end wrapped if size is large
576 lowest_start = MAX2(lowest_start, (char*)unscaled_end);
577 }
578 lowest_start = align_up(lowest_start, attach_point_alignment);
579 try_reserve_range(highest_start, lowest_start, attach_point_alignment,
580 aligned_heap_base_min_address, zerobased_max, size, alignment, page_size);
581 }
582
583 // Now we go for heaps with base != 0. We need a noaccess prefix to efficiently
584 // implement null checks.
585 noaccess_prefix = noaccess_prefix_size(alignment);
586
587 // Try to attach at addresses that are aligned to OopEncodingHeapMax. Disjointbase mode.
588 char** addresses = get_attach_addresses_for_disjoint_mode();
589 int i = 0;
590 while (addresses[i] && // End of array not yet reached.
591 ((_base == nullptr) || // No previous try succeeded.
592 (_base + size > (char *)OopEncodingHeapMax && // Not zerobased or unscaled address.
593 !CompressedOops::is_disjoint_heap_base_address((address)_base)))) { // Not disjoint address.
594 char* const attach_point = addresses[i];
595 assert(attach_point >= aligned_heap_base_min_address, "Flag support broken");
596 try_reserve_heap(size + noaccess_prefix, alignment, page_size, attach_point);
597 i++;
598 }
599
600 // Last, desperate try without any placement.
601 if (_base == nullptr) {
602 log_trace(gc, heap, coops)("Trying to allocate at address nullptr heap of size " SIZE_FORMAT_X, size + noaccess_prefix);
603 initialize(size + noaccess_prefix, alignment, page_size, nullptr, false);
604 }
605 }
606 }
607
608 ReservedHeapSpace::ReservedHeapSpace(size_t size, size_t alignment, size_t page_size, const char* heap_allocation_directory) : ReservedSpace() {
609
610 if (size == 0) {
611 return;
612 }
613
614 if (heap_allocation_directory != nullptr) {
615 _fd_for_heap = os::create_file_for_heap(heap_allocation_directory);
616 if (_fd_for_heap == -1) {
617 vm_exit_during_initialization(
618 err_msg("Could not create file for Heap at location %s", heap_allocation_directory));
619 }
620 // When there is a backing file directory for this space then whether
621 // large pages are allocated is up to the filesystem of the backing file.
622 // If requested, let the user know that explicit large pages can't be used.
623 if (use_explicit_large_pages(page_size) && large_pages_requested()) {
624 log_debug(gc, heap)("Cannot allocate explicit large pages for Java Heap when AllocateHeapAt option is set.");
625 }
626 }
627
628 // Heap size should be aligned to alignment, too.
629 guarantee(is_aligned(size, alignment), "set by caller");
630
631 if (UseCompressedOops) {
632 initialize_compressed_heap(size, alignment, page_size);
633 if (_size > size) {
634 // We allocated heap with noaccess prefix.
635 // It can happen we get a zerobased/unscaled heap with noaccess prefix,
636 // if we had to try at arbitrary address.
637 establish_noaccess_prefix();
638 }
639 } else {
640 initialize(size, alignment, page_size, nullptr, false);
641 }
642
643 assert(markWord::encode_pointer_as_mark(_base).decode_pointer() == _base,
644 "area must be distinguishable from marks for mark-sweep");
645 assert(markWord::encode_pointer_as_mark(&_base[size]).decode_pointer() == &_base[size],
646 "area must be distinguishable from marks for mark-sweep");
647
648 if (base() != nullptr) {
649 MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap);
650 }
651
652 if (_fd_for_heap != -1) {
653 ::close(_fd_for_heap);
654 }
655 }
656
657 MemRegion ReservedHeapSpace::region() const {
658 return MemRegion((HeapWord*)base(), (HeapWord*)end());
659 }
660
661 // Reserve space for code segment. Same as Java heap only we mark this as
662 // executable.
663 ReservedCodeSpace::ReservedCodeSpace(size_t r_size,
664 size_t rs_align,
665 size_t rs_page_size) : ReservedSpace() {
666 initialize(r_size, rs_align, rs_page_size, /*requested address*/ nullptr, /*executable*/ true);
667 MemTracker::record_virtual_memory_type((address)base(), mtCode);
668 }
669
670 // VirtualSpace
671
672 VirtualSpace::VirtualSpace() {
673 _low_boundary = nullptr;
674 _high_boundary = nullptr;
675 _low = nullptr;
676 _high = nullptr;
677 _lower_high = nullptr;
678 _middle_high = nullptr;
679 _upper_high = nullptr;
680 _lower_high_boundary = nullptr;
681 _middle_high_boundary = nullptr;
682 _upper_high_boundary = nullptr;
683 _lower_alignment = 0;
684 _middle_alignment = 0;
685 _upper_alignment = 0;
686 _special = false;
687 _executable = false;
688 }
689
690
691 bool VirtualSpace::initialize(ReservedSpace rs, size_t committed_size) {
692 const size_t max_commit_granularity = os::page_size_for_region_unaligned(rs.size(), 1);
693 return initialize_with_granularity(rs, committed_size, max_commit_granularity);
694 }
695
696 bool VirtualSpace::initialize_with_granularity(ReservedSpace rs, size_t committed_size, size_t max_commit_granularity) {
697 if(!rs.is_reserved()) return false; // allocation failed.
698 assert(_low_boundary == nullptr, "VirtualSpace already initialized");
699 assert(max_commit_granularity > 0, "Granularity must be non-zero.");
700
701 _low_boundary = rs.base();
702 _high_boundary = low_boundary() + rs.size();
703
704 _low = low_boundary();
705 _high = low();
706
707 _special = rs.special();
708 _executable = rs.executable();
709
710 // When a VirtualSpace begins life at a large size, make all future expansion
711 // and shrinking occur aligned to a granularity of large pages. This avoids
712 // fragmentation of physical addresses that inhibits the use of large pages
713 // by the OS virtual memory system. Empirically, we see that with a 4MB
714 // page size, the only spaces that get handled this way are codecache and
715 // the heap itself, both of which provide a substantial performance
716 // boost in many benchmarks when covered by large pages.
717 //
718 // No attempt is made to force large page alignment at the very top and
719 // bottom of the space if they are not aligned so already.
720 _lower_alignment = os::vm_page_size();
721 _middle_alignment = max_commit_granularity;
722 _upper_alignment = os::vm_page_size();
723
724 // End of each region
725 _lower_high_boundary = align_up(low_boundary(), middle_alignment());
726 _middle_high_boundary = align_down(high_boundary(), middle_alignment());
727 _upper_high_boundary = high_boundary();
728
729 // High address of each region
730 _lower_high = low_boundary();
731 _middle_high = lower_high_boundary();
732 _upper_high = middle_high_boundary();
733
734 // commit to initial size
735 if (committed_size > 0) {
736 if (!expand_by(committed_size)) {
737 return false;
738 }
739 }
740 return true;
741 }
742
743
744 VirtualSpace::~VirtualSpace() {
745 release();
746 }
747
748
749 void VirtualSpace::release() {
750 // This does not release memory it reserved.
751 // Caller must release via rs.release();
752 _low_boundary = nullptr;
753 _high_boundary = nullptr;
754 _low = nullptr;
755 _high = nullptr;
756 _lower_high = nullptr;
757 _middle_high = nullptr;
758 _upper_high = nullptr;
759 _lower_high_boundary = nullptr;
760 _middle_high_boundary = nullptr;
761 _upper_high_boundary = nullptr;
762 _lower_alignment = 0;
763 _middle_alignment = 0;
764 _upper_alignment = 0;
765 _special = false;
766 _executable = false;
767 }
768
769
770 size_t VirtualSpace::committed_size() const {
771 return pointer_delta(high(), low(), sizeof(char));
772 }
773
774
775 size_t VirtualSpace::reserved_size() const {
776 return pointer_delta(high_boundary(), low_boundary(), sizeof(char));
777 }
778
779
780 size_t VirtualSpace::uncommitted_size() const {
781 return reserved_size() - committed_size();
782 }
783
784 size_t VirtualSpace::actual_committed_size() const {
785 // Special VirtualSpaces commit all reserved space up front.
786 if (special()) {
787 return reserved_size();
788 }
789
790 size_t committed_low = pointer_delta(_lower_high, _low_boundary, sizeof(char));
791 size_t committed_middle = pointer_delta(_middle_high, _lower_high_boundary, sizeof(char));
792 size_t committed_high = pointer_delta(_upper_high, _middle_high_boundary, sizeof(char));
793
794 #ifdef ASSERT
795 size_t lower = pointer_delta(_lower_high_boundary, _low_boundary, sizeof(char));
796 size_t middle = pointer_delta(_middle_high_boundary, _lower_high_boundary, sizeof(char));
797 size_t upper = pointer_delta(_upper_high_boundary, _middle_high_boundary, sizeof(char));
798
799 if (committed_high > 0) {
800 assert(committed_low == lower, "Must be");
801 assert(committed_middle == middle, "Must be");
802 }
803
804 if (committed_middle > 0) {
805 assert(committed_low == lower, "Must be");
806 }
807 if (committed_middle < middle) {
808 assert(committed_high == 0, "Must be");
809 }
810
811 if (committed_low < lower) {
812 assert(committed_high == 0, "Must be");
813 assert(committed_middle == 0, "Must be");
814 }
815 #endif
816
817 return committed_low + committed_middle + committed_high;
818 }
819
820
821 bool VirtualSpace::contains(const void* p) const {
822 return low() <= (const char*) p && (const char*) p < high();
823 }
824
825 static void pretouch_expanded_memory(void* start, void* end) {
826 assert(is_aligned(start, os::vm_page_size()), "Unexpected alignment");
827 assert(is_aligned(end, os::vm_page_size()), "Unexpected alignment");
828
829 os::pretouch_memory(start, end);
830 }
831
832 static bool commit_expanded(char* start, size_t size, size_t alignment, bool pre_touch, bool executable) {
833 if (os::commit_memory(start, size, alignment, executable)) {
834 if (pre_touch || AlwaysPreTouch) {
835 pretouch_expanded_memory(start, start + size);
836 }
837 return true;
838 }
839
840 debug_only(warning(
841 "INFO: os::commit_memory(" PTR_FORMAT ", " PTR_FORMAT
842 " size=" SIZE_FORMAT ", executable=%d) failed",
843 p2i(start), p2i(start + size), size, executable);)
844
845 return false;
846 }
847
848 /*
849 First we need to determine if a particular virtual space is using large
850 pages. This is done at the initialize function and only virtual spaces
851 that are larger than LargePageSizeInBytes use large pages. Once we
852 have determined this, all expand_by and shrink_by calls must grow and
853 shrink by large page size chunks. If a particular request
854 is within the current large page, the call to commit and uncommit memory
855 can be ignored. In the case that the low and high boundaries of this
856 space is not large page aligned, the pages leading to the first large
857 page address and the pages after the last large page address must be
858 allocated with default pages.
859 */
860 bool VirtualSpace::expand_by(size_t bytes, bool pre_touch) {
861 if (uncommitted_size() < bytes) {
862 return false;
863 }
864
865 if (special()) {
866 // don't commit memory if the entire space is pinned in memory
867 _high += bytes;
868 return true;
869 }
870
871 char* previous_high = high();
872 char* unaligned_new_high = high() + bytes;
873 assert(unaligned_new_high <= high_boundary(), "cannot expand by more than upper boundary");
874
875 // Calculate where the new high for each of the regions should be. If
876 // the low_boundary() and high_boundary() are LargePageSizeInBytes aligned
877 // then the unaligned lower and upper new highs would be the
878 // lower_high() and upper_high() respectively.
879 char* unaligned_lower_new_high = MIN2(unaligned_new_high, lower_high_boundary());
880 char* unaligned_middle_new_high = MIN2(unaligned_new_high, middle_high_boundary());
881 char* unaligned_upper_new_high = MIN2(unaligned_new_high, upper_high_boundary());
882
883 // Align the new highs based on the regions alignment. lower and upper
884 // alignment will always be default page size. middle alignment will be
885 // LargePageSizeInBytes if the actual size of the virtual space is in
886 // fact larger than LargePageSizeInBytes.
887 char* aligned_lower_new_high = align_up(unaligned_lower_new_high, lower_alignment());
888 char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
889 char* aligned_upper_new_high = align_up(unaligned_upper_new_high, upper_alignment());
890
891 // Determine which regions need to grow in this expand_by call.
892 // If you are growing in the lower region, high() must be in that
893 // region so calculate the size based on high(). For the middle and
894 // upper regions, determine the starting point of growth based on the
895 // location of high(). By getting the MAX of the region's low address
896 // (or the previous region's high address) and high(), we can tell if it
897 // is an intra or inter region growth.
898 size_t lower_needs = 0;
899 if (aligned_lower_new_high > lower_high()) {
900 lower_needs = pointer_delta(aligned_lower_new_high, lower_high(), sizeof(char));
901 }
902 size_t middle_needs = 0;
903 if (aligned_middle_new_high > middle_high()) {
904 middle_needs = pointer_delta(aligned_middle_new_high, middle_high(), sizeof(char));
905 }
906 size_t upper_needs = 0;
907 if (aligned_upper_new_high > upper_high()) {
908 upper_needs = pointer_delta(aligned_upper_new_high, upper_high(), sizeof(char));
909 }
910
911 // Check contiguity.
912 assert(low_boundary() <= lower_high() && lower_high() <= lower_high_boundary(),
913 "high address must be contained within the region");
914 assert(lower_high_boundary() <= middle_high() && middle_high() <= middle_high_boundary(),
915 "high address must be contained within the region");
916 assert(middle_high_boundary() <= upper_high() && upper_high() <= upper_high_boundary(),
917 "high address must be contained within the region");
918
919 // Commit regions
920 if (lower_needs > 0) {
921 assert(lower_high() + lower_needs <= lower_high_boundary(), "must not expand beyond region");
922 if (!commit_expanded(lower_high(), lower_needs, _lower_alignment, pre_touch, _executable)) {
923 return false;
924 }
925 _lower_high += lower_needs;
926 }
927
928 if (middle_needs > 0) {
929 assert(middle_high() + middle_needs <= middle_high_boundary(), "must not expand beyond region");
930 if (!commit_expanded(middle_high(), middle_needs, _middle_alignment, pre_touch, _executable)) {
931 return false;
932 }
933 _middle_high += middle_needs;
934 }
935
936 if (upper_needs > 0) {
937 assert(upper_high() + upper_needs <= upper_high_boundary(), "must not expand beyond region");
938 if (!commit_expanded(upper_high(), upper_needs, _upper_alignment, pre_touch, _executable)) {
939 return false;
940 }
941 _upper_high += upper_needs;
942 }
943
944 _high += bytes;
945 return true;
946 }
947
948 // A page is uncommitted if the contents of the entire page is deemed unusable.
949 // Continue to decrement the high() pointer until it reaches a page boundary
950 // in which case that particular page can now be uncommitted.
951 void VirtualSpace::shrink_by(size_t size) {
952 if (committed_size() < size)
953 fatal("Cannot shrink virtual space to negative size");
954
955 if (special()) {
956 // don't uncommit if the entire space is pinned in memory
957 _high -= size;
958 return;
959 }
960
961 char* unaligned_new_high = high() - size;
962 assert(unaligned_new_high >= low_boundary(), "cannot shrink past lower boundary");
963
964 // Calculate new unaligned address
965 char* unaligned_upper_new_high =
966 MAX2(unaligned_new_high, middle_high_boundary());
967 char* unaligned_middle_new_high =
968 MAX2(unaligned_new_high, lower_high_boundary());
969 char* unaligned_lower_new_high =
970 MAX2(unaligned_new_high, low_boundary());
971
972 // Align address to region's alignment
973 char* aligned_upper_new_high = align_up(unaligned_upper_new_high, upper_alignment());
974 char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
975 char* aligned_lower_new_high = align_up(unaligned_lower_new_high, lower_alignment());
976
977 // Determine which regions need to shrink
978 size_t upper_needs = 0;
979 if (aligned_upper_new_high < upper_high()) {
980 upper_needs =
981 pointer_delta(upper_high(), aligned_upper_new_high, sizeof(char));
982 }
983 size_t middle_needs = 0;
984 if (aligned_middle_new_high < middle_high()) {
985 middle_needs =
986 pointer_delta(middle_high(), aligned_middle_new_high, sizeof(char));
987 }
988 size_t lower_needs = 0;
989 if (aligned_lower_new_high < lower_high()) {
990 lower_needs =
991 pointer_delta(lower_high(), aligned_lower_new_high, sizeof(char));
992 }
993
994 // Check contiguity.
995 assert(middle_high_boundary() <= upper_high() &&
996 upper_high() <= upper_high_boundary(),
997 "high address must be contained within the region");
998 assert(lower_high_boundary() <= middle_high() &&
999 middle_high() <= middle_high_boundary(),
1000 "high address must be contained within the region");
1001 assert(low_boundary() <= lower_high() &&
1002 lower_high() <= lower_high_boundary(),
1003 "high address must be contained within the region");
1004
1005 // Uncommit
1006 if (upper_needs > 0) {
1007 assert(middle_high_boundary() <= aligned_upper_new_high &&
1008 aligned_upper_new_high + upper_needs <= upper_high_boundary(),
1009 "must not shrink beyond region");
1010 if (!os::uncommit_memory(aligned_upper_new_high, upper_needs, _executable)) {
1011 debug_only(warning("os::uncommit_memory failed"));
1012 return;
1013 } else {
1014 _upper_high -= upper_needs;
1015 }
1016 }
1017 if (middle_needs > 0) {
1018 assert(lower_high_boundary() <= aligned_middle_new_high &&
1019 aligned_middle_new_high + middle_needs <= middle_high_boundary(),
1020 "must not shrink beyond region");
1021 if (!os::uncommit_memory(aligned_middle_new_high, middle_needs, _executable)) {
1022 debug_only(warning("os::uncommit_memory failed"));
1023 return;
1024 } else {
1025 _middle_high -= middle_needs;
1026 }
1027 }
1028 if (lower_needs > 0) {
1029 assert(low_boundary() <= aligned_lower_new_high &&
1030 aligned_lower_new_high + lower_needs <= lower_high_boundary(),
1031 "must not shrink beyond region");
1032 if (!os::uncommit_memory(aligned_lower_new_high, lower_needs, _executable)) {
1033 debug_only(warning("os::uncommit_memory failed"));
1034 return;
1035 } else {
1036 _lower_high -= lower_needs;
1037 }
1038 }
1039
1040 _high -= size;
1041 }
1042
1043 #ifndef PRODUCT
1044 void VirtualSpace::check_for_contiguity() {
1045 // Check contiguity.
1046 assert(low_boundary() <= lower_high() &&
1047 lower_high() <= lower_high_boundary(),
1048 "high address must be contained within the region");
1049 assert(lower_high_boundary() <= middle_high() &&
1050 middle_high() <= middle_high_boundary(),
1051 "high address must be contained within the region");
1052 assert(middle_high_boundary() <= upper_high() &&
1053 upper_high() <= upper_high_boundary(),
1054 "high address must be contained within the region");
1055 assert(low() >= low_boundary(), "low");
1056 assert(low_boundary() <= lower_high_boundary(), "lower high boundary");
1057 assert(upper_high_boundary() <= high_boundary(), "upper high boundary");
1058 assert(high() <= upper_high(), "upper high");
1059 }
1060
1061 void VirtualSpace::print_on(outputStream* out) const {
1062 out->print ("Virtual space:");
1063 if (special()) out->print(" (pinned in memory)");
1064 out->cr();
1065 out->print_cr(" - committed: " SIZE_FORMAT, committed_size());
1066 out->print_cr(" - reserved: " SIZE_FORMAT, reserved_size());
1067 out->print_cr(" - [low, high]: [" PTR_FORMAT ", " PTR_FORMAT "]", p2i(low()), p2i(high()));
1068 out->print_cr(" - [low_b, high_b]: [" PTR_FORMAT ", " PTR_FORMAT "]", p2i(low_boundary()), p2i(high_boundary()));
1069 }
1070
1071 void VirtualSpace::print() const {
1072 print_on(tty);
1073 }
1074
1075 #endif