1 /*
2 * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "logging/log.hpp"
27 #include "memory/resourceArea.hpp"
28 #include "memory/virtualspace.hpp"
29 #include "oops/compressedOops.hpp"
30 #include "oops/markWord.hpp"
31 #include "oops/oop.inline.hpp"
32 #include "runtime/globals_extension.hpp"
33 #include "runtime/java.hpp"
34 #include "runtime/os.hpp"
35 #include "services/memTracker.hpp"
36 #include "utilities/align.hpp"
37 #include "utilities/formatBuffer.hpp"
38 #include "utilities/powerOfTwo.hpp"
39
40 // ReservedSpace
41
42 // Dummy constructor
43 ReservedSpace::ReservedSpace() : _base(NULL), _size(0), _noaccess_prefix(0),
44 _alignment(0), _special(false), _fd_for_heap(-1), _executable(false) {
45 }
46
47 ReservedSpace::ReservedSpace(size_t size) : _fd_for_heap(-1) {
48 // Want to use large pages where possible. If the size is
49 // not large page aligned the mapping will be a mix of
50 // large and normal pages.
51 size_t page_size = os::page_size_for_region_unaligned(size, 1);
52 size_t alignment = os::vm_allocation_granularity();
53 initialize(size, alignment, page_size, NULL, false);
54 }
55
56 ReservedSpace::ReservedSpace(size_t size, size_t preferred_page_size) : _fd_for_heap(-1) {
57 // When a page size is given we don't want to mix large
58 // and normal pages. If the size is not a multiple of the
59 // page size it will be aligned up to achieve this.
60 size_t alignment = os::vm_allocation_granularity();;
61 if (preferred_page_size != (size_t)os::vm_page_size()) {
62 alignment = MAX2(preferred_page_size, alignment);
63 size = align_up(size, alignment);
64 }
65 initialize(size, alignment, preferred_page_size, NULL, false);
66 }
67
68 ReservedSpace::ReservedSpace(size_t size,
69 size_t alignment,
70 size_t page_size,
71 char* requested_address) : _fd_for_heap(-1) {
72 initialize(size, alignment, page_size, requested_address, false);
73 }
74
75 ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment, size_t page_size,
76 bool special, bool executable) : _fd_for_heap(-1) {
77 assert((size % os::vm_allocation_granularity()) == 0,
78 "size not allocation aligned");
79 initialize_members(base, size, alignment, page_size, special, executable);
80 }
81
82 // Helper method
83 static char* attempt_map_or_reserve_memory_at(char* base, size_t size, int fd, bool executable) {
84 if (fd != -1) {
85 return os::attempt_map_memory_to_file_at(base, size, fd);
86 }
87 return os::attempt_reserve_memory_at(base, size, executable);
88 }
89
90 // Helper method
91 static char* map_or_reserve_memory(size_t size, int fd, bool executable) {
92 if (fd != -1) {
93 return os::map_memory_to_file(size, fd);
94 }
95 return os::reserve_memory(size, executable);
96 }
97
98 // Helper method
99 static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int fd, bool executable) {
100 if (fd != -1) {
101 return os::map_memory_to_file_aligned(size, alignment, fd);
102 }
103 return os::reserve_memory_aligned(size, alignment, executable);
104 }
105
106 // Helper method
107 static void unmap_or_release_memory(char* base, size_t size, bool is_file_mapped) {
108 if (is_file_mapped) {
109 if (!os::unmap_memory(base, size)) {
110 fatal("os::unmap_memory failed");
111 }
112 } else if (!os::release_memory(base, size)) {
113 fatal("os::release_memory failed");
114 }
115 }
116
117 // Helper method
118 static bool failed_to_reserve_as_requested(char* base, char* requested_address) {
119 if (base == requested_address || requested_address == NULL) {
120 return false; // did not fail
121 }
122
123 if (base != NULL) {
124 // Different reserve address may be acceptable in other cases
125 // but for compressed oops heap should be at requested address.
126 assert(UseCompressedOops, "currently requested address used only for compressed oops");
127 log_debug(gc, heap, coops)("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, p2i(base), p2i(requested_address));
128 }
129 return true;
130 }
131
132 static bool use_explicit_large_pages(size_t page_size) {
133 return !os::can_commit_large_page_memory() &&
134 page_size != (size_t) os::vm_page_size();
135 }
136
137 static bool large_pages_requested() {
138 return UseLargePages &&
139 (!FLAG_IS_DEFAULT(UseLargePages) || !FLAG_IS_DEFAULT(LargePageSizeInBytes));
140 }
141
142 static void log_on_large_pages_failure(char* req_addr, size_t bytes) {
143 if (large_pages_requested()) {
144 // Compressed oops logging.
145 log_debug(gc, heap, coops)("Reserve regular memory without large pages");
146 // JVM style warning that we did not succeed in using large pages.
147 char msg[128];
148 jio_snprintf(msg, sizeof(msg), "Failed to reserve and commit memory using large pages. "
149 "req_addr: " PTR_FORMAT " bytes: " SIZE_FORMAT,
150 req_addr, bytes);
151 warning("%s", msg);
152 }
153 }
154
155 static char* reserve_memory(char* requested_address, const size_t size,
156 const size_t alignment, int fd, bool exec) {
157 char* base;
158 // If the memory was requested at a particular address, use
159 // os::attempt_reserve_memory_at() to avoid mapping over something
160 // important. If the reservation fails, return NULL.
161 if (requested_address != 0) {
162 assert(is_aligned(requested_address, alignment),
163 "Requested address " PTR_FORMAT " must be aligned to " SIZE_FORMAT,
164 p2i(requested_address), alignment);
165 base = attempt_map_or_reserve_memory_at(requested_address, size, fd, exec);
166 } else {
167 // Optimistically assume that the OS returns an aligned base pointer.
168 // When reserving a large address range, most OSes seem to align to at
169 // least 64K.
170 base = map_or_reserve_memory(size, fd, exec);
171 // Check alignment constraints. This is only needed when there is
172 // no requested address.
173 if (!is_aligned(base, alignment)) {
174 // Base not aligned, retry.
175 unmap_or_release_memory(base, size, fd != -1 /*is_file_mapped*/);
176 // Map using the requested alignment.
177 base = map_or_reserve_memory_aligned(size, alignment, fd, exec);
178 }
179 }
180
181 return base;
182 }
183
184 static char* reserve_memory_special(char* requested_address, const size_t size,
185 const size_t alignment, const size_t page_size, bool exec) {
186
187 log_trace(pagesize)("Attempt special mapping: size: " SIZE_FORMAT "%s, "
188 "alignment: " SIZE_FORMAT "%s",
189 byte_size_in_exact_unit(size), exact_unit_for_byte_size(size),
190 byte_size_in_exact_unit(alignment), exact_unit_for_byte_size(alignment));
191
192 char* base = os::reserve_memory_special(size, alignment, page_size, requested_address, exec);
193 if (base != NULL) {
194 // Check alignment constraints.
195 assert(is_aligned(base, alignment),
196 "reserve_memory_special() returned an unaligned address, base: " PTR_FORMAT
197 " alignment: " SIZE_FORMAT_HEX,
198 p2i(base), alignment);
199 }
200 return base;
201 }
202
203 void ReservedSpace::clear_members() {
204 initialize_members(NULL, 0, 0, 0, false, false);
205 }
206
207 void ReservedSpace::initialize_members(char* base, size_t size, size_t alignment,
208 size_t page_size, bool special, bool executable) {
209 _base = base;
210 _size = size;
211 _alignment = alignment;
212 _page_size = page_size;
213 _special = special;
214 _executable = executable;
215 _noaccess_prefix = 0;
216 }
217
218 void ReservedSpace::reserve(size_t size,
219 size_t alignment,
220 size_t page_size,
221 char* requested_address,
222 bool executable) {
223 assert(is_aligned(size, alignment), "Size must be aligned to the requested alignment");
224
225 // There are basically three different cases that we need to handle below:
226 // - Mapping backed by a file
227 // - Mapping backed by explicit large pages
228 // - Mapping backed by normal pages or transparent huge pages
229 // The first two have restrictions that requires the whole mapping to be
230 // committed up front. To record this the ReservedSpace is marked 'special'.
231
232 if (_fd_for_heap != -1) {
233 // When there is a backing file directory for this space then whether
234 // large pages are allocated is up to the filesystem of the backing file.
235 // So UseLargePages is not taken into account for this reservation.
236 char* base = reserve_memory(requested_address, size, alignment, _fd_for_heap, executable);
237 if (base != NULL) {
238 initialize_members(base, size, alignment, os::vm_page_size(), true, executable);
239 }
240 // Always return, not possible to fall back to reservation not using a file.
241 return;
242 } else if (use_explicit_large_pages(page_size)) {
243 // System can't commit large pages i.e. use transparent huge pages and
244 // the caller requested large pages. To satisfy this request we use
245 // explicit large pages and these have to be committed up front to ensure
246 // no reservations are lost.
247 size_t used_page_size = page_size;
248 char* base = NULL;
249
250 do {
251 base = reserve_memory_special(requested_address, size, alignment, used_page_size, executable);
252 if (base != NULL) {
253 break;
254 }
255 used_page_size = os::page_sizes().next_smaller(used_page_size);
256 } while (used_page_size > (size_t) os::vm_page_size());
257
258 if (base != NULL) {
259 // Successful reservation using large pages.
260 initialize_members(base, size, alignment, used_page_size, true, executable);
261 return;
262 }
263 // Failed to reserve explicit large pages, do proper logging.
264 log_on_large_pages_failure(requested_address, size);
265 // Now fall back to normal reservation.
266 page_size = os::vm_page_size();
267 }
268
269 // Not a 'special' reservation.
270 char* base = reserve_memory(requested_address, size, alignment, -1, executable);
271 if (base != NULL) {
272 // Successful mapping.
273 initialize_members(base, size, alignment, page_size, false, executable);
274 }
275 }
276
277 void ReservedSpace::initialize(size_t size,
278 size_t alignment,
279 size_t page_size,
280 char* requested_address,
281 bool executable) {
282 const size_t granularity = os::vm_allocation_granularity();
283 assert((size & (granularity - 1)) == 0,
284 "size not aligned to os::vm_allocation_granularity()");
285 assert((alignment & (granularity - 1)) == 0,
286 "alignment not aligned to os::vm_allocation_granularity()");
287 assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
288 "not a power of 2");
289 assert(page_size >= (size_t) os::vm_page_size(), "Invalid page size");
290 assert(is_power_of_2(page_size), "Invalid page size");
291
292 clear_members();
293
294 if (size == 0) {
295 return;
296 }
297
298 // Adjust alignment to not be 0.
299 alignment = MAX2(alignment, (size_t)os::vm_page_size());
300
301 // Reserve the memory.
302 reserve(size, alignment, page_size, requested_address, executable);
303
304 // Check that the requested address is used if given.
305 if (failed_to_reserve_as_requested(_base, requested_address)) {
306 // OS ignored the requested address, release the reservation.
307 release();
308 return;
309 }
310 }
311
312 ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment) {
313 assert(partition_size <= size(), "partition failed");
314 ReservedSpace result(base(), partition_size, alignment, page_size(), special(), executable());
315 return result;
316 }
317
318
319 ReservedSpace
320 ReservedSpace::last_part(size_t partition_size, size_t alignment) {
321 assert(partition_size <= size(), "partition failed");
322 ReservedSpace result(base() + partition_size, size() - partition_size,
323 alignment, page_size(), special(), executable());
324 return result;
325 }
326
327
328 size_t ReservedSpace::page_align_size_up(size_t size) {
329 return align_up(size, os::vm_page_size());
330 }
331
332
333 size_t ReservedSpace::page_align_size_down(size_t size) {
334 return align_down(size, os::vm_page_size());
335 }
336
337
338 size_t ReservedSpace::allocation_align_size_up(size_t size) {
339 return align_up(size, os::vm_allocation_granularity());
340 }
341
342 void ReservedSpace::release() {
343 if (is_reserved()) {
344 char *real_base = _base - _noaccess_prefix;
345 const size_t real_size = _size + _noaccess_prefix;
346 if (special()) {
347 if (_fd_for_heap != -1) {
348 os::unmap_memory(real_base, real_size);
349 } else {
350 os::release_memory_special(real_base, real_size);
351 }
352 } else{
353 os::release_memory(real_base, real_size);
354 }
355 clear_members();
356 }
357 }
358
359 static size_t noaccess_prefix_size(size_t alignment) {
360 return lcm(os::vm_page_size(), alignment);
361 }
362
363 void ReservedHeapSpace::establish_noaccess_prefix() {
364 assert(_alignment >= (size_t)os::vm_page_size(), "must be at least page size big");
365 _noaccess_prefix = noaccess_prefix_size(_alignment);
366
367 if (base() && base() + _size > (char *)OopEncodingHeapMax) {
368 if (true
369 WIN64_ONLY(&& !UseLargePages)
370 AIX_ONLY(&& os::vm_page_size() != 64*K)) {
371 // Protect memory at the base of the allocated region.
372 // If special, the page was committed (only matters on windows)
373 if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, _special)) {
374 fatal("cannot protect protection page");
375 }
376 log_debug(gc, heap, coops)("Protected page at the reserved heap base: "
377 PTR_FORMAT " / " INTX_FORMAT " bytes",
378 p2i(_base),
379 _noaccess_prefix);
380 assert(CompressedOops::use_implicit_null_checks() == true, "not initialized?");
381 } else {
382 CompressedOops::set_use_implicit_null_checks(false);
383 }
384 }
385
386 _base += _noaccess_prefix;
387 _size -= _noaccess_prefix;
388 assert(((uintptr_t)_base % _alignment == 0), "must be exactly of required alignment");
389 }
390
391 // Tries to allocate memory of size 'size' at address requested_address with alignment 'alignment'.
392 // Does not check whether the reserved memory actually is at requested_address, as the memory returned
393 // might still fulfill the wishes of the caller.
394 // Assures the memory is aligned to 'alignment'.
395 // NOTE: If ReservedHeapSpace already points to some reserved memory this is freed, first.
396 void ReservedHeapSpace::try_reserve_heap(size_t size,
397 size_t alignment,
398 size_t page_size,
399 char* requested_address) {
400 if (_base != NULL) {
401 // We tried before, but we didn't like the address delivered.
402 release();
403 }
404
405 // Try to reserve the memory for the heap.
406 log_trace(gc, heap, coops)("Trying to allocate at address " PTR_FORMAT
407 " heap of size " SIZE_FORMAT_HEX,
408 p2i(requested_address),
409 size);
410
411 reserve(size, alignment, page_size, requested_address, false);
412
413 // Check alignment constraints.
414 if (is_reserved() && !is_aligned(_base, _alignment)) {
415 // Base not aligned, retry.
416 release();
417 }
418 }
419
420 void ReservedHeapSpace::try_reserve_range(char *highest_start,
421 char *lowest_start,
422 size_t attach_point_alignment,
423 char *aligned_heap_base_min_address,
424 char *upper_bound,
425 size_t size,
426 size_t alignment,
427 size_t page_size) {
428 const size_t attach_range = highest_start - lowest_start;
429 // Cap num_attempts at possible number.
430 // At least one is possible even for 0 sized attach range.
431 const uint64_t num_attempts_possible = (attach_range / attach_point_alignment) + 1;
432 const uint64_t num_attempts_to_try = MIN2((uint64_t)HeapSearchSteps, num_attempts_possible);
433
434 const size_t stepsize = (attach_range == 0) ? // Only one try.
435 (size_t) highest_start : align_up(attach_range / num_attempts_to_try, attach_point_alignment);
436
437 // Try attach points from top to bottom.
438 char* attach_point = highest_start;
439 while (attach_point >= lowest_start &&
440 attach_point <= highest_start && // Avoid wrap around.
441 ((_base == NULL) ||
442 (_base < aligned_heap_base_min_address || _base + size > upper_bound))) {
443 try_reserve_heap(size, alignment, page_size, attach_point);
444 attach_point -= stepsize;
445 }
446 }
447
448 #define SIZE_64K ((uint64_t) UCONST64( 0x10000))
449 #define SIZE_256M ((uint64_t) UCONST64( 0x10000000))
450 #define SIZE_32G ((uint64_t) UCONST64( 0x800000000))
451
452 // Helper for heap allocation. Returns an array with addresses
453 // (OS-specific) which are suited for disjoint base mode. Array is
454 // NULL terminated.
455 static char** get_attach_addresses_for_disjoint_mode() {
456 static uint64_t addresses[] = {
457 2 * SIZE_32G,
458 3 * SIZE_32G,
459 4 * SIZE_32G,
460 8 * SIZE_32G,
461 10 * SIZE_32G,
462 1 * SIZE_64K * SIZE_32G,
463 2 * SIZE_64K * SIZE_32G,
464 3 * SIZE_64K * SIZE_32G,
465 4 * SIZE_64K * SIZE_32G,
466 16 * SIZE_64K * SIZE_32G,
467 32 * SIZE_64K * SIZE_32G,
468 34 * SIZE_64K * SIZE_32G,
469 0
470 };
471
472 // Sort out addresses smaller than HeapBaseMinAddress. This assumes
473 // the array is sorted.
474 uint i = 0;
475 while (addresses[i] != 0 &&
476 (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) {
477 i++;
478 }
479 uint start = i;
480
481 // Avoid more steps than requested.
482 i = 0;
483 while (addresses[start+i] != 0) {
484 if (i == HeapSearchSteps) {
485 addresses[start+i] = 0;
486 break;
487 }
488 i++;
489 }
490
491 return (char**) &addresses[start];
492 }
493
494 void ReservedHeapSpace::initialize_compressed_heap(const size_t size, size_t alignment, size_t page_size) {
495 guarantee(size + noaccess_prefix_size(alignment) <= OopEncodingHeapMax,
496 "can not allocate compressed oop heap for this size");
497 guarantee(alignment == MAX2(alignment, (size_t)os::vm_page_size()), "alignment too small");
498
499 const size_t granularity = os::vm_allocation_granularity();
500 assert((size & (granularity - 1)) == 0,
501 "size not aligned to os::vm_allocation_granularity()");
502 assert((alignment & (granularity - 1)) == 0,
503 "alignment not aligned to os::vm_allocation_granularity()");
504 assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
505 "not a power of 2");
506
507 // The necessary attach point alignment for generated wish addresses.
508 // This is needed to increase the chance of attaching for mmap and shmat.
509 const size_t os_attach_point_alignment =
510 AIX_ONLY(SIZE_256M) // Known shm boundary alignment.
511 NOT_AIX(os::vm_allocation_granularity());
512 const size_t attach_point_alignment = lcm(alignment, os_attach_point_alignment);
513
514 char *aligned_heap_base_min_address = (char *)align_up((void *)HeapBaseMinAddress, alignment);
515 size_t noaccess_prefix = ((aligned_heap_base_min_address + size) > (char*)OopEncodingHeapMax) ?
516 noaccess_prefix_size(alignment) : 0;
517
518 // Attempt to alloc at user-given address.
519 if (!FLAG_IS_DEFAULT(HeapBaseMinAddress)) {
520 try_reserve_heap(size + noaccess_prefix, alignment, page_size, aligned_heap_base_min_address);
521 if (_base != aligned_heap_base_min_address) { // Enforce this exact address.
522 release();
523 }
524 }
525
526 // Keep heap at HeapBaseMinAddress.
527 if (_base == NULL) {
528
529 // Try to allocate the heap at addresses that allow efficient oop compression.
530 // Different schemes are tried, in order of decreasing optimization potential.
531 //
532 // For this, try_reserve_heap() is called with the desired heap base addresses.
533 // A call into the os layer to allocate at a given address can return memory
534 // at a different address than requested. Still, this might be memory at a useful
535 // address. try_reserve_heap() always returns this allocated memory, as only here
536 // the criteria for a good heap are checked.
537
538 // Attempt to allocate so that we can run without base and scale (32-Bit unscaled compressed oops).
539 // Give it several tries from top of range to bottom.
540 if (aligned_heap_base_min_address + size <= (char *)UnscaledOopHeapMax) {
541
542 // Calc address range within we try to attach (range of possible start addresses).
543 char* const highest_start = align_down((char *)UnscaledOopHeapMax - size, attach_point_alignment);
544 char* const lowest_start = align_up(aligned_heap_base_min_address, attach_point_alignment);
545 try_reserve_range(highest_start, lowest_start, attach_point_alignment,
546 aligned_heap_base_min_address, (char *)UnscaledOopHeapMax, size, alignment, page_size);
547 }
548
549 // zerobased: Attempt to allocate in the lower 32G.
550 // But leave room for the compressed class pointers, which is allocated above
551 // the heap.
552 // Note Lilliput: the advantages of this strategy were questionable before
553 // (since CDS=off + Compressed oops + heap large enough to suffocate us out of lower 32g
554 // is rare) and with Lilliput the encoding range drastically shrank. We may just do away
555 // with this altogether.
556 char *zerobased_max = (char *)OopEncodingHeapMax;
557 const size_t class_space = align_up(CompressedClassSpaceSize, alignment);
558 // For small heaps, save some space for compressed class pointer
559 // space so it can be decoded with no base.
560 if (UseCompressedClassPointers && !UseSharedSpaces &&
561 OopEncodingHeapMax <= KlassEncodingMetaspaceMax &&
562 (uint64_t)(aligned_heap_base_min_address + size + class_space) <= KlassEncodingMetaspaceMax) {
563 zerobased_max = (char *)OopEncodingHeapMax - class_space;
564 }
565
566 // Give it several tries from top of range to bottom.
567 if (aligned_heap_base_min_address + size <= zerobased_max && // Zerobased theoretical possible.
568 ((_base == NULL) || // No previous try succeeded.
569 (_base + size > zerobased_max))) { // Unscaled delivered an arbitrary address.
570
571 // Calc address range within we try to attach (range of possible start addresses).
572 char *const highest_start = align_down(zerobased_max - size, attach_point_alignment);
573 // Need to be careful about size being guaranteed to be less
574 // than UnscaledOopHeapMax due to type constraints.
575 char *lowest_start = aligned_heap_base_min_address;
576 uint64_t unscaled_end = UnscaledOopHeapMax - size;
577 if (unscaled_end < UnscaledOopHeapMax) { // unscaled_end wrapped if size is large
578 lowest_start = MAX2(lowest_start, (char*)unscaled_end);
579 }
580 lowest_start = align_up(lowest_start, attach_point_alignment);
581 try_reserve_range(highest_start, lowest_start, attach_point_alignment,
582 aligned_heap_base_min_address, zerobased_max, size, alignment, page_size);
583 }
584
585 // Now we go for heaps with base != 0. We need a noaccess prefix to efficiently
586 // implement null checks.
587 noaccess_prefix = noaccess_prefix_size(alignment);
588
589 // Try to attach at addresses that are aligned to OopEncodingHeapMax. Disjointbase mode.
590 char** addresses = get_attach_addresses_for_disjoint_mode();
591 int i = 0;
592 while (addresses[i] && // End of array not yet reached.
593 ((_base == NULL) || // No previous try succeeded.
594 (_base + size > (char *)OopEncodingHeapMax && // Not zerobased or unscaled address.
595 !CompressedOops::is_disjoint_heap_base_address((address)_base)))) { // Not disjoint address.
596 char* const attach_point = addresses[i];
597 assert(attach_point >= aligned_heap_base_min_address, "Flag support broken");
598 try_reserve_heap(size + noaccess_prefix, alignment, page_size, attach_point);
599 i++;
600 }
601
602 // Last, desperate try without any placement.
603 if (_base == NULL) {
604 log_trace(gc, heap, coops)("Trying to allocate at address NULL heap of size " SIZE_FORMAT_HEX, size + noaccess_prefix);
605 initialize(size + noaccess_prefix, alignment, page_size, NULL, false);
606 }
607 }
608 }
609
610 ReservedHeapSpace::ReservedHeapSpace(size_t size, size_t alignment, size_t page_size, const char* heap_allocation_directory) : ReservedSpace() {
611
612 if (size == 0) {
613 return;
614 }
615
616 if (heap_allocation_directory != NULL) {
617 _fd_for_heap = os::create_file_for_heap(heap_allocation_directory);
618 if (_fd_for_heap == -1) {
619 vm_exit_during_initialization(
620 err_msg("Could not create file for Heap at location %s", heap_allocation_directory));
621 }
622 // When there is a backing file directory for this space then whether
623 // large pages are allocated is up to the filesystem of the backing file.
624 // If requested, let the user know that explicit large pages can't be used.
625 if (use_explicit_large_pages(page_size) && large_pages_requested()) {
626 log_debug(gc, heap)("Cannot allocate explicit large pages for Java Heap when AllocateHeapAt option is set.");
627 }
628 }
629
630 // Heap size should be aligned to alignment, too.
631 guarantee(is_aligned(size, alignment), "set by caller");
632
633 if (UseCompressedOops) {
634 initialize_compressed_heap(size, alignment, page_size);
635 if (_size > size) {
636 // We allocated heap with noaccess prefix.
637 // It can happen we get a zerobased/unscaled heap with noaccess prefix,
638 // if we had to try at arbitrary address.
639 establish_noaccess_prefix();
640 }
641 } else {
642 initialize(size, alignment, page_size, NULL, false);
643 }
644
645 assert(markWord::encode_pointer_as_mark(_base).decode_pointer() == _base,
646 "area must be distinguishable from marks for mark-sweep");
647 assert(markWord::encode_pointer_as_mark(&_base[size]).decode_pointer() == &_base[size],
648 "area must be distinguishable from marks for mark-sweep");
649
650 if (base() != NULL) {
651 MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap);
652 }
653
654 if (_fd_for_heap != -1) {
655 ::close(_fd_for_heap);
656 }
657 }
658
659 MemRegion ReservedHeapSpace::region() const {
660 return MemRegion((HeapWord*)base(), (HeapWord*)end());
661 }
662
663 // Reserve space for code segment. Same as Java heap only we mark this as
664 // executable.
665 ReservedCodeSpace::ReservedCodeSpace(size_t r_size,
666 size_t rs_align,
667 size_t rs_page_size) : ReservedSpace() {
668 initialize(r_size, rs_align, rs_page_size, /*requested address*/ NULL, /*executable*/ true);
669 MemTracker::record_virtual_memory_type((address)base(), mtCode);
670 }
671
672 // VirtualSpace
673
674 VirtualSpace::VirtualSpace() {
675 _low_boundary = NULL;
676 _high_boundary = NULL;
677 _low = NULL;
678 _high = NULL;
679 _lower_high = NULL;
680 _middle_high = NULL;
681 _upper_high = NULL;
682 _lower_high_boundary = NULL;
683 _middle_high_boundary = NULL;
684 _upper_high_boundary = NULL;
685 _lower_alignment = 0;
686 _middle_alignment = 0;
687 _upper_alignment = 0;
688 _special = false;
689 _executable = false;
690 }
691
692
693 bool VirtualSpace::initialize(ReservedSpace rs, size_t committed_size) {
694 const size_t max_commit_granularity = os::page_size_for_region_unaligned(rs.size(), 1);
695 return initialize_with_granularity(rs, committed_size, max_commit_granularity);
696 }
697
698 bool VirtualSpace::initialize_with_granularity(ReservedSpace rs, size_t committed_size, size_t max_commit_granularity) {
699 if(!rs.is_reserved()) return false; // allocation failed.
700 assert(_low_boundary == NULL, "VirtualSpace already initialized");
701 assert(max_commit_granularity > 0, "Granularity must be non-zero.");
702
703 _low_boundary = rs.base();
704 _high_boundary = low_boundary() + rs.size();
705
706 _low = low_boundary();
707 _high = low();
708
709 _special = rs.special();
710 _executable = rs.executable();
711
712 // When a VirtualSpace begins life at a large size, make all future expansion
713 // and shrinking occur aligned to a granularity of large pages. This avoids
714 // fragmentation of physical addresses that inhibits the use of large pages
715 // by the OS virtual memory system. Empirically, we see that with a 4MB
716 // page size, the only spaces that get handled this way are codecache and
717 // the heap itself, both of which provide a substantial performance
718 // boost in many benchmarks when covered by large pages.
719 //
720 // No attempt is made to force large page alignment at the very top and
721 // bottom of the space if they are not aligned so already.
722 _lower_alignment = os::vm_page_size();
723 _middle_alignment = max_commit_granularity;
724 _upper_alignment = os::vm_page_size();
725
726 // End of each region
727 _lower_high_boundary = align_up(low_boundary(), middle_alignment());
728 _middle_high_boundary = align_down(high_boundary(), middle_alignment());
729 _upper_high_boundary = high_boundary();
730
731 // High address of each region
732 _lower_high = low_boundary();
733 _middle_high = lower_high_boundary();
734 _upper_high = middle_high_boundary();
735
736 // commit to initial size
737 if (committed_size > 0) {
738 if (!expand_by(committed_size)) {
739 return false;
740 }
741 }
742 return true;
743 }
744
745
746 VirtualSpace::~VirtualSpace() {
747 release();
748 }
749
750
751 void VirtualSpace::release() {
752 // This does not release memory it reserved.
753 // Caller must release via rs.release();
754 _low_boundary = NULL;
755 _high_boundary = NULL;
756 _low = NULL;
757 _high = NULL;
758 _lower_high = NULL;
759 _middle_high = NULL;
760 _upper_high = NULL;
761 _lower_high_boundary = NULL;
762 _middle_high_boundary = NULL;
763 _upper_high_boundary = NULL;
764 _lower_alignment = 0;
765 _middle_alignment = 0;
766 _upper_alignment = 0;
767 _special = false;
768 _executable = false;
769 }
770
771
772 size_t VirtualSpace::committed_size() const {
773 return pointer_delta(high(), low(), sizeof(char));
774 }
775
776
777 size_t VirtualSpace::reserved_size() const {
778 return pointer_delta(high_boundary(), low_boundary(), sizeof(char));
779 }
780
781
782 size_t VirtualSpace::uncommitted_size() const {
783 return reserved_size() - committed_size();
784 }
785
786 size_t VirtualSpace::actual_committed_size() const {
787 // Special VirtualSpaces commit all reserved space up front.
788 if (special()) {
789 return reserved_size();
790 }
791
792 size_t committed_low = pointer_delta(_lower_high, _low_boundary, sizeof(char));
793 size_t committed_middle = pointer_delta(_middle_high, _lower_high_boundary, sizeof(char));
794 size_t committed_high = pointer_delta(_upper_high, _middle_high_boundary, sizeof(char));
795
796 #ifdef ASSERT
797 size_t lower = pointer_delta(_lower_high_boundary, _low_boundary, sizeof(char));
798 size_t middle = pointer_delta(_middle_high_boundary, _lower_high_boundary, sizeof(char));
799 size_t upper = pointer_delta(_upper_high_boundary, _middle_high_boundary, sizeof(char));
800
801 if (committed_high > 0) {
802 assert(committed_low == lower, "Must be");
803 assert(committed_middle == middle, "Must be");
804 }
805
806 if (committed_middle > 0) {
807 assert(committed_low == lower, "Must be");
808 }
809 if (committed_middle < middle) {
810 assert(committed_high == 0, "Must be");
811 }
812
813 if (committed_low < lower) {
814 assert(committed_high == 0, "Must be");
815 assert(committed_middle == 0, "Must be");
816 }
817 #endif
818
819 return committed_low + committed_middle + committed_high;
820 }
821
822
823 bool VirtualSpace::contains(const void* p) const {
824 return low() <= (const char*) p && (const char*) p < high();
825 }
826
827 static void pretouch_expanded_memory(void* start, void* end) {
828 assert(is_aligned(start, os::vm_page_size()), "Unexpected alignment");
829 assert(is_aligned(end, os::vm_page_size()), "Unexpected alignment");
830
831 os::pretouch_memory(start, end);
832 }
833
834 static bool commit_expanded(char* start, size_t size, size_t alignment, bool pre_touch, bool executable) {
835 if (os::commit_memory(start, size, alignment, executable)) {
836 if (pre_touch || AlwaysPreTouch) {
837 pretouch_expanded_memory(start, start + size);
838 }
839 return true;
840 }
841
842 debug_only(warning(
843 "INFO: os::commit_memory(" PTR_FORMAT ", " PTR_FORMAT
844 " size=" SIZE_FORMAT ", executable=%d) failed",
845 p2i(start), p2i(start + size), size, executable);)
846
847 return false;
848 }
849
850 /*
851 First we need to determine if a particular virtual space is using large
852 pages. This is done at the initialize function and only virtual spaces
853 that are larger than LargePageSizeInBytes use large pages. Once we
854 have determined this, all expand_by and shrink_by calls must grow and
855 shrink by large page size chunks. If a particular request
856 is within the current large page, the call to commit and uncommit memory
857 can be ignored. In the case that the low and high boundaries of this
858 space is not large page aligned, the pages leading to the first large
859 page address and the pages after the last large page address must be
860 allocated with default pages.
861 */
862 bool VirtualSpace::expand_by(size_t bytes, bool pre_touch) {
863 if (uncommitted_size() < bytes) {
864 return false;
865 }
866
867 if (special()) {
868 // don't commit memory if the entire space is pinned in memory
869 _high += bytes;
870 return true;
871 }
872
873 char* previous_high = high();
874 char* unaligned_new_high = high() + bytes;
875 assert(unaligned_new_high <= high_boundary(), "cannot expand by more than upper boundary");
876
877 // Calculate where the new high for each of the regions should be. If
878 // the low_boundary() and high_boundary() are LargePageSizeInBytes aligned
879 // then the unaligned lower and upper new highs would be the
880 // lower_high() and upper_high() respectively.
881 char* unaligned_lower_new_high = MIN2(unaligned_new_high, lower_high_boundary());
882 char* unaligned_middle_new_high = MIN2(unaligned_new_high, middle_high_boundary());
883 char* unaligned_upper_new_high = MIN2(unaligned_new_high, upper_high_boundary());
884
885 // Align the new highs based on the regions alignment. lower and upper
886 // alignment will always be default page size. middle alignment will be
887 // LargePageSizeInBytes if the actual size of the virtual space is in
888 // fact larger than LargePageSizeInBytes.
889 char* aligned_lower_new_high = align_up(unaligned_lower_new_high, lower_alignment());
890 char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
891 char* aligned_upper_new_high = align_up(unaligned_upper_new_high, upper_alignment());
892
893 // Determine which regions need to grow in this expand_by call.
894 // If you are growing in the lower region, high() must be in that
895 // region so calculate the size based on high(). For the middle and
896 // upper regions, determine the starting point of growth based on the
897 // location of high(). By getting the MAX of the region's low address
898 // (or the previous region's high address) and high(), we can tell if it
899 // is an intra or inter region growth.
900 size_t lower_needs = 0;
901 if (aligned_lower_new_high > lower_high()) {
902 lower_needs = pointer_delta(aligned_lower_new_high, lower_high(), sizeof(char));
903 }
904 size_t middle_needs = 0;
905 if (aligned_middle_new_high > middle_high()) {
906 middle_needs = pointer_delta(aligned_middle_new_high, middle_high(), sizeof(char));
907 }
908 size_t upper_needs = 0;
909 if (aligned_upper_new_high > upper_high()) {
910 upper_needs = pointer_delta(aligned_upper_new_high, upper_high(), sizeof(char));
911 }
912
913 // Check contiguity.
914 assert(low_boundary() <= lower_high() && lower_high() <= lower_high_boundary(),
915 "high address must be contained within the region");
916 assert(lower_high_boundary() <= middle_high() && middle_high() <= middle_high_boundary(),
917 "high address must be contained within the region");
918 assert(middle_high_boundary() <= upper_high() && upper_high() <= upper_high_boundary(),
919 "high address must be contained within the region");
920
921 // Commit regions
922 if (lower_needs > 0) {
923 assert(lower_high() + lower_needs <= lower_high_boundary(), "must not expand beyond region");
924 if (!commit_expanded(lower_high(), lower_needs, _lower_alignment, pre_touch, _executable)) {
925 return false;
926 }
927 _lower_high += lower_needs;
928 }
929
930 if (middle_needs > 0) {
931 assert(middle_high() + middle_needs <= middle_high_boundary(), "must not expand beyond region");
932 if (!commit_expanded(middle_high(), middle_needs, _middle_alignment, pre_touch, _executable)) {
933 return false;
934 }
935 _middle_high += middle_needs;
936 }
937
938 if (upper_needs > 0) {
939 assert(upper_high() + upper_needs <= upper_high_boundary(), "must not expand beyond region");
940 if (!commit_expanded(upper_high(), upper_needs, _upper_alignment, pre_touch, _executable)) {
941 return false;
942 }
943 _upper_high += upper_needs;
944 }
945
946 _high += bytes;
947 return true;
948 }
949
950 // A page is uncommitted if the contents of the entire page is deemed unusable.
951 // Continue to decrement the high() pointer until it reaches a page boundary
952 // in which case that particular page can now be uncommitted.
953 void VirtualSpace::shrink_by(size_t size) {
954 if (committed_size() < size)
955 fatal("Cannot shrink virtual space to negative size");
956
957 if (special()) {
958 // don't uncommit if the entire space is pinned in memory
959 _high -= size;
960 return;
961 }
962
963 char* unaligned_new_high = high() - size;
964 assert(unaligned_new_high >= low_boundary(), "cannot shrink past lower boundary");
965
966 // Calculate new unaligned address
967 char* unaligned_upper_new_high =
968 MAX2(unaligned_new_high, middle_high_boundary());
969 char* unaligned_middle_new_high =
970 MAX2(unaligned_new_high, lower_high_boundary());
971 char* unaligned_lower_new_high =
972 MAX2(unaligned_new_high, low_boundary());
973
974 // Align address to region's alignment
975 char* aligned_upper_new_high = align_up(unaligned_upper_new_high, upper_alignment());
976 char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
977 char* aligned_lower_new_high = align_up(unaligned_lower_new_high, lower_alignment());
978
979 // Determine which regions need to shrink
980 size_t upper_needs = 0;
981 if (aligned_upper_new_high < upper_high()) {
982 upper_needs =
983 pointer_delta(upper_high(), aligned_upper_new_high, sizeof(char));
984 }
985 size_t middle_needs = 0;
986 if (aligned_middle_new_high < middle_high()) {
987 middle_needs =
988 pointer_delta(middle_high(), aligned_middle_new_high, sizeof(char));
989 }
990 size_t lower_needs = 0;
991 if (aligned_lower_new_high < lower_high()) {
992 lower_needs =
993 pointer_delta(lower_high(), aligned_lower_new_high, sizeof(char));
994 }
995
996 // Check contiguity.
997 assert(middle_high_boundary() <= upper_high() &&
998 upper_high() <= upper_high_boundary(),
999 "high address must be contained within the region");
1000 assert(lower_high_boundary() <= middle_high() &&
1001 middle_high() <= middle_high_boundary(),
1002 "high address must be contained within the region");
1003 assert(low_boundary() <= lower_high() &&
1004 lower_high() <= lower_high_boundary(),
1005 "high address must be contained within the region");
1006
1007 // Uncommit
1008 if (upper_needs > 0) {
1009 assert(middle_high_boundary() <= aligned_upper_new_high &&
1010 aligned_upper_new_high + upper_needs <= upper_high_boundary(),
1011 "must not shrink beyond region");
1012 if (!os::uncommit_memory(aligned_upper_new_high, upper_needs, _executable)) {
1013 debug_only(warning("os::uncommit_memory failed"));
1014 return;
1015 } else {
1016 _upper_high -= upper_needs;
1017 }
1018 }
1019 if (middle_needs > 0) {
1020 assert(lower_high_boundary() <= aligned_middle_new_high &&
1021 aligned_middle_new_high + middle_needs <= middle_high_boundary(),
1022 "must not shrink beyond region");
1023 if (!os::uncommit_memory(aligned_middle_new_high, middle_needs, _executable)) {
1024 debug_only(warning("os::uncommit_memory failed"));
1025 return;
1026 } else {
1027 _middle_high -= middle_needs;
1028 }
1029 }
1030 if (lower_needs > 0) {
1031 assert(low_boundary() <= aligned_lower_new_high &&
1032 aligned_lower_new_high + lower_needs <= lower_high_boundary(),
1033 "must not shrink beyond region");
1034 if (!os::uncommit_memory(aligned_lower_new_high, lower_needs, _executable)) {
1035 debug_only(warning("os::uncommit_memory failed"));
1036 return;
1037 } else {
1038 _lower_high -= lower_needs;
1039 }
1040 }
1041
1042 _high -= size;
1043 }
1044
1045 #ifndef PRODUCT
1046 void VirtualSpace::check_for_contiguity() {
1047 // Check contiguity.
1048 assert(low_boundary() <= lower_high() &&
1049 lower_high() <= lower_high_boundary(),
1050 "high address must be contained within the region");
1051 assert(lower_high_boundary() <= middle_high() &&
1052 middle_high() <= middle_high_boundary(),
1053 "high address must be contained within the region");
1054 assert(middle_high_boundary() <= upper_high() &&
1055 upper_high() <= upper_high_boundary(),
1056 "high address must be contained within the region");
1057 assert(low() >= low_boundary(), "low");
1058 assert(low_boundary() <= lower_high_boundary(), "lower high boundary");
1059 assert(upper_high_boundary() <= high_boundary(), "upper high boundary");
1060 assert(high() <= upper_high(), "upper high");
1061 }
1062
1063 void VirtualSpace::print_on(outputStream* out) const {
1064 out->print ("Virtual space:");
1065 if (special()) out->print(" (pinned in memory)");
1066 out->cr();
1067 out->print_cr(" - committed: " SIZE_FORMAT, committed_size());
1068 out->print_cr(" - reserved: " SIZE_FORMAT, reserved_size());
1069 out->print_cr(" - [low, high]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", p2i(low()), p2i(high()));
1070 out->print_cr(" - [low_b, high_b]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", p2i(low_boundary()), p2i(high_boundary()));
1071 }
1072
1073 void VirtualSpace::print() const {
1074 print_on(tty);
1075 }
1076
1077 #endif
--- EOF ---