1 /* 2 * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "logging/log.hpp" 27 #include "memory/resourceArea.hpp" 28 #include "memory/virtualspace.hpp" 29 #include "oops/compressedOops.hpp" 30 #include "oops/markWord.hpp" 31 #include "oops/oop.inline.hpp" 32 #include "runtime/globals_extension.hpp" 33 #include "runtime/java.hpp" 34 #include "runtime/os.hpp" 35 #include "services/memTracker.hpp" 36 #include "utilities/align.hpp" 37 #include "utilities/formatBuffer.hpp" 38 #include "utilities/powerOfTwo.hpp" 39 40 // ReservedSpace 41 42 // Dummy constructor 43 ReservedSpace::ReservedSpace() : _base(NULL), _size(0), _noaccess_prefix(0), 44 _alignment(0), _special(false), _fd_for_heap(-1), _executable(false) { 45 } 46 47 ReservedSpace::ReservedSpace(size_t size) : _fd_for_heap(-1) { 48 // Want to use large pages where possible. If the size is 49 // not large page aligned the mapping will be a mix of 50 // large and normal pages. 51 size_t page_size = os::page_size_for_region_unaligned(size, 1); 52 size_t alignment = os::vm_allocation_granularity(); 53 initialize(size, alignment, page_size, NULL, false); 54 } 55 56 ReservedSpace::ReservedSpace(size_t size, size_t preferred_page_size) : _fd_for_heap(-1) { 57 // When a page size is given we don't want to mix large 58 // and normal pages. If the size is not a multiple of the 59 // page size it will be aligned up to achieve this. 60 size_t alignment = os::vm_allocation_granularity();; 61 if (preferred_page_size != (size_t)os::vm_page_size()) { 62 alignment = MAX2(preferred_page_size, alignment); 63 size = align_up(size, alignment); 64 } 65 initialize(size, alignment, preferred_page_size, NULL, false); 66 } 67 68 ReservedSpace::ReservedSpace(size_t size, 69 size_t alignment, 70 size_t page_size, 71 char* requested_address) : _fd_for_heap(-1) { 72 initialize(size, alignment, page_size, requested_address, false); 73 } 74 75 ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment, size_t page_size, 76 bool special, bool executable) : _fd_for_heap(-1) { 77 assert((size % os::vm_allocation_granularity()) == 0, 78 "size not allocation aligned"); 79 initialize_members(base, size, alignment, page_size, special, executable); 80 } 81 82 // Helper method 83 static char* attempt_map_or_reserve_memory_at(char* base, size_t size, int fd, bool executable) { 84 if (fd != -1) { 85 return os::attempt_map_memory_to_file_at(base, size, fd); 86 } 87 return os::attempt_reserve_memory_at(base, size, executable); 88 } 89 90 // Helper method 91 static char* map_or_reserve_memory(size_t size, int fd, bool executable) { 92 if (fd != -1) { 93 return os::map_memory_to_file(size, fd); 94 } 95 return os::reserve_memory(size, executable); 96 } 97 98 // Helper method 99 static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int fd, bool executable) { 100 if (fd != -1) { 101 return os::map_memory_to_file_aligned(size, alignment, fd); 102 } 103 return os::reserve_memory_aligned(size, alignment, executable); 104 } 105 106 // Helper method 107 static void unmap_or_release_memory(char* base, size_t size, bool is_file_mapped) { 108 if (is_file_mapped) { 109 if (!os::unmap_memory(base, size)) { 110 fatal("os::unmap_memory failed"); 111 } 112 } else if (!os::release_memory(base, size)) { 113 fatal("os::release_memory failed"); 114 } 115 } 116 117 // Helper method 118 static bool failed_to_reserve_as_requested(char* base, char* requested_address) { 119 if (base == requested_address || requested_address == NULL) { 120 return false; // did not fail 121 } 122 123 if (base != NULL) { 124 // Different reserve address may be acceptable in other cases 125 // but for compressed oops heap should be at requested address. 126 assert(UseCompressedOops, "currently requested address used only for compressed oops"); 127 log_debug(gc, heap, coops)("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, p2i(base), p2i(requested_address)); 128 } 129 return true; 130 } 131 132 static bool use_explicit_large_pages(size_t page_size) { 133 return !os::can_commit_large_page_memory() && 134 page_size != (size_t) os::vm_page_size(); 135 } 136 137 static bool large_pages_requested() { 138 return UseLargePages && 139 (!FLAG_IS_DEFAULT(UseLargePages) || !FLAG_IS_DEFAULT(LargePageSizeInBytes)); 140 } 141 142 static void log_on_large_pages_failure(char* req_addr, size_t bytes) { 143 if (large_pages_requested()) { 144 // Compressed oops logging. 145 log_debug(gc, heap, coops)("Reserve regular memory without large pages"); 146 // JVM style warning that we did not succeed in using large pages. 147 char msg[128]; 148 jio_snprintf(msg, sizeof(msg), "Failed to reserve and commit memory using large pages. " 149 "req_addr: " PTR_FORMAT " bytes: " SIZE_FORMAT, 150 req_addr, bytes); 151 warning("%s", msg); 152 } 153 } 154 155 static char* reserve_memory(char* requested_address, const size_t size, 156 const size_t alignment, int fd, bool exec) { 157 char* base; 158 // If the memory was requested at a particular address, use 159 // os::attempt_reserve_memory_at() to avoid mapping over something 160 // important. If the reservation fails, return NULL. 161 if (requested_address != 0) { 162 assert(is_aligned(requested_address, alignment), 163 "Requested address " PTR_FORMAT " must be aligned to " SIZE_FORMAT, 164 p2i(requested_address), alignment); 165 base = attempt_map_or_reserve_memory_at(requested_address, size, fd, exec); 166 } else { 167 // Optimistically assume that the OS returns an aligned base pointer. 168 // When reserving a large address range, most OSes seem to align to at 169 // least 64K. 170 base = map_or_reserve_memory(size, fd, exec); 171 // Check alignment constraints. This is only needed when there is 172 // no requested address. 173 if (!is_aligned(base, alignment)) { 174 // Base not aligned, retry. 175 unmap_or_release_memory(base, size, fd != -1 /*is_file_mapped*/); 176 // Map using the requested alignment. 177 base = map_or_reserve_memory_aligned(size, alignment, fd, exec); 178 } 179 } 180 181 return base; 182 } 183 184 static char* reserve_memory_special(char* requested_address, const size_t size, 185 const size_t alignment, const size_t page_size, bool exec) { 186 187 log_trace(pagesize)("Attempt special mapping: size: " SIZE_FORMAT "%s, " 188 "alignment: " SIZE_FORMAT "%s", 189 byte_size_in_exact_unit(size), exact_unit_for_byte_size(size), 190 byte_size_in_exact_unit(alignment), exact_unit_for_byte_size(alignment)); 191 192 char* base = os::reserve_memory_special(size, alignment, page_size, requested_address, exec); 193 if (base != NULL) { 194 // Check alignment constraints. 195 assert(is_aligned(base, alignment), 196 "reserve_memory_special() returned an unaligned address, base: " PTR_FORMAT 197 " alignment: " SIZE_FORMAT_HEX, 198 p2i(base), alignment); 199 } 200 return base; 201 } 202 203 void ReservedSpace::clear_members() { 204 initialize_members(NULL, 0, 0, 0, false, false); 205 } 206 207 void ReservedSpace::initialize_members(char* base, size_t size, size_t alignment, 208 size_t page_size, bool special, bool executable) { 209 _base = base; 210 _size = size; 211 _alignment = alignment; 212 _page_size = page_size; 213 _special = special; 214 _executable = executable; 215 _noaccess_prefix = 0; 216 } 217 218 void ReservedSpace::reserve(size_t size, 219 size_t alignment, 220 size_t page_size, 221 char* requested_address, 222 bool executable) { 223 assert(is_aligned(size, alignment), "Size must be aligned to the requested alignment"); 224 225 // There are basically three different cases that we need to handle below: 226 // - Mapping backed by a file 227 // - Mapping backed by explicit large pages 228 // - Mapping backed by normal pages or transparent huge pages 229 // The first two have restrictions that requires the whole mapping to be 230 // committed up front. To record this the ReservedSpace is marked 'special'. 231 232 if (_fd_for_heap != -1) { 233 // When there is a backing file directory for this space then whether 234 // large pages are allocated is up to the filesystem of the backing file. 235 // So UseLargePages is not taken into account for this reservation. 236 char* base = reserve_memory(requested_address, size, alignment, _fd_for_heap, executable); 237 if (base != NULL) { 238 initialize_members(base, size, alignment, os::vm_page_size(), true, executable); 239 } 240 // Always return, not possible to fall back to reservation not using a file. 241 return; 242 } else if (use_explicit_large_pages(page_size)) { 243 // System can't commit large pages i.e. use transparent huge pages and 244 // the caller requested large pages. To satisfy this request we use 245 // explicit large pages and these have to be committed up front to ensure 246 // no reservations are lost. 247 size_t used_page_size = page_size; 248 char* base = NULL; 249 250 do { 251 base = reserve_memory_special(requested_address, size, alignment, used_page_size, executable); 252 if (base != NULL) { 253 break; 254 } 255 used_page_size = os::page_sizes().next_smaller(used_page_size); 256 } while (used_page_size > (size_t) os::vm_page_size()); 257 258 if (base != NULL) { 259 // Successful reservation using large pages. 260 initialize_members(base, size, alignment, used_page_size, true, executable); 261 return; 262 } 263 // Failed to reserve explicit large pages, do proper logging. 264 log_on_large_pages_failure(requested_address, size); 265 // Now fall back to normal reservation. 266 page_size = os::vm_page_size(); 267 } 268 269 // Not a 'special' reservation. 270 char* base = reserve_memory(requested_address, size, alignment, -1, executable); 271 if (base != NULL) { 272 // Successful mapping. 273 initialize_members(base, size, alignment, page_size, false, executable); 274 } 275 } 276 277 void ReservedSpace::initialize(size_t size, 278 size_t alignment, 279 size_t page_size, 280 char* requested_address, 281 bool executable) { 282 const size_t granularity = os::vm_allocation_granularity(); 283 assert((size & (granularity - 1)) == 0, 284 "size not aligned to os::vm_allocation_granularity()"); 285 assert((alignment & (granularity - 1)) == 0, 286 "alignment not aligned to os::vm_allocation_granularity()"); 287 assert(alignment == 0 || is_power_of_2((intptr_t)alignment), 288 "not a power of 2"); 289 assert(page_size >= (size_t) os::vm_page_size(), "Invalid page size"); 290 assert(is_power_of_2(page_size), "Invalid page size"); 291 292 clear_members(); 293 294 if (size == 0) { 295 return; 296 } 297 298 // Adjust alignment to not be 0. 299 alignment = MAX2(alignment, (size_t)os::vm_page_size()); 300 301 // Reserve the memory. 302 reserve(size, alignment, page_size, requested_address, executable); 303 304 // Check that the requested address is used if given. 305 if (failed_to_reserve_as_requested(_base, requested_address)) { 306 // OS ignored the requested address, release the reservation. 307 release(); 308 return; 309 } 310 } 311 312 ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment) { 313 assert(partition_size <= size(), "partition failed"); 314 ReservedSpace result(base(), partition_size, alignment, page_size(), special(), executable()); 315 return result; 316 } 317 318 319 ReservedSpace 320 ReservedSpace::last_part(size_t partition_size, size_t alignment) { 321 assert(partition_size <= size(), "partition failed"); 322 ReservedSpace result(base() + partition_size, size() - partition_size, 323 alignment, page_size(), special(), executable()); 324 return result; 325 } 326 327 328 size_t ReservedSpace::page_align_size_up(size_t size) { 329 return align_up(size, os::vm_page_size()); 330 } 331 332 333 size_t ReservedSpace::page_align_size_down(size_t size) { 334 return align_down(size, os::vm_page_size()); 335 } 336 337 338 size_t ReservedSpace::allocation_align_size_up(size_t size) { 339 return align_up(size, os::vm_allocation_granularity()); 340 } 341 342 void ReservedSpace::release() { 343 if (is_reserved()) { 344 char *real_base = _base - _noaccess_prefix; 345 const size_t real_size = _size + _noaccess_prefix; 346 if (special()) { 347 if (_fd_for_heap != -1) { 348 os::unmap_memory(real_base, real_size); 349 } else { 350 os::release_memory_special(real_base, real_size); 351 } 352 } else{ 353 os::release_memory(real_base, real_size); 354 } 355 clear_members(); 356 } 357 } 358 359 static size_t noaccess_prefix_size(size_t alignment) { 360 return lcm(os::vm_page_size(), alignment); 361 } 362 363 void ReservedHeapSpace::establish_noaccess_prefix() { 364 assert(_alignment >= (size_t)os::vm_page_size(), "must be at least page size big"); 365 _noaccess_prefix = noaccess_prefix_size(_alignment); 366 367 if (base() && base() + _size > (char *)OopEncodingHeapMax) { 368 if (true 369 WIN64_ONLY(&& !UseLargePages) 370 AIX_ONLY(&& os::vm_page_size() != 64*K)) { 371 // Protect memory at the base of the allocated region. 372 // If special, the page was committed (only matters on windows) 373 if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, _special)) { 374 fatal("cannot protect protection page"); 375 } 376 log_debug(gc, heap, coops)("Protected page at the reserved heap base: " 377 PTR_FORMAT " / " INTX_FORMAT " bytes", 378 p2i(_base), 379 _noaccess_prefix); 380 assert(CompressedOops::use_implicit_null_checks() == true, "not initialized?"); 381 } else { 382 CompressedOops::set_use_implicit_null_checks(false); 383 } 384 } 385 386 _base += _noaccess_prefix; 387 _size -= _noaccess_prefix; 388 assert(((uintptr_t)_base % _alignment == 0), "must be exactly of required alignment"); 389 } 390 391 // Tries to allocate memory of size 'size' at address requested_address with alignment 'alignment'. 392 // Does not check whether the reserved memory actually is at requested_address, as the memory returned 393 // might still fulfill the wishes of the caller. 394 // Assures the memory is aligned to 'alignment'. 395 // NOTE: If ReservedHeapSpace already points to some reserved memory this is freed, first. 396 void ReservedHeapSpace::try_reserve_heap(size_t size, 397 size_t alignment, 398 size_t page_size, 399 char* requested_address) { 400 if (_base != NULL) { 401 // We tried before, but we didn't like the address delivered. 402 release(); 403 } 404 405 // Try to reserve the memory for the heap. 406 log_trace(gc, heap, coops)("Trying to allocate at address " PTR_FORMAT 407 " heap of size " SIZE_FORMAT_HEX, 408 p2i(requested_address), 409 size); 410 411 reserve(size, alignment, page_size, requested_address, false); 412 413 // Check alignment constraints. 414 if (is_reserved() && !is_aligned(_base, _alignment)) { 415 // Base not aligned, retry. 416 release(); 417 } 418 } 419 420 void ReservedHeapSpace::try_reserve_range(char *highest_start, 421 char *lowest_start, 422 size_t attach_point_alignment, 423 char *aligned_heap_base_min_address, 424 char *upper_bound, 425 size_t size, 426 size_t alignment, 427 size_t page_size) { 428 const size_t attach_range = highest_start - lowest_start; 429 // Cap num_attempts at possible number. 430 // At least one is possible even for 0 sized attach range. 431 const uint64_t num_attempts_possible = (attach_range / attach_point_alignment) + 1; 432 const uint64_t num_attempts_to_try = MIN2((uint64_t)HeapSearchSteps, num_attempts_possible); 433 434 const size_t stepsize = (attach_range == 0) ? // Only one try. 435 (size_t) highest_start : align_up(attach_range / num_attempts_to_try, attach_point_alignment); 436 437 // Try attach points from top to bottom. 438 char* attach_point = highest_start; 439 while (attach_point >= lowest_start && 440 attach_point <= highest_start && // Avoid wrap around. 441 ((_base == NULL) || 442 (_base < aligned_heap_base_min_address || _base + size > upper_bound))) { 443 try_reserve_heap(size, alignment, page_size, attach_point); 444 attach_point -= stepsize; 445 } 446 } 447 448 #define SIZE_64K ((uint64_t) UCONST64( 0x10000)) 449 #define SIZE_256M ((uint64_t) UCONST64( 0x10000000)) 450 #define SIZE_32G ((uint64_t) UCONST64( 0x800000000)) 451 452 // Helper for heap allocation. Returns an array with addresses 453 // (OS-specific) which are suited for disjoint base mode. Array is 454 // NULL terminated. 455 static char** get_attach_addresses_for_disjoint_mode() { 456 static uint64_t addresses[] = { 457 2 * SIZE_32G, 458 3 * SIZE_32G, 459 4 * SIZE_32G, 460 8 * SIZE_32G, 461 10 * SIZE_32G, 462 1 * SIZE_64K * SIZE_32G, 463 2 * SIZE_64K * SIZE_32G, 464 3 * SIZE_64K * SIZE_32G, 465 4 * SIZE_64K * SIZE_32G, 466 16 * SIZE_64K * SIZE_32G, 467 32 * SIZE_64K * SIZE_32G, 468 34 * SIZE_64K * SIZE_32G, 469 0 470 }; 471 472 // Sort out addresses smaller than HeapBaseMinAddress. This assumes 473 // the array is sorted. 474 uint i = 0; 475 while (addresses[i] != 0 && 476 (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) { 477 i++; 478 } 479 uint start = i; 480 481 // Avoid more steps than requested. 482 i = 0; 483 while (addresses[start+i] != 0) { 484 if (i == HeapSearchSteps) { 485 addresses[start+i] = 0; 486 break; 487 } 488 i++; 489 } 490 491 return (char**) &addresses[start]; 492 } 493 494 void ReservedHeapSpace::initialize_compressed_heap(const size_t size, size_t alignment, size_t page_size) { 495 guarantee(size + noaccess_prefix_size(alignment) <= OopEncodingHeapMax, 496 "can not allocate compressed oop heap for this size"); 497 guarantee(alignment == MAX2(alignment, (size_t)os::vm_page_size()), "alignment too small"); 498 499 const size_t granularity = os::vm_allocation_granularity(); 500 assert((size & (granularity - 1)) == 0, 501 "size not aligned to os::vm_allocation_granularity()"); 502 assert((alignment & (granularity - 1)) == 0, 503 "alignment not aligned to os::vm_allocation_granularity()"); 504 assert(alignment == 0 || is_power_of_2((intptr_t)alignment), 505 "not a power of 2"); 506 507 // The necessary attach point alignment for generated wish addresses. 508 // This is needed to increase the chance of attaching for mmap and shmat. 509 const size_t os_attach_point_alignment = 510 AIX_ONLY(SIZE_256M) // Known shm boundary alignment. 511 NOT_AIX(os::vm_allocation_granularity()); 512 const size_t attach_point_alignment = lcm(alignment, os_attach_point_alignment); 513 514 char *aligned_heap_base_min_address = (char *)align_up((void *)HeapBaseMinAddress, alignment); 515 size_t noaccess_prefix = ((aligned_heap_base_min_address + size) > (char*)OopEncodingHeapMax) ? 516 noaccess_prefix_size(alignment) : 0; 517 518 // Attempt to alloc at user-given address. 519 if (!FLAG_IS_DEFAULT(HeapBaseMinAddress)) { 520 try_reserve_heap(size + noaccess_prefix, alignment, page_size, aligned_heap_base_min_address); 521 if (_base != aligned_heap_base_min_address) { // Enforce this exact address. 522 release(); 523 } 524 } 525 526 // Keep heap at HeapBaseMinAddress. 527 if (_base == NULL) { 528 529 // Try to allocate the heap at addresses that allow efficient oop compression. 530 // Different schemes are tried, in order of decreasing optimization potential. 531 // 532 // For this, try_reserve_heap() is called with the desired heap base addresses. 533 // A call into the os layer to allocate at a given address can return memory 534 // at a different address than requested. Still, this might be memory at a useful 535 // address. try_reserve_heap() always returns this allocated memory, as only here 536 // the criteria for a good heap are checked. 537 538 // Attempt to allocate so that we can run without base and scale (32-Bit unscaled compressed oops). 539 // Give it several tries from top of range to bottom. 540 if (aligned_heap_base_min_address + size <= (char *)UnscaledOopHeapMax) { 541 542 // Calc address range within we try to attach (range of possible start addresses). 543 char* const highest_start = align_down((char *)UnscaledOopHeapMax - size, attach_point_alignment); 544 char* const lowest_start = align_up(aligned_heap_base_min_address, attach_point_alignment); 545 try_reserve_range(highest_start, lowest_start, attach_point_alignment, 546 aligned_heap_base_min_address, (char *)UnscaledOopHeapMax, size, alignment, page_size); 547 } 548 549 // zerobased: Attempt to allocate in the lower 32G. 550 // But leave room for the compressed class pointers, which is allocated above 551 // the heap. 552 char *zerobased_max = (char *)OopEncodingHeapMax; 553 const size_t class_space = align_up(CompressedClassSpaceSize, alignment); 554 // For small heaps, save some space for compressed class pointer 555 // space so it can be decoded with no base. 556 if (UseCompressedClassPointers && !UseSharedSpaces && 557 OopEncodingHeapMax <= KlassEncodingMetaspaceMax && 558 (uint64_t)(aligned_heap_base_min_address + size + class_space) <= KlassEncodingMetaspaceMax) { 559 zerobased_max = (char *)OopEncodingHeapMax - class_space; 560 } 561 562 // Give it several tries from top of range to bottom. 563 if (aligned_heap_base_min_address + size <= zerobased_max && // Zerobased theoretical possible. 564 ((_base == NULL) || // No previous try succeeded. 565 (_base + size > zerobased_max))) { // Unscaled delivered an arbitrary address. 566 567 // Calc address range within we try to attach (range of possible start addresses). 568 char *const highest_start = align_down(zerobased_max - size, attach_point_alignment); 569 // Need to be careful about size being guaranteed to be less 570 // than UnscaledOopHeapMax due to type constraints. 571 char *lowest_start = aligned_heap_base_min_address; 572 uint64_t unscaled_end = UnscaledOopHeapMax - size; 573 if (unscaled_end < UnscaledOopHeapMax) { // unscaled_end wrapped if size is large 574 lowest_start = MAX2(lowest_start, (char*)unscaled_end); 575 } 576 lowest_start = align_up(lowest_start, attach_point_alignment); 577 try_reserve_range(highest_start, lowest_start, attach_point_alignment, 578 aligned_heap_base_min_address, zerobased_max, size, alignment, page_size); 579 } 580 581 // Now we go for heaps with base != 0. We need a noaccess prefix to efficiently 582 // implement null checks. 583 noaccess_prefix = noaccess_prefix_size(alignment); 584 585 // Try to attach at addresses that are aligned to OopEncodingHeapMax. Disjointbase mode. 586 char** addresses = get_attach_addresses_for_disjoint_mode(); 587 int i = 0; 588 while (addresses[i] && // End of array not yet reached. 589 ((_base == NULL) || // No previous try succeeded. 590 (_base + size > (char *)OopEncodingHeapMax && // Not zerobased or unscaled address. 591 !CompressedOops::is_disjoint_heap_base_address((address)_base)))) { // Not disjoint address. 592 char* const attach_point = addresses[i]; 593 assert(attach_point >= aligned_heap_base_min_address, "Flag support broken"); 594 try_reserve_heap(size + noaccess_prefix, alignment, page_size, attach_point); 595 i++; 596 } 597 598 // Last, desperate try without any placement. 599 if (_base == NULL) { 600 log_trace(gc, heap, coops)("Trying to allocate at address NULL heap of size " SIZE_FORMAT_HEX, size + noaccess_prefix); 601 initialize(size + noaccess_prefix, alignment, page_size, NULL, false); 602 } 603 } 604 } 605 606 ReservedHeapSpace::ReservedHeapSpace(size_t size, size_t alignment, size_t page_size, const char* heap_allocation_directory) : ReservedSpace() { 607 608 if (size == 0) { 609 return; 610 } 611 612 if (heap_allocation_directory != NULL) { 613 _fd_for_heap = os::create_file_for_heap(heap_allocation_directory); 614 if (_fd_for_heap == -1) { 615 vm_exit_during_initialization( 616 err_msg("Could not create file for Heap at location %s", heap_allocation_directory)); 617 } 618 // When there is a backing file directory for this space then whether 619 // large pages are allocated is up to the filesystem of the backing file. 620 // If requested, let the user know that explicit large pages can't be used. 621 if (use_explicit_large_pages(page_size) && large_pages_requested()) { 622 log_debug(gc, heap)("Cannot allocate explicit large pages for Java Heap when AllocateHeapAt option is set."); 623 } 624 } 625 626 // Heap size should be aligned to alignment, too. 627 guarantee(is_aligned(size, alignment), "set by caller"); 628 629 if (UseCompressedOops) { 630 initialize_compressed_heap(size, alignment, page_size); 631 if (_size > size) { 632 // We allocated heap with noaccess prefix. 633 // It can happen we get a zerobased/unscaled heap with noaccess prefix, 634 // if we had to try at arbitrary address. 635 establish_noaccess_prefix(); 636 } 637 } else { 638 initialize(size, alignment, page_size, NULL, false); 639 } 640 641 assert(markWord::encode_pointer_as_mark(_base).decode_pointer() == _base, 642 "area must be distinguishable from marks for mark-sweep"); 643 assert(markWord::encode_pointer_as_mark(&_base[size]).decode_pointer() == &_base[size], 644 "area must be distinguishable from marks for mark-sweep"); 645 646 if (base() != NULL) { 647 MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap); 648 } 649 650 if (_fd_for_heap != -1) { 651 ::close(_fd_for_heap); 652 } 653 } 654 655 MemRegion ReservedHeapSpace::region() const { 656 return MemRegion((HeapWord*)base(), (HeapWord*)end()); 657 } 658 659 // Reserve space for code segment. Same as Java heap only we mark this as 660 // executable. 661 ReservedCodeSpace::ReservedCodeSpace(size_t r_size, 662 size_t rs_align, 663 size_t rs_page_size) : ReservedSpace() { 664 initialize(r_size, rs_align, rs_page_size, /*requested address*/ NULL, /*executable*/ true); 665 MemTracker::record_virtual_memory_type((address)base(), mtCode); 666 } 667 668 // VirtualSpace 669 670 VirtualSpace::VirtualSpace() { 671 _low_boundary = NULL; 672 _high_boundary = NULL; 673 _low = NULL; 674 _high = NULL; 675 _lower_high = NULL; 676 _middle_high = NULL; 677 _upper_high = NULL; 678 _lower_high_boundary = NULL; 679 _middle_high_boundary = NULL; 680 _upper_high_boundary = NULL; 681 _lower_alignment = 0; 682 _middle_alignment = 0; 683 _upper_alignment = 0; 684 _special = false; 685 _executable = false; 686 } 687 688 689 bool VirtualSpace::initialize(ReservedSpace rs, size_t committed_size) { 690 const size_t max_commit_granularity = os::page_size_for_region_unaligned(rs.size(), 1); 691 return initialize_with_granularity(rs, committed_size, max_commit_granularity); 692 } 693 694 bool VirtualSpace::initialize_with_granularity(ReservedSpace rs, size_t committed_size, size_t max_commit_granularity) { 695 if(!rs.is_reserved()) return false; // allocation failed. 696 assert(_low_boundary == NULL, "VirtualSpace already initialized"); 697 assert(max_commit_granularity > 0, "Granularity must be non-zero."); 698 699 _low_boundary = rs.base(); 700 _high_boundary = low_boundary() + rs.size(); 701 702 _low = low_boundary(); 703 _high = low(); 704 705 _special = rs.special(); 706 _executable = rs.executable(); 707 708 // When a VirtualSpace begins life at a large size, make all future expansion 709 // and shrinking occur aligned to a granularity of large pages. This avoids 710 // fragmentation of physical addresses that inhibits the use of large pages 711 // by the OS virtual memory system. Empirically, we see that with a 4MB 712 // page size, the only spaces that get handled this way are codecache and 713 // the heap itself, both of which provide a substantial performance 714 // boost in many benchmarks when covered by large pages. 715 // 716 // No attempt is made to force large page alignment at the very top and 717 // bottom of the space if they are not aligned so already. 718 _lower_alignment = os::vm_page_size(); 719 _middle_alignment = max_commit_granularity; 720 _upper_alignment = os::vm_page_size(); 721 722 // End of each region 723 _lower_high_boundary = align_up(low_boundary(), middle_alignment()); 724 _middle_high_boundary = align_down(high_boundary(), middle_alignment()); 725 _upper_high_boundary = high_boundary(); 726 727 // High address of each region 728 _lower_high = low_boundary(); 729 _middle_high = lower_high_boundary(); 730 _upper_high = middle_high_boundary(); 731 732 // commit to initial size 733 if (committed_size > 0) { 734 if (!expand_by(committed_size)) { 735 return false; 736 } 737 } 738 return true; 739 } 740 741 742 VirtualSpace::~VirtualSpace() { 743 release(); 744 } 745 746 747 void VirtualSpace::release() { 748 // This does not release memory it reserved. 749 // Caller must release via rs.release(); 750 _low_boundary = NULL; 751 _high_boundary = NULL; 752 _low = NULL; 753 _high = NULL; 754 _lower_high = NULL; 755 _middle_high = NULL; 756 _upper_high = NULL; 757 _lower_high_boundary = NULL; 758 _middle_high_boundary = NULL; 759 _upper_high_boundary = NULL; 760 _lower_alignment = 0; 761 _middle_alignment = 0; 762 _upper_alignment = 0; 763 _special = false; 764 _executable = false; 765 } 766 767 768 size_t VirtualSpace::committed_size() const { 769 return pointer_delta(high(), low(), sizeof(char)); 770 } 771 772 773 size_t VirtualSpace::reserved_size() const { 774 return pointer_delta(high_boundary(), low_boundary(), sizeof(char)); 775 } 776 777 778 size_t VirtualSpace::uncommitted_size() const { 779 return reserved_size() - committed_size(); 780 } 781 782 size_t VirtualSpace::actual_committed_size() const { 783 // Special VirtualSpaces commit all reserved space up front. 784 if (special()) { 785 return reserved_size(); 786 } 787 788 size_t committed_low = pointer_delta(_lower_high, _low_boundary, sizeof(char)); 789 size_t committed_middle = pointer_delta(_middle_high, _lower_high_boundary, sizeof(char)); 790 size_t committed_high = pointer_delta(_upper_high, _middle_high_boundary, sizeof(char)); 791 792 #ifdef ASSERT 793 size_t lower = pointer_delta(_lower_high_boundary, _low_boundary, sizeof(char)); 794 size_t middle = pointer_delta(_middle_high_boundary, _lower_high_boundary, sizeof(char)); 795 size_t upper = pointer_delta(_upper_high_boundary, _middle_high_boundary, sizeof(char)); 796 797 if (committed_high > 0) { 798 assert(committed_low == lower, "Must be"); 799 assert(committed_middle == middle, "Must be"); 800 } 801 802 if (committed_middle > 0) { 803 assert(committed_low == lower, "Must be"); 804 } 805 if (committed_middle < middle) { 806 assert(committed_high == 0, "Must be"); 807 } 808 809 if (committed_low < lower) { 810 assert(committed_high == 0, "Must be"); 811 assert(committed_middle == 0, "Must be"); 812 } 813 #endif 814 815 return committed_low + committed_middle + committed_high; 816 } 817 818 819 bool VirtualSpace::contains(const void* p) const { 820 return low() <= (const char*) p && (const char*) p < high(); 821 } 822 823 static void pretouch_expanded_memory(void* start, void* end) { 824 assert(is_aligned(start, os::vm_page_size()), "Unexpected alignment"); 825 assert(is_aligned(end, os::vm_page_size()), "Unexpected alignment"); 826 827 os::pretouch_memory(start, end); 828 } 829 830 static bool commit_expanded(char* start, size_t size, size_t alignment, bool pre_touch, bool executable) { 831 if (os::commit_memory(start, size, alignment, executable)) { 832 if (pre_touch || AlwaysPreTouch) { 833 pretouch_expanded_memory(start, start + size); 834 } 835 return true; 836 } 837 838 debug_only(warning( 839 "INFO: os::commit_memory(" PTR_FORMAT ", " PTR_FORMAT 840 " size=" SIZE_FORMAT ", executable=%d) failed", 841 p2i(start), p2i(start + size), size, executable);) 842 843 return false; 844 } 845 846 /* 847 First we need to determine if a particular virtual space is using large 848 pages. This is done at the initialize function and only virtual spaces 849 that are larger than LargePageSizeInBytes use large pages. Once we 850 have determined this, all expand_by and shrink_by calls must grow and 851 shrink by large page size chunks. If a particular request 852 is within the current large page, the call to commit and uncommit memory 853 can be ignored. In the case that the low and high boundaries of this 854 space is not large page aligned, the pages leading to the first large 855 page address and the pages after the last large page address must be 856 allocated with default pages. 857 */ 858 bool VirtualSpace::expand_by(size_t bytes, bool pre_touch) { 859 if (uncommitted_size() < bytes) { 860 return false; 861 } 862 863 if (special()) { 864 // don't commit memory if the entire space is pinned in memory 865 _high += bytes; 866 return true; 867 } 868 869 char* previous_high = high(); 870 char* unaligned_new_high = high() + bytes; 871 assert(unaligned_new_high <= high_boundary(), "cannot expand by more than upper boundary"); 872 873 // Calculate where the new high for each of the regions should be. If 874 // the low_boundary() and high_boundary() are LargePageSizeInBytes aligned 875 // then the unaligned lower and upper new highs would be the 876 // lower_high() and upper_high() respectively. 877 char* unaligned_lower_new_high = MIN2(unaligned_new_high, lower_high_boundary()); 878 char* unaligned_middle_new_high = MIN2(unaligned_new_high, middle_high_boundary()); 879 char* unaligned_upper_new_high = MIN2(unaligned_new_high, upper_high_boundary()); 880 881 // Align the new highs based on the regions alignment. lower and upper 882 // alignment will always be default page size. middle alignment will be 883 // LargePageSizeInBytes if the actual size of the virtual space is in 884 // fact larger than LargePageSizeInBytes. 885 char* aligned_lower_new_high = align_up(unaligned_lower_new_high, lower_alignment()); 886 char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment()); 887 char* aligned_upper_new_high = align_up(unaligned_upper_new_high, upper_alignment()); 888 889 // Determine which regions need to grow in this expand_by call. 890 // If you are growing in the lower region, high() must be in that 891 // region so calculate the size based on high(). For the middle and 892 // upper regions, determine the starting point of growth based on the 893 // location of high(). By getting the MAX of the region's low address 894 // (or the previous region's high address) and high(), we can tell if it 895 // is an intra or inter region growth. 896 size_t lower_needs = 0; 897 if (aligned_lower_new_high > lower_high()) { 898 lower_needs = pointer_delta(aligned_lower_new_high, lower_high(), sizeof(char)); 899 } 900 size_t middle_needs = 0; 901 if (aligned_middle_new_high > middle_high()) { 902 middle_needs = pointer_delta(aligned_middle_new_high, middle_high(), sizeof(char)); 903 } 904 size_t upper_needs = 0; 905 if (aligned_upper_new_high > upper_high()) { 906 upper_needs = pointer_delta(aligned_upper_new_high, upper_high(), sizeof(char)); 907 } 908 909 // Check contiguity. 910 assert(low_boundary() <= lower_high() && lower_high() <= lower_high_boundary(), 911 "high address must be contained within the region"); 912 assert(lower_high_boundary() <= middle_high() && middle_high() <= middle_high_boundary(), 913 "high address must be contained within the region"); 914 assert(middle_high_boundary() <= upper_high() && upper_high() <= upper_high_boundary(), 915 "high address must be contained within the region"); 916 917 // Commit regions 918 if (lower_needs > 0) { 919 assert(lower_high() + lower_needs <= lower_high_boundary(), "must not expand beyond region"); 920 if (!commit_expanded(lower_high(), lower_needs, _lower_alignment, pre_touch, _executable)) { 921 return false; 922 } 923 _lower_high += lower_needs; 924 } 925 926 if (middle_needs > 0) { 927 assert(middle_high() + middle_needs <= middle_high_boundary(), "must not expand beyond region"); 928 if (!commit_expanded(middle_high(), middle_needs, _middle_alignment, pre_touch, _executable)) { 929 return false; 930 } 931 _middle_high += middle_needs; 932 } 933 934 if (upper_needs > 0) { 935 assert(upper_high() + upper_needs <= upper_high_boundary(), "must not expand beyond region"); 936 if (!commit_expanded(upper_high(), upper_needs, _upper_alignment, pre_touch, _executable)) { 937 return false; 938 } 939 _upper_high += upper_needs; 940 } 941 942 _high += bytes; 943 return true; 944 } 945 946 // A page is uncommitted if the contents of the entire page is deemed unusable. 947 // Continue to decrement the high() pointer until it reaches a page boundary 948 // in which case that particular page can now be uncommitted. 949 void VirtualSpace::shrink_by(size_t size) { 950 if (committed_size() < size) 951 fatal("Cannot shrink virtual space to negative size"); 952 953 if (special()) { 954 // don't uncommit if the entire space is pinned in memory 955 _high -= size; 956 return; 957 } 958 959 char* unaligned_new_high = high() - size; 960 assert(unaligned_new_high >= low_boundary(), "cannot shrink past lower boundary"); 961 962 // Calculate new unaligned address 963 char* unaligned_upper_new_high = 964 MAX2(unaligned_new_high, middle_high_boundary()); 965 char* unaligned_middle_new_high = 966 MAX2(unaligned_new_high, lower_high_boundary()); 967 char* unaligned_lower_new_high = 968 MAX2(unaligned_new_high, low_boundary()); 969 970 // Align address to region's alignment 971 char* aligned_upper_new_high = align_up(unaligned_upper_new_high, upper_alignment()); 972 char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment()); 973 char* aligned_lower_new_high = align_up(unaligned_lower_new_high, lower_alignment()); 974 975 // Determine which regions need to shrink 976 size_t upper_needs = 0; 977 if (aligned_upper_new_high < upper_high()) { 978 upper_needs = 979 pointer_delta(upper_high(), aligned_upper_new_high, sizeof(char)); 980 } 981 size_t middle_needs = 0; 982 if (aligned_middle_new_high < middle_high()) { 983 middle_needs = 984 pointer_delta(middle_high(), aligned_middle_new_high, sizeof(char)); 985 } 986 size_t lower_needs = 0; 987 if (aligned_lower_new_high < lower_high()) { 988 lower_needs = 989 pointer_delta(lower_high(), aligned_lower_new_high, sizeof(char)); 990 } 991 992 // Check contiguity. 993 assert(middle_high_boundary() <= upper_high() && 994 upper_high() <= upper_high_boundary(), 995 "high address must be contained within the region"); 996 assert(lower_high_boundary() <= middle_high() && 997 middle_high() <= middle_high_boundary(), 998 "high address must be contained within the region"); 999 assert(low_boundary() <= lower_high() && 1000 lower_high() <= lower_high_boundary(), 1001 "high address must be contained within the region"); 1002 1003 // Uncommit 1004 if (upper_needs > 0) { 1005 assert(middle_high_boundary() <= aligned_upper_new_high && 1006 aligned_upper_new_high + upper_needs <= upper_high_boundary(), 1007 "must not shrink beyond region"); 1008 if (!os::uncommit_memory(aligned_upper_new_high, upper_needs, _executable)) { 1009 debug_only(warning("os::uncommit_memory failed")); 1010 return; 1011 } else { 1012 _upper_high -= upper_needs; 1013 } 1014 } 1015 if (middle_needs > 0) { 1016 assert(lower_high_boundary() <= aligned_middle_new_high && 1017 aligned_middle_new_high + middle_needs <= middle_high_boundary(), 1018 "must not shrink beyond region"); 1019 if (!os::uncommit_memory(aligned_middle_new_high, middle_needs, _executable)) { 1020 debug_only(warning("os::uncommit_memory failed")); 1021 return; 1022 } else { 1023 _middle_high -= middle_needs; 1024 } 1025 } 1026 if (lower_needs > 0) { 1027 assert(low_boundary() <= aligned_lower_new_high && 1028 aligned_lower_new_high + lower_needs <= lower_high_boundary(), 1029 "must not shrink beyond region"); 1030 if (!os::uncommit_memory(aligned_lower_new_high, lower_needs, _executable)) { 1031 debug_only(warning("os::uncommit_memory failed")); 1032 return; 1033 } else { 1034 _lower_high -= lower_needs; 1035 } 1036 } 1037 1038 _high -= size; 1039 } 1040 1041 #ifndef PRODUCT 1042 void VirtualSpace::check_for_contiguity() { 1043 // Check contiguity. 1044 assert(low_boundary() <= lower_high() && 1045 lower_high() <= lower_high_boundary(), 1046 "high address must be contained within the region"); 1047 assert(lower_high_boundary() <= middle_high() && 1048 middle_high() <= middle_high_boundary(), 1049 "high address must be contained within the region"); 1050 assert(middle_high_boundary() <= upper_high() && 1051 upper_high() <= upper_high_boundary(), 1052 "high address must be contained within the region"); 1053 assert(low() >= low_boundary(), "low"); 1054 assert(low_boundary() <= lower_high_boundary(), "lower high boundary"); 1055 assert(upper_high_boundary() <= high_boundary(), "upper high boundary"); 1056 assert(high() <= upper_high(), "upper high"); 1057 } 1058 1059 void VirtualSpace::print_on(outputStream* out) const { 1060 out->print ("Virtual space:"); 1061 if (special()) out->print(" (pinned in memory)"); 1062 out->cr(); 1063 out->print_cr(" - committed: " SIZE_FORMAT, committed_size()); 1064 out->print_cr(" - reserved: " SIZE_FORMAT, reserved_size()); 1065 out->print_cr(" - [low, high]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", p2i(low()), p2i(high())); 1066 out->print_cr(" - [low_b, high_b]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", p2i(low_boundary()), p2i(high_boundary())); 1067 } 1068 1069 void VirtualSpace::print() const { 1070 print_on(tty); 1071 } 1072 1073 #endif