1 /* 2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "logging/log.hpp" 27 #include "memory/resourceArea.hpp" 28 #include "memory/virtualspace.hpp" 29 #include "oops/compressedOops.hpp" 30 #include "oops/markWord.hpp" 31 #include "oops/oop.inline.hpp" 32 #include "runtime/globals_extension.hpp" 33 #include "runtime/java.hpp" 34 #include "runtime/os.hpp" 35 #include "services/memTracker.hpp" 36 #include "utilities/align.hpp" 37 #include "utilities/formatBuffer.hpp" 38 #include "utilities/powerOfTwo.hpp" 39 40 // ReservedSpace 41 42 // Dummy constructor 43 ReservedSpace::ReservedSpace() : _base(nullptr), _size(0), _noaccess_prefix(0), 44 _alignment(0), _special(false), _fd_for_heap(-1), _executable(false) { 45 } 46 47 ReservedSpace::ReservedSpace(size_t size) : _fd_for_heap(-1) { 48 // Want to use large pages where possible. If the size is 49 // not large page aligned the mapping will be a mix of 50 // large and normal pages. 51 size_t page_size = os::page_size_for_region_unaligned(size, 1); 52 size_t alignment = os::vm_allocation_granularity(); 53 initialize(size, alignment, page_size, nullptr, false); 54 } 55 56 ReservedSpace::ReservedSpace(size_t size, size_t preferred_page_size) : _fd_for_heap(-1) { 57 // When a page size is given we don't want to mix large 58 // and normal pages. If the size is not a multiple of the 59 // page size it will be aligned up to achieve this. 60 size_t alignment = os::vm_allocation_granularity();; 61 if (preferred_page_size != os::vm_page_size()) { 62 alignment = MAX2(preferred_page_size, alignment); 63 size = align_up(size, alignment); 64 } 65 initialize(size, alignment, preferred_page_size, nullptr, false); 66 } 67 68 ReservedSpace::ReservedSpace(size_t size, 69 size_t alignment, 70 size_t page_size, 71 char* requested_address) : _fd_for_heap(-1) { 72 initialize(size, alignment, page_size, requested_address, false); 73 } 74 75 ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment, size_t page_size, 76 bool special, bool executable) : _fd_for_heap(-1) { 77 assert((size % os::vm_allocation_granularity()) == 0, 78 "size not allocation aligned"); 79 initialize_members(base, size, alignment, page_size, special, executable); 80 } 81 82 // Helper method 83 static char* attempt_map_or_reserve_memory_at(char* base, size_t size, int fd, bool executable) { 84 if (fd != -1) { 85 return os::attempt_map_memory_to_file_at(base, size, fd); 86 } 87 return os::attempt_reserve_memory_at(base, size, executable); 88 } 89 90 // Helper method 91 static char* map_or_reserve_memory(size_t size, int fd, bool executable) { 92 if (fd != -1) { 93 return os::map_memory_to_file(size, fd); 94 } 95 return os::reserve_memory(size, executable); 96 } 97 98 // Helper method 99 static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int fd, bool executable) { 100 if (fd != -1) { 101 return os::map_memory_to_file_aligned(size, alignment, fd); 102 } 103 return os::reserve_memory_aligned(size, alignment, executable); 104 } 105 106 // Helper method 107 static void unmap_or_release_memory(char* base, size_t size, bool is_file_mapped) { 108 if (is_file_mapped) { 109 if (!os::unmap_memory(base, size)) { 110 fatal("os::unmap_memory failed"); 111 } 112 } else if (!os::release_memory(base, size)) { 113 fatal("os::release_memory failed"); 114 } 115 } 116 117 // Helper method 118 static bool failed_to_reserve_as_requested(char* base, char* requested_address) { 119 if (base == requested_address || requested_address == nullptr) { 120 return false; // did not fail 121 } 122 123 if (base != nullptr) { 124 // Different reserve address may be acceptable in other cases 125 // but for compressed oops heap should be at requested address. 126 assert(UseCompressedOops, "currently requested address used only for compressed oops"); 127 log_debug(gc, heap, coops)("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, p2i(base), p2i(requested_address)); 128 } 129 return true; 130 } 131 132 static bool use_explicit_large_pages(size_t page_size) { 133 return !os::can_commit_large_page_memory() && 134 page_size != os::vm_page_size(); 135 } 136 137 static bool large_pages_requested() { 138 return UseLargePages && 139 (!FLAG_IS_DEFAULT(UseLargePages) || !FLAG_IS_DEFAULT(LargePageSizeInBytes)); 140 } 141 142 static void log_on_large_pages_failure(char* req_addr, size_t bytes) { 143 if (large_pages_requested()) { 144 // Compressed oops logging. 145 log_debug(gc, heap, coops)("Reserve regular memory without large pages"); 146 // JVM style warning that we did not succeed in using large pages. 147 char msg[128]; 148 jio_snprintf(msg, sizeof(msg), "Failed to reserve and commit memory using large pages. " 149 "req_addr: " PTR_FORMAT " bytes: " SIZE_FORMAT, 150 req_addr, bytes); 151 warning("%s", msg); 152 } 153 } 154 155 static char* reserve_memory(char* requested_address, const size_t size, 156 const size_t alignment, int fd, bool exec) { 157 char* base; 158 // If the memory was requested at a particular address, use 159 // os::attempt_reserve_memory_at() to avoid mapping over something 160 // important. If the reservation fails, return null. 161 if (requested_address != 0) { 162 assert(is_aligned(requested_address, alignment), 163 "Requested address " PTR_FORMAT " must be aligned to " SIZE_FORMAT, 164 p2i(requested_address), alignment); 165 base = attempt_map_or_reserve_memory_at(requested_address, size, fd, exec); 166 } else { 167 // Optimistically assume that the OS returns an aligned base pointer. 168 // When reserving a large address range, most OSes seem to align to at 169 // least 64K. 170 base = map_or_reserve_memory(size, fd, exec); 171 // Check alignment constraints. This is only needed when there is 172 // no requested address. 173 if (!is_aligned(base, alignment)) { 174 // Base not aligned, retry. 175 unmap_or_release_memory(base, size, fd != -1 /*is_file_mapped*/); 176 // Map using the requested alignment. 177 base = map_or_reserve_memory_aligned(size, alignment, fd, exec); 178 } 179 } 180 181 return base; 182 } 183 184 static char* reserve_memory_special(char* requested_address, const size_t size, 185 const size_t alignment, const size_t page_size, bool exec) { 186 187 log_trace(pagesize)("Attempt special mapping: size: " SIZE_FORMAT "%s, " 188 "alignment: " SIZE_FORMAT "%s", 189 byte_size_in_exact_unit(size), exact_unit_for_byte_size(size), 190 byte_size_in_exact_unit(alignment), exact_unit_for_byte_size(alignment)); 191 192 char* base = os::reserve_memory_special(size, alignment, page_size, requested_address, exec); 193 if (base != nullptr) { 194 // Check alignment constraints. 195 assert(is_aligned(base, alignment), 196 "reserve_memory_special() returned an unaligned address, base: " PTR_FORMAT 197 " alignment: " SIZE_FORMAT_X, 198 p2i(base), alignment); 199 } 200 return base; 201 } 202 203 void ReservedSpace::clear_members() { 204 initialize_members(nullptr, 0, 0, 0, false, false); 205 } 206 207 void ReservedSpace::initialize_members(char* base, size_t size, size_t alignment, 208 size_t page_size, bool special, bool executable) { 209 _base = base; 210 _size = size; 211 _alignment = alignment; 212 _page_size = page_size; 213 _special = special; 214 _executable = executable; 215 _noaccess_prefix = 0; 216 } 217 218 void ReservedSpace::reserve(size_t size, 219 size_t alignment, 220 size_t page_size, 221 char* requested_address, 222 bool executable) { 223 assert(is_aligned(size, alignment), "Size must be aligned to the requested alignment"); 224 225 // There are basically three different cases that we need to handle below: 226 // 1. Mapping backed by a file 227 // 2. Mapping backed by explicit large pages 228 // 3. Mapping backed by normal pages or transparent huge pages 229 // The first two have restrictions that requires the whole mapping to be 230 // committed up front. To record this the ReservedSpace is marked 'special'. 231 232 // == Case 1 == 233 if (_fd_for_heap != -1) { 234 // When there is a backing file directory for this space then whether 235 // large pages are allocated is up to the filesystem of the backing file. 236 // So UseLargePages is not taken into account for this reservation. 237 char* base = reserve_memory(requested_address, size, alignment, _fd_for_heap, executable); 238 if (base != nullptr) { 239 initialize_members(base, size, alignment, os::vm_page_size(), true, executable); 240 } 241 // Always return, not possible to fall back to reservation not using a file. 242 return; 243 } 244 245 // == Case 2 == 246 if (use_explicit_large_pages(page_size)) { 247 // System can't commit large pages i.e. use transparent huge pages and 248 // the caller requested large pages. To satisfy this request we use 249 // explicit large pages and these have to be committed up front to ensure 250 // no reservations are lost. 251 do { 252 char* base = reserve_memory_special(requested_address, size, alignment, page_size, executable); 253 if (base != nullptr) { 254 // Successful reservation using large pages. 255 initialize_members(base, size, alignment, page_size, true, executable); 256 return; 257 } 258 page_size = os::page_sizes().next_smaller(page_size); 259 } while (page_size > os::vm_page_size()); 260 261 // Failed to reserve explicit large pages, do proper logging. 262 log_on_large_pages_failure(requested_address, size); 263 // Now fall back to normal reservation. 264 assert(page_size == os::vm_page_size(), "inv"); 265 } 266 267 // == Case 3 == 268 char* base = reserve_memory(requested_address, size, alignment, -1, executable); 269 if (base != nullptr) { 270 // Successful mapping. 271 initialize_members(base, size, alignment, page_size, false, executable); 272 } 273 } 274 275 void ReservedSpace::initialize(size_t size, 276 size_t alignment, 277 size_t page_size, 278 char* requested_address, 279 bool executable) { 280 const size_t granularity = os::vm_allocation_granularity(); 281 assert((size & (granularity - 1)) == 0, 282 "size not aligned to os::vm_allocation_granularity()"); 283 assert((alignment & (granularity - 1)) == 0, 284 "alignment not aligned to os::vm_allocation_granularity()"); 285 assert(alignment == 0 || is_power_of_2((intptr_t)alignment), 286 "not a power of 2"); 287 assert(page_size >= os::vm_page_size(), "Invalid page size"); 288 assert(is_power_of_2(page_size), "Invalid page size"); 289 290 clear_members(); 291 292 if (size == 0) { 293 return; 294 } 295 296 // Adjust alignment to not be 0. 297 alignment = MAX2(alignment, os::vm_page_size()); 298 299 // Reserve the memory. 300 reserve(size, alignment, page_size, requested_address, executable); 301 302 // Check that the requested address is used if given. 303 if (failed_to_reserve_as_requested(_base, requested_address)) { 304 // OS ignored the requested address, release the reservation. 305 release(); 306 return; 307 } 308 } 309 310 ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment) { 311 assert(partition_size <= size(), "partition failed"); 312 ReservedSpace result(base(), partition_size, alignment, page_size(), special(), executable()); 313 return result; 314 } 315 316 317 ReservedSpace 318 ReservedSpace::last_part(size_t partition_size, size_t alignment) { 319 assert(partition_size <= size(), "partition failed"); 320 ReservedSpace result(base() + partition_size, size() - partition_size, 321 alignment, page_size(), special(), executable()); 322 return result; 323 } 324 325 326 size_t ReservedSpace::page_align_size_up(size_t size) { 327 return align_up(size, os::vm_page_size()); 328 } 329 330 331 size_t ReservedSpace::page_align_size_down(size_t size) { 332 return align_down(size, os::vm_page_size()); 333 } 334 335 336 size_t ReservedSpace::allocation_align_size_up(size_t size) { 337 return align_up(size, os::vm_allocation_granularity()); 338 } 339 340 void ReservedSpace::release() { 341 if (is_reserved()) { 342 char *real_base = _base - _noaccess_prefix; 343 const size_t real_size = _size + _noaccess_prefix; 344 if (special()) { 345 if (_fd_for_heap != -1) { 346 os::unmap_memory(real_base, real_size); 347 } else { 348 os::release_memory_special(real_base, real_size); 349 } 350 } else{ 351 os::release_memory(real_base, real_size); 352 } 353 clear_members(); 354 } 355 } 356 357 static size_t noaccess_prefix_size(size_t alignment) { 358 return lcm(os::vm_page_size(), alignment); 359 } 360 361 void ReservedHeapSpace::establish_noaccess_prefix() { 362 assert(_alignment >= os::vm_page_size(), "must be at least page size big"); 363 _noaccess_prefix = noaccess_prefix_size(_alignment); 364 365 if (base() && base() + _size > (char *)OopEncodingHeapMax) { 366 if (true 367 WIN64_ONLY(&& !UseLargePages) 368 AIX_ONLY(&& os::vm_page_size() != 64*K)) { 369 // Protect memory at the base of the allocated region. 370 // If special, the page was committed (only matters on windows) 371 if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, _special)) { 372 fatal("cannot protect protection page"); 373 } 374 log_debug(gc, heap, coops)("Protected page at the reserved heap base: " 375 PTR_FORMAT " / " INTX_FORMAT " bytes", 376 p2i(_base), 377 _noaccess_prefix); 378 assert(CompressedOops::use_implicit_null_checks() == true, "not initialized?"); 379 } else { 380 CompressedOops::set_use_implicit_null_checks(false); 381 } 382 } 383 384 _base += _noaccess_prefix; 385 _size -= _noaccess_prefix; 386 assert(((uintptr_t)_base % _alignment == 0), "must be exactly of required alignment"); 387 } 388 389 // Tries to allocate memory of size 'size' at address requested_address with alignment 'alignment'. 390 // Does not check whether the reserved memory actually is at requested_address, as the memory returned 391 // might still fulfill the wishes of the caller. 392 // Assures the memory is aligned to 'alignment'. 393 // NOTE: If ReservedHeapSpace already points to some reserved memory this is freed, first. 394 void ReservedHeapSpace::try_reserve_heap(size_t size, 395 size_t alignment, 396 size_t page_size, 397 char* requested_address) { 398 if (_base != nullptr) { 399 // We tried before, but we didn't like the address delivered. 400 release(); 401 } 402 403 // Try to reserve the memory for the heap. 404 log_trace(gc, heap, coops)("Trying to allocate at address " PTR_FORMAT 405 " heap of size " SIZE_FORMAT_X, 406 p2i(requested_address), 407 size); 408 409 reserve(size, alignment, page_size, requested_address, false); 410 411 // Check alignment constraints. 412 if (is_reserved() && !is_aligned(_base, _alignment)) { 413 // Base not aligned, retry. 414 release(); 415 } 416 } 417 418 void ReservedHeapSpace::try_reserve_range(char *highest_start, 419 char *lowest_start, 420 size_t attach_point_alignment, 421 char *aligned_heap_base_min_address, 422 char *upper_bound, 423 size_t size, 424 size_t alignment, 425 size_t page_size) { 426 const size_t attach_range = highest_start - lowest_start; 427 // Cap num_attempts at possible number. 428 // At least one is possible even for 0 sized attach range. 429 const uint64_t num_attempts_possible = (attach_range / attach_point_alignment) + 1; 430 const uint64_t num_attempts_to_try = MIN2((uint64_t)HeapSearchSteps, num_attempts_possible); 431 432 const size_t stepsize = (attach_range == 0) ? // Only one try. 433 (size_t) highest_start : align_up(attach_range / num_attempts_to_try, attach_point_alignment); 434 435 // Try attach points from top to bottom. 436 char* attach_point = highest_start; 437 while (attach_point >= lowest_start && 438 attach_point <= highest_start && // Avoid wrap around. 439 ((_base == nullptr) || 440 (_base < aligned_heap_base_min_address || _base + size > upper_bound))) { 441 try_reserve_heap(size, alignment, page_size, attach_point); 442 attach_point -= stepsize; 443 } 444 } 445 446 #define SIZE_64K ((uint64_t) UCONST64( 0x10000)) 447 #define SIZE_256M ((uint64_t) UCONST64( 0x10000000)) 448 #define SIZE_32G ((uint64_t) UCONST64( 0x800000000)) 449 450 // Helper for heap allocation. Returns an array with addresses 451 // (OS-specific) which are suited for disjoint base mode. Array is 452 // null terminated. 453 static char** get_attach_addresses_for_disjoint_mode() { 454 static uint64_t addresses[] = { 455 2 * SIZE_32G, 456 3 * SIZE_32G, 457 4 * SIZE_32G, 458 8 * SIZE_32G, 459 10 * SIZE_32G, 460 1 * SIZE_64K * SIZE_32G, 461 2 * SIZE_64K * SIZE_32G, 462 3 * SIZE_64K * SIZE_32G, 463 4 * SIZE_64K * SIZE_32G, 464 16 * SIZE_64K * SIZE_32G, 465 32 * SIZE_64K * SIZE_32G, 466 34 * SIZE_64K * SIZE_32G, 467 0 468 }; 469 470 // Sort out addresses smaller than HeapBaseMinAddress. This assumes 471 // the array is sorted. 472 uint i = 0; 473 while (addresses[i] != 0 && 474 (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) { 475 i++; 476 } 477 uint start = i; 478 479 // Avoid more steps than requested. 480 i = 0; 481 while (addresses[start+i] != 0) { 482 if (i == HeapSearchSteps) { 483 addresses[start+i] = 0; 484 break; 485 } 486 i++; 487 } 488 489 return (char**) &addresses[start]; 490 } 491 492 void ReservedHeapSpace::initialize_compressed_heap(const size_t size, size_t alignment, size_t page_size) { 493 guarantee(size + noaccess_prefix_size(alignment) <= OopEncodingHeapMax, 494 "can not allocate compressed oop heap for this size"); 495 guarantee(alignment == MAX2(alignment, os::vm_page_size()), "alignment too small"); 496 497 const size_t granularity = os::vm_allocation_granularity(); 498 assert((size & (granularity - 1)) == 0, 499 "size not aligned to os::vm_allocation_granularity()"); 500 assert((alignment & (granularity - 1)) == 0, 501 "alignment not aligned to os::vm_allocation_granularity()"); 502 assert(alignment == 0 || is_power_of_2((intptr_t)alignment), 503 "not a power of 2"); 504 505 // The necessary attach point alignment for generated wish addresses. 506 // This is needed to increase the chance of attaching for mmap and shmat. 507 const size_t os_attach_point_alignment = 508 AIX_ONLY(SIZE_256M) // Known shm boundary alignment. 509 NOT_AIX(os::vm_allocation_granularity()); 510 const size_t attach_point_alignment = lcm(alignment, os_attach_point_alignment); 511 512 char *aligned_heap_base_min_address = (char *)align_up((void *)HeapBaseMinAddress, alignment); 513 size_t noaccess_prefix = ((aligned_heap_base_min_address + size) > (char*)OopEncodingHeapMax) ? 514 noaccess_prefix_size(alignment) : 0; 515 516 // Attempt to alloc at user-given address. 517 if (!FLAG_IS_DEFAULT(HeapBaseMinAddress)) { 518 try_reserve_heap(size + noaccess_prefix, alignment, page_size, aligned_heap_base_min_address); 519 if (_base != aligned_heap_base_min_address) { // Enforce this exact address. 520 release(); 521 } 522 } 523 524 // Keep heap at HeapBaseMinAddress. 525 if (_base == nullptr) { 526 527 // Try to allocate the heap at addresses that allow efficient oop compression. 528 // Different schemes are tried, in order of decreasing optimization potential. 529 // 530 // For this, try_reserve_heap() is called with the desired heap base addresses. 531 // A call into the os layer to allocate at a given address can return memory 532 // at a different address than requested. Still, this might be memory at a useful 533 // address. try_reserve_heap() always returns this allocated memory, as only here 534 // the criteria for a good heap are checked. 535 536 // Attempt to allocate so that we can run without base and scale (32-Bit unscaled compressed oops). 537 // Give it several tries from top of range to bottom. 538 if (aligned_heap_base_min_address + size <= (char *)UnscaledOopHeapMax) { 539 540 // Calc address range within we try to attach (range of possible start addresses). 541 char* const highest_start = align_down((char *)UnscaledOopHeapMax - size, attach_point_alignment); 542 char* const lowest_start = align_up(aligned_heap_base_min_address, attach_point_alignment); 543 try_reserve_range(highest_start, lowest_start, attach_point_alignment, 544 aligned_heap_base_min_address, (char *)UnscaledOopHeapMax, size, alignment, page_size); 545 } 546 547 // zerobased: Attempt to allocate in the lower 32G. 548 // But leave room for the compressed class pointers, which is allocated above 549 // the heap. 550 char *zerobased_max = (char *)OopEncodingHeapMax; 551 const size_t class_space = align_up(CompressedClassSpaceSize, alignment); 552 // For small heaps, save some space for compressed class pointer 553 // space so it can be decoded with no base. 554 if (UseCompressedClassPointers && !UseSharedSpaces && 555 OopEncodingHeapMax <= KlassEncodingMetaspaceMax && 556 (uint64_t)(aligned_heap_base_min_address + size + class_space) <= KlassEncodingMetaspaceMax) { 557 zerobased_max = (char *)OopEncodingHeapMax - class_space; 558 } 559 560 // Give it several tries from top of range to bottom. 561 if (aligned_heap_base_min_address + size <= zerobased_max && // Zerobased theoretical possible. 562 ((_base == nullptr) || // No previous try succeeded. 563 (_base + size > zerobased_max))) { // Unscaled delivered an arbitrary address. 564 565 // Calc address range within we try to attach (range of possible start addresses). 566 char *const highest_start = align_down(zerobased_max - size, attach_point_alignment); 567 // Need to be careful about size being guaranteed to be less 568 // than UnscaledOopHeapMax due to type constraints. 569 char *lowest_start = aligned_heap_base_min_address; 570 uint64_t unscaled_end = UnscaledOopHeapMax - size; 571 if (unscaled_end < UnscaledOopHeapMax) { // unscaled_end wrapped if size is large 572 lowest_start = MAX2(lowest_start, (char*)unscaled_end); 573 } 574 lowest_start = align_up(lowest_start, attach_point_alignment); 575 try_reserve_range(highest_start, lowest_start, attach_point_alignment, 576 aligned_heap_base_min_address, zerobased_max, size, alignment, page_size); 577 } 578 579 // Now we go for heaps with base != 0. We need a noaccess prefix to efficiently 580 // implement null checks. 581 noaccess_prefix = noaccess_prefix_size(alignment); 582 583 // Try to attach at addresses that are aligned to OopEncodingHeapMax. Disjointbase mode. 584 char** addresses = get_attach_addresses_for_disjoint_mode(); 585 int i = 0; 586 while (addresses[i] && // End of array not yet reached. 587 ((_base == nullptr) || // No previous try succeeded. 588 (_base + size > (char *)OopEncodingHeapMax && // Not zerobased or unscaled address. 589 !CompressedOops::is_disjoint_heap_base_address((address)_base)))) { // Not disjoint address. 590 char* const attach_point = addresses[i]; 591 assert(attach_point >= aligned_heap_base_min_address, "Flag support broken"); 592 try_reserve_heap(size + noaccess_prefix, alignment, page_size, attach_point); 593 i++; 594 } 595 596 // Last, desperate try without any placement. 597 if (_base == nullptr) { 598 log_trace(gc, heap, coops)("Trying to allocate at address nullptr heap of size " SIZE_FORMAT_X, size + noaccess_prefix); 599 initialize(size + noaccess_prefix, alignment, page_size, nullptr, false); 600 } 601 } 602 } 603 604 ReservedHeapSpace::ReservedHeapSpace(size_t size, size_t alignment, size_t page_size, const char* heap_allocation_directory) : ReservedSpace() { 605 606 if (size == 0) { 607 return; 608 } 609 610 if (heap_allocation_directory != nullptr) { 611 _fd_for_heap = os::create_file_for_heap(heap_allocation_directory); 612 if (_fd_for_heap == -1) { 613 vm_exit_during_initialization( 614 err_msg("Could not create file for Heap at location %s", heap_allocation_directory)); 615 } 616 // When there is a backing file directory for this space then whether 617 // large pages are allocated is up to the filesystem of the backing file. 618 // If requested, let the user know that explicit large pages can't be used. 619 if (use_explicit_large_pages(page_size) && large_pages_requested()) { 620 log_debug(gc, heap)("Cannot allocate explicit large pages for Java Heap when AllocateHeapAt option is set."); 621 } 622 } 623 624 // Heap size should be aligned to alignment, too. 625 guarantee(is_aligned(size, alignment), "set by caller"); 626 627 if (UseCompressedOops) { 628 initialize_compressed_heap(size, alignment, page_size); 629 if (_size > size) { 630 // We allocated heap with noaccess prefix. 631 // It can happen we get a zerobased/unscaled heap with noaccess prefix, 632 // if we had to try at arbitrary address. 633 establish_noaccess_prefix(); 634 } 635 } else { 636 initialize(size, alignment, page_size, nullptr, false); 637 } 638 639 assert(markWord::encode_pointer_as_mark(_base).decode_pointer() == _base, 640 "area must be distinguishable from marks for mark-sweep"); 641 assert(markWord::encode_pointer_as_mark(&_base[size]).decode_pointer() == &_base[size], 642 "area must be distinguishable from marks for mark-sweep"); 643 644 if (base() != nullptr) { 645 MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap); 646 } 647 648 if (_fd_for_heap != -1) { 649 ::close(_fd_for_heap); 650 } 651 } 652 653 MemRegion ReservedHeapSpace::region() const { 654 return MemRegion((HeapWord*)base(), (HeapWord*)end()); 655 } 656 657 // Reserve space for code segment. Same as Java heap only we mark this as 658 // executable. 659 ReservedCodeSpace::ReservedCodeSpace(size_t r_size, 660 size_t rs_align, 661 size_t rs_page_size) : ReservedSpace() { 662 initialize(r_size, rs_align, rs_page_size, /*requested address*/ nullptr, /*executable*/ true); 663 MemTracker::record_virtual_memory_type((address)base(), mtCode); 664 } 665 666 // VirtualSpace 667 668 VirtualSpace::VirtualSpace() { 669 _low_boundary = nullptr; 670 _high_boundary = nullptr; 671 _low = nullptr; 672 _high = nullptr; 673 _lower_high = nullptr; 674 _middle_high = nullptr; 675 _upper_high = nullptr; 676 _lower_high_boundary = nullptr; 677 _middle_high_boundary = nullptr; 678 _upper_high_boundary = nullptr; 679 _lower_alignment = 0; 680 _middle_alignment = 0; 681 _upper_alignment = 0; 682 _special = false; 683 _executable = false; 684 } 685 686 687 bool VirtualSpace::initialize(ReservedSpace rs, size_t committed_size) { 688 const size_t max_commit_granularity = os::page_size_for_region_unaligned(rs.size(), 1); 689 return initialize_with_granularity(rs, committed_size, max_commit_granularity); 690 } 691 692 bool VirtualSpace::initialize_with_granularity(ReservedSpace rs, size_t committed_size, size_t max_commit_granularity) { 693 if(!rs.is_reserved()) return false; // allocation failed. 694 assert(_low_boundary == nullptr, "VirtualSpace already initialized"); 695 assert(max_commit_granularity > 0, "Granularity must be non-zero."); 696 697 _low_boundary = rs.base(); 698 _high_boundary = low_boundary() + rs.size(); 699 700 _low = low_boundary(); 701 _high = low(); 702 703 _special = rs.special(); 704 _executable = rs.executable(); 705 706 // When a VirtualSpace begins life at a large size, make all future expansion 707 // and shrinking occur aligned to a granularity of large pages. This avoids 708 // fragmentation of physical addresses that inhibits the use of large pages 709 // by the OS virtual memory system. Empirically, we see that with a 4MB 710 // page size, the only spaces that get handled this way are codecache and 711 // the heap itself, both of which provide a substantial performance 712 // boost in many benchmarks when covered by large pages. 713 // 714 // No attempt is made to force large page alignment at the very top and 715 // bottom of the space if they are not aligned so already. 716 _lower_alignment = os::vm_page_size(); 717 _middle_alignment = max_commit_granularity; 718 _upper_alignment = os::vm_page_size(); 719 720 // End of each region 721 _lower_high_boundary = align_up(low_boundary(), middle_alignment()); 722 _middle_high_boundary = align_down(high_boundary(), middle_alignment()); 723 _upper_high_boundary = high_boundary(); 724 725 // High address of each region 726 _lower_high = low_boundary(); 727 _middle_high = lower_high_boundary(); 728 _upper_high = middle_high_boundary(); 729 730 // commit to initial size 731 if (committed_size > 0) { 732 if (!expand_by(committed_size)) { 733 return false; 734 } 735 } 736 return true; 737 } 738 739 740 VirtualSpace::~VirtualSpace() { 741 release(); 742 } 743 744 745 void VirtualSpace::release() { 746 // This does not release memory it reserved. 747 // Caller must release via rs.release(); 748 _low_boundary = nullptr; 749 _high_boundary = nullptr; 750 _low = nullptr; 751 _high = nullptr; 752 _lower_high = nullptr; 753 _middle_high = nullptr; 754 _upper_high = nullptr; 755 _lower_high_boundary = nullptr; 756 _middle_high_boundary = nullptr; 757 _upper_high_boundary = nullptr; 758 _lower_alignment = 0; 759 _middle_alignment = 0; 760 _upper_alignment = 0; 761 _special = false; 762 _executable = false; 763 } 764 765 766 size_t VirtualSpace::committed_size() const { 767 return pointer_delta(high(), low(), sizeof(char)); 768 } 769 770 771 size_t VirtualSpace::reserved_size() const { 772 return pointer_delta(high_boundary(), low_boundary(), sizeof(char)); 773 } 774 775 776 size_t VirtualSpace::uncommitted_size() const { 777 return reserved_size() - committed_size(); 778 } 779 780 size_t VirtualSpace::actual_committed_size() const { 781 // Special VirtualSpaces commit all reserved space up front. 782 if (special()) { 783 return reserved_size(); 784 } 785 786 size_t committed_low = pointer_delta(_lower_high, _low_boundary, sizeof(char)); 787 size_t committed_middle = pointer_delta(_middle_high, _lower_high_boundary, sizeof(char)); 788 size_t committed_high = pointer_delta(_upper_high, _middle_high_boundary, sizeof(char)); 789 790 #ifdef ASSERT 791 size_t lower = pointer_delta(_lower_high_boundary, _low_boundary, sizeof(char)); 792 size_t middle = pointer_delta(_middle_high_boundary, _lower_high_boundary, sizeof(char)); 793 size_t upper = pointer_delta(_upper_high_boundary, _middle_high_boundary, sizeof(char)); 794 795 if (committed_high > 0) { 796 assert(committed_low == lower, "Must be"); 797 assert(committed_middle == middle, "Must be"); 798 } 799 800 if (committed_middle > 0) { 801 assert(committed_low == lower, "Must be"); 802 } 803 if (committed_middle < middle) { 804 assert(committed_high == 0, "Must be"); 805 } 806 807 if (committed_low < lower) { 808 assert(committed_high == 0, "Must be"); 809 assert(committed_middle == 0, "Must be"); 810 } 811 #endif 812 813 return committed_low + committed_middle + committed_high; 814 } 815 816 817 bool VirtualSpace::contains(const void* p) const { 818 return low() <= (const char*) p && (const char*) p < high(); 819 } 820 821 static void pretouch_expanded_memory(void* start, void* end) { 822 assert(is_aligned(start, os::vm_page_size()), "Unexpected alignment"); 823 assert(is_aligned(end, os::vm_page_size()), "Unexpected alignment"); 824 825 os::pretouch_memory(start, end); 826 } 827 828 static bool commit_expanded(char* start, size_t size, size_t alignment, bool pre_touch, bool executable) { 829 if (os::commit_memory(start, size, alignment, executable)) { 830 if (pre_touch || AlwaysPreTouch) { 831 pretouch_expanded_memory(start, start + size); 832 } 833 return true; 834 } 835 836 debug_only(warning( 837 "INFO: os::commit_memory(" PTR_FORMAT ", " PTR_FORMAT 838 " size=" SIZE_FORMAT ", executable=%d) failed", 839 p2i(start), p2i(start + size), size, executable);) 840 841 return false; 842 } 843 844 /* 845 First we need to determine if a particular virtual space is using large 846 pages. This is done at the initialize function and only virtual spaces 847 that are larger than LargePageSizeInBytes use large pages. Once we 848 have determined this, all expand_by and shrink_by calls must grow and 849 shrink by large page size chunks. If a particular request 850 is within the current large page, the call to commit and uncommit memory 851 can be ignored. In the case that the low and high boundaries of this 852 space is not large page aligned, the pages leading to the first large 853 page address and the pages after the last large page address must be 854 allocated with default pages. 855 */ 856 bool VirtualSpace::expand_by(size_t bytes, bool pre_touch) { 857 if (uncommitted_size() < bytes) { 858 return false; 859 } 860 861 if (special()) { 862 // don't commit memory if the entire space is pinned in memory 863 _high += bytes; 864 return true; 865 } 866 867 char* previous_high = high(); 868 char* unaligned_new_high = high() + bytes; 869 assert(unaligned_new_high <= high_boundary(), "cannot expand by more than upper boundary"); 870 871 // Calculate where the new high for each of the regions should be. If 872 // the low_boundary() and high_boundary() are LargePageSizeInBytes aligned 873 // then the unaligned lower and upper new highs would be the 874 // lower_high() and upper_high() respectively. 875 char* unaligned_lower_new_high = MIN2(unaligned_new_high, lower_high_boundary()); 876 char* unaligned_middle_new_high = MIN2(unaligned_new_high, middle_high_boundary()); 877 char* unaligned_upper_new_high = MIN2(unaligned_new_high, upper_high_boundary()); 878 879 // Align the new highs based on the regions alignment. lower and upper 880 // alignment will always be default page size. middle alignment will be 881 // LargePageSizeInBytes if the actual size of the virtual space is in 882 // fact larger than LargePageSizeInBytes. 883 char* aligned_lower_new_high = align_up(unaligned_lower_new_high, lower_alignment()); 884 char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment()); 885 char* aligned_upper_new_high = align_up(unaligned_upper_new_high, upper_alignment()); 886 887 // Determine which regions need to grow in this expand_by call. 888 // If you are growing in the lower region, high() must be in that 889 // region so calculate the size based on high(). For the middle and 890 // upper regions, determine the starting point of growth based on the 891 // location of high(). By getting the MAX of the region's low address 892 // (or the previous region's high address) and high(), we can tell if it 893 // is an intra or inter region growth. 894 size_t lower_needs = 0; 895 if (aligned_lower_new_high > lower_high()) { 896 lower_needs = pointer_delta(aligned_lower_new_high, lower_high(), sizeof(char)); 897 } 898 size_t middle_needs = 0; 899 if (aligned_middle_new_high > middle_high()) { 900 middle_needs = pointer_delta(aligned_middle_new_high, middle_high(), sizeof(char)); 901 } 902 size_t upper_needs = 0; 903 if (aligned_upper_new_high > upper_high()) { 904 upper_needs = pointer_delta(aligned_upper_new_high, upper_high(), sizeof(char)); 905 } 906 907 // Check contiguity. 908 assert(low_boundary() <= lower_high() && lower_high() <= lower_high_boundary(), 909 "high address must be contained within the region"); 910 assert(lower_high_boundary() <= middle_high() && middle_high() <= middle_high_boundary(), 911 "high address must be contained within the region"); 912 assert(middle_high_boundary() <= upper_high() && upper_high() <= upper_high_boundary(), 913 "high address must be contained within the region"); 914 915 // Commit regions 916 if (lower_needs > 0) { 917 assert(lower_high() + lower_needs <= lower_high_boundary(), "must not expand beyond region"); 918 if (!commit_expanded(lower_high(), lower_needs, _lower_alignment, pre_touch, _executable)) { 919 return false; 920 } 921 _lower_high += lower_needs; 922 } 923 924 if (middle_needs > 0) { 925 assert(middle_high() + middle_needs <= middle_high_boundary(), "must not expand beyond region"); 926 if (!commit_expanded(middle_high(), middle_needs, _middle_alignment, pre_touch, _executable)) { 927 return false; 928 } 929 _middle_high += middle_needs; 930 } 931 932 if (upper_needs > 0) { 933 assert(upper_high() + upper_needs <= upper_high_boundary(), "must not expand beyond region"); 934 if (!commit_expanded(upper_high(), upper_needs, _upper_alignment, pre_touch, _executable)) { 935 return false; 936 } 937 _upper_high += upper_needs; 938 } 939 940 _high += bytes; 941 return true; 942 } 943 944 // A page is uncommitted if the contents of the entire page is deemed unusable. 945 // Continue to decrement the high() pointer until it reaches a page boundary 946 // in which case that particular page can now be uncommitted. 947 void VirtualSpace::shrink_by(size_t size) { 948 if (committed_size() < size) 949 fatal("Cannot shrink virtual space to negative size"); 950 951 if (special()) { 952 // don't uncommit if the entire space is pinned in memory 953 _high -= size; 954 return; 955 } 956 957 char* unaligned_new_high = high() - size; 958 assert(unaligned_new_high >= low_boundary(), "cannot shrink past lower boundary"); 959 960 // Calculate new unaligned address 961 char* unaligned_upper_new_high = 962 MAX2(unaligned_new_high, middle_high_boundary()); 963 char* unaligned_middle_new_high = 964 MAX2(unaligned_new_high, lower_high_boundary()); 965 char* unaligned_lower_new_high = 966 MAX2(unaligned_new_high, low_boundary()); 967 968 // Align address to region's alignment 969 char* aligned_upper_new_high = align_up(unaligned_upper_new_high, upper_alignment()); 970 char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment()); 971 char* aligned_lower_new_high = align_up(unaligned_lower_new_high, lower_alignment()); 972 973 // Determine which regions need to shrink 974 size_t upper_needs = 0; 975 if (aligned_upper_new_high < upper_high()) { 976 upper_needs = 977 pointer_delta(upper_high(), aligned_upper_new_high, sizeof(char)); 978 } 979 size_t middle_needs = 0; 980 if (aligned_middle_new_high < middle_high()) { 981 middle_needs = 982 pointer_delta(middle_high(), aligned_middle_new_high, sizeof(char)); 983 } 984 size_t lower_needs = 0; 985 if (aligned_lower_new_high < lower_high()) { 986 lower_needs = 987 pointer_delta(lower_high(), aligned_lower_new_high, sizeof(char)); 988 } 989 990 // Check contiguity. 991 assert(middle_high_boundary() <= upper_high() && 992 upper_high() <= upper_high_boundary(), 993 "high address must be contained within the region"); 994 assert(lower_high_boundary() <= middle_high() && 995 middle_high() <= middle_high_boundary(), 996 "high address must be contained within the region"); 997 assert(low_boundary() <= lower_high() && 998 lower_high() <= lower_high_boundary(), 999 "high address must be contained within the region"); 1000 1001 // Uncommit 1002 if (upper_needs > 0) { 1003 assert(middle_high_boundary() <= aligned_upper_new_high && 1004 aligned_upper_new_high + upper_needs <= upper_high_boundary(), 1005 "must not shrink beyond region"); 1006 if (!os::uncommit_memory(aligned_upper_new_high, upper_needs, _executable)) { 1007 debug_only(warning("os::uncommit_memory failed")); 1008 return; 1009 } else { 1010 _upper_high -= upper_needs; 1011 } 1012 } 1013 if (middle_needs > 0) { 1014 assert(lower_high_boundary() <= aligned_middle_new_high && 1015 aligned_middle_new_high + middle_needs <= middle_high_boundary(), 1016 "must not shrink beyond region"); 1017 if (!os::uncommit_memory(aligned_middle_new_high, middle_needs, _executable)) { 1018 debug_only(warning("os::uncommit_memory failed")); 1019 return; 1020 } else { 1021 _middle_high -= middle_needs; 1022 } 1023 } 1024 if (lower_needs > 0) { 1025 assert(low_boundary() <= aligned_lower_new_high && 1026 aligned_lower_new_high + lower_needs <= lower_high_boundary(), 1027 "must not shrink beyond region"); 1028 if (!os::uncommit_memory(aligned_lower_new_high, lower_needs, _executable)) { 1029 debug_only(warning("os::uncommit_memory failed")); 1030 return; 1031 } else { 1032 _lower_high -= lower_needs; 1033 } 1034 } 1035 1036 _high -= size; 1037 } 1038 1039 #ifndef PRODUCT 1040 void VirtualSpace::check_for_contiguity() { 1041 // Check contiguity. 1042 assert(low_boundary() <= lower_high() && 1043 lower_high() <= lower_high_boundary(), 1044 "high address must be contained within the region"); 1045 assert(lower_high_boundary() <= middle_high() && 1046 middle_high() <= middle_high_boundary(), 1047 "high address must be contained within the region"); 1048 assert(middle_high_boundary() <= upper_high() && 1049 upper_high() <= upper_high_boundary(), 1050 "high address must be contained within the region"); 1051 assert(low() >= low_boundary(), "low"); 1052 assert(low_boundary() <= lower_high_boundary(), "lower high boundary"); 1053 assert(upper_high_boundary() <= high_boundary(), "upper high boundary"); 1054 assert(high() <= upper_high(), "upper high"); 1055 } 1056 1057 void VirtualSpace::print_on(outputStream* out) const { 1058 out->print ("Virtual space:"); 1059 if (special()) out->print(" (pinned in memory)"); 1060 out->cr(); 1061 out->print_cr(" - committed: " SIZE_FORMAT, committed_size()); 1062 out->print_cr(" - reserved: " SIZE_FORMAT, reserved_size()); 1063 out->print_cr(" - [low, high]: [" PTR_FORMAT ", " PTR_FORMAT "]", p2i(low()), p2i(high())); 1064 out->print_cr(" - [low_b, high_b]: [" PTR_FORMAT ", " PTR_FORMAT "]", p2i(low_boundary()), p2i(high_boundary())); 1065 } 1066 1067 void VirtualSpace::print() const { 1068 print_on(tty); 1069 } 1070 1071 #endif