1 /*
   2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "logging/log.hpp"
  27 #include "memory/resourceArea.hpp"
  28 #include "memory/virtualspace.hpp"
  29 #include "oops/compressedOops.hpp"
  30 #include "oops/markWord.hpp"
  31 #include "oops/oop.inline.hpp"
  32 #include "runtime/globals_extension.hpp"
  33 #include "runtime/java.hpp"
  34 #include "runtime/os.hpp"
  35 #include "services/memTracker.hpp"
  36 #include "utilities/align.hpp"
  37 #include "utilities/formatBuffer.hpp"
  38 #include "utilities/powerOfTwo.hpp"
  39 
  40 // ReservedSpace
  41 
  42 // Dummy constructor
  43 ReservedSpace::ReservedSpace() : _base(nullptr), _size(0), _noaccess_prefix(0),
  44     _alignment(0), _special(false), _fd_for_heap(-1), _executable(false) {
  45 }
  46 
  47 ReservedSpace::ReservedSpace(size_t size) : _fd_for_heap(-1) {
  48   // Want to use large pages where possible. If the size is
  49   // not large page aligned the mapping will be a mix of
  50   // large and normal pages.
  51   size_t page_size = os::page_size_for_region_unaligned(size, 1);
  52   size_t alignment = os::vm_allocation_granularity();
  53   initialize(size, alignment, page_size, nullptr, false);
  54 }
  55 
  56 ReservedSpace::ReservedSpace(size_t size, size_t preferred_page_size) : _fd_for_heap(-1) {
  57   // When a page size is given we don't want to mix large
  58   // and normal pages. If the size is not a multiple of the
  59   // page size it will be aligned up to achieve this.
  60   size_t alignment = os::vm_allocation_granularity();;
  61   if (preferred_page_size != os::vm_page_size()) {
  62     alignment = MAX2(preferred_page_size, alignment);
  63     size = align_up(size, alignment);
  64   }
  65   initialize(size, alignment, preferred_page_size, nullptr, false);
  66 }
  67 
  68 ReservedSpace::ReservedSpace(size_t size,
  69                              size_t alignment,
  70                              size_t page_size,
  71                              char* requested_address) : _fd_for_heap(-1) {
  72   initialize(size, alignment, page_size, requested_address, false);
  73 }
  74 
  75 ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment, size_t page_size,
  76                              bool special, bool executable) : _fd_for_heap(-1) {
  77   assert((size % os::vm_allocation_granularity()) == 0,
  78          "size not allocation aligned");
  79   initialize_members(base, size, alignment, page_size, special, executable);
  80 }
  81 
  82 // Helper method
  83 static char* attempt_map_or_reserve_memory_at(char* base, size_t size, int fd, bool executable) {
  84   if (fd != -1) {
  85     return os::attempt_map_memory_to_file_at(base, size, fd);
  86   }
  87   return os::attempt_reserve_memory_at(base, size, executable);
  88 }
  89 
  90 // Helper method
  91 static char* map_or_reserve_memory(size_t size, int fd, bool executable) {
  92   if (fd != -1) {
  93     return os::map_memory_to_file(size, fd);
  94   }
  95   return os::reserve_memory(size, executable);
  96 }
  97 
  98 // Helper method
  99 static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int fd, bool executable) {
 100   if (fd != -1) {
 101     return os::map_memory_to_file_aligned(size, alignment, fd);
 102   }
 103   return os::reserve_memory_aligned(size, alignment, executable);
 104 }
 105 
 106 // Helper method
 107 static void unmap_or_release_memory(char* base, size_t size, bool is_file_mapped) {
 108   if (is_file_mapped) {
 109     if (!os::unmap_memory(base, size)) {
 110       fatal("os::unmap_memory failed");
 111     }
 112   } else if (!os::release_memory(base, size)) {
 113     fatal("os::release_memory failed");
 114   }
 115 }
 116 
 117 // Helper method
 118 static bool failed_to_reserve_as_requested(char* base, char* requested_address) {
 119   if (base == requested_address || requested_address == nullptr) {
 120     return false; // did not fail
 121   }
 122 
 123   if (base != nullptr) {
 124     // Different reserve address may be acceptable in other cases
 125     // but for compressed oops heap should be at requested address.
 126     assert(UseCompressedOops, "currently requested address used only for compressed oops");
 127     log_debug(gc, heap, coops)("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, p2i(base), p2i(requested_address));
 128   }
 129   return true;
 130 }
 131 
 132 static bool use_explicit_large_pages(size_t page_size) {
 133   return !os::can_commit_large_page_memory() &&
 134          page_size != os::vm_page_size();
 135 }
 136 
 137 static bool large_pages_requested() {
 138   return UseLargePages &&
 139          (!FLAG_IS_DEFAULT(UseLargePages) || !FLAG_IS_DEFAULT(LargePageSizeInBytes));
 140 }
 141 
 142 static void log_on_large_pages_failure(char* req_addr, size_t bytes) {
 143   if (large_pages_requested()) {
 144     // Compressed oops logging.
 145     log_debug(gc, heap, coops)("Reserve regular memory without large pages");
 146     // JVM style warning that we did not succeed in using large pages.
 147     char msg[128];
 148     jio_snprintf(msg, sizeof(msg), "Failed to reserve and commit memory using large pages. "
 149                                    "req_addr: " PTR_FORMAT " bytes: " SIZE_FORMAT,
 150                                    req_addr, bytes);
 151     warning("%s", msg);
 152   }
 153 }
 154 
 155 static char* reserve_memory(char* requested_address, const size_t size,
 156                             const size_t alignment, int fd, bool exec) {
 157   char* base;
 158   // If the memory was requested at a particular address, use
 159   // os::attempt_reserve_memory_at() to avoid mapping over something
 160   // important.  If the reservation fails, return null.
 161   if (requested_address != 0) {
 162     assert(is_aligned(requested_address, alignment),
 163            "Requested address " PTR_FORMAT " must be aligned to " SIZE_FORMAT,
 164            p2i(requested_address), alignment);
 165     base = attempt_map_or_reserve_memory_at(requested_address, size, fd, exec);
 166   } else {
 167     // Optimistically assume that the OS returns an aligned base pointer.
 168     // When reserving a large address range, most OSes seem to align to at
 169     // least 64K.
 170     base = map_or_reserve_memory(size, fd, exec);
 171     // Check alignment constraints. This is only needed when there is
 172     // no requested address.
 173     if (!is_aligned(base, alignment)) {
 174       // Base not aligned, retry.
 175       unmap_or_release_memory(base, size, fd != -1 /*is_file_mapped*/);
 176       // Map using the requested alignment.
 177       base = map_or_reserve_memory_aligned(size, alignment, fd, exec);
 178     }
 179   }
 180 
 181   return base;
 182 }
 183 
 184 static char* reserve_memory_special(char* requested_address, const size_t size,
 185                                     const size_t alignment, const size_t page_size, bool exec) {
 186 
 187   log_trace(pagesize)("Attempt special mapping: size: " SIZE_FORMAT "%s, "
 188                       "alignment: " SIZE_FORMAT "%s",
 189                       byte_size_in_exact_unit(size), exact_unit_for_byte_size(size),
 190                       byte_size_in_exact_unit(alignment), exact_unit_for_byte_size(alignment));
 191 
 192   char* base = os::reserve_memory_special(size, alignment, page_size, requested_address, exec);
 193   if (base != nullptr) {
 194     // Check alignment constraints.
 195     assert(is_aligned(base, alignment),
 196            "reserve_memory_special() returned an unaligned address, base: " PTR_FORMAT
 197            " alignment: " SIZE_FORMAT_X,
 198            p2i(base), alignment);
 199   }
 200   return base;
 201 }
 202 
 203 void ReservedSpace::clear_members() {
 204   initialize_members(nullptr, 0, 0, 0, false, false);
 205 }
 206 
 207 void ReservedSpace::initialize_members(char* base, size_t size, size_t alignment,
 208                                        size_t page_size, bool special, bool executable) {
 209   _base = base;
 210   _size = size;
 211   _alignment = alignment;
 212   _page_size = page_size;
 213   _special = special;
 214   _executable = executable;
 215   _noaccess_prefix = 0;
 216 }
 217 
 218 void ReservedSpace::reserve(size_t size,
 219                             size_t alignment,
 220                             size_t page_size,
 221                             char* requested_address,
 222                             bool executable) {
 223   assert(is_aligned(size, alignment), "Size must be aligned to the requested alignment");
 224 
 225   // There are basically three different cases that we need to handle below:
 226   // 1. Mapping backed by a file
 227   // 2. Mapping backed by explicit large pages
 228   // 3. Mapping backed by normal pages or transparent huge pages
 229   // The first two have restrictions that requires the whole mapping to be
 230   // committed up front. To record this the ReservedSpace is marked 'special'.
 231 
 232   // == Case 1 ==
 233   if (_fd_for_heap != -1) {
 234     // When there is a backing file directory for this space then whether
 235     // large pages are allocated is up to the filesystem of the backing file.
 236     // So UseLargePages is not taken into account for this reservation.
 237     char* base = reserve_memory(requested_address, size, alignment, _fd_for_heap, executable);
 238     if (base != nullptr) {
 239       initialize_members(base, size, alignment, os::vm_page_size(), true, executable);
 240     }
 241     // Always return, not possible to fall back to reservation not using a file.
 242     return;
 243   }
 244 
 245   // == Case 2 ==
 246   if (use_explicit_large_pages(page_size)) {
 247     // System can't commit large pages i.e. use transparent huge pages and
 248     // the caller requested large pages. To satisfy this request we use
 249     // explicit large pages and these have to be committed up front to ensure
 250     // no reservations are lost.
 251     do {
 252       char* base = reserve_memory_special(requested_address, size, alignment, page_size, executable);
 253       if (base != nullptr) {
 254         // Successful reservation using large pages.
 255         initialize_members(base, size, alignment, page_size, true, executable);
 256         return;
 257       }
 258       page_size = os::page_sizes().next_smaller(page_size);
 259     } while (page_size > os::vm_page_size());
 260 
 261     // Failed to reserve explicit large pages, do proper logging.
 262     log_on_large_pages_failure(requested_address, size);
 263     // Now fall back to normal reservation.
 264     assert(page_size == os::vm_page_size(), "inv");
 265   }
 266 
 267   // == Case 3 ==
 268   char* base = reserve_memory(requested_address, size, alignment, -1, executable);
 269   if (base != nullptr) {
 270     // Successful mapping.
 271     initialize_members(base, size, alignment, page_size, false, executable);
 272   }
 273 }
 274 
 275 void ReservedSpace::initialize(size_t size,
 276                                size_t alignment,
 277                                size_t page_size,
 278                                char* requested_address,
 279                                bool executable) {
 280   const size_t granularity = os::vm_allocation_granularity();
 281   assert((size & (granularity - 1)) == 0,
 282          "size not aligned to os::vm_allocation_granularity()");
 283   assert((alignment & (granularity - 1)) == 0,
 284          "alignment not aligned to os::vm_allocation_granularity()");
 285   assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
 286          "not a power of 2");
 287   assert(page_size >= os::vm_page_size(), "Invalid page size");
 288   assert(is_power_of_2(page_size), "Invalid page size");
 289 
 290   clear_members();
 291 
 292   if (size == 0) {
 293     return;
 294   }
 295 
 296   // Adjust alignment to not be 0.
 297   alignment = MAX2(alignment, os::vm_page_size());
 298 
 299   // Reserve the memory.
 300   reserve(size, alignment, page_size, requested_address, executable);
 301 
 302   // Check that the requested address is used if given.
 303   if (failed_to_reserve_as_requested(_base, requested_address)) {
 304     // OS ignored the requested address, release the reservation.
 305     release();
 306     return;
 307   }
 308 }
 309 
 310 ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment) {
 311   assert(partition_size <= size(), "partition failed");
 312   ReservedSpace result(base(), partition_size, alignment, page_size(), special(), executable());
 313   return result;
 314 }
 315 
 316 
 317 ReservedSpace
 318 ReservedSpace::last_part(size_t partition_size, size_t alignment) {
 319   assert(partition_size <= size(), "partition failed");
 320   ReservedSpace result(base() + partition_size, size() - partition_size,
 321                        alignment, page_size(), special(), executable());
 322   return result;
 323 }
 324 
 325 
 326 size_t ReservedSpace::page_align_size_up(size_t size) {
 327   return align_up(size, os::vm_page_size());
 328 }
 329 
 330 
 331 size_t ReservedSpace::page_align_size_down(size_t size) {
 332   return align_down(size, os::vm_page_size());
 333 }
 334 
 335 
 336 size_t ReservedSpace::allocation_align_size_up(size_t size) {
 337   return align_up(size, os::vm_allocation_granularity());
 338 }
 339 
 340 void ReservedSpace::release() {
 341   if (is_reserved()) {
 342     char *real_base = _base - _noaccess_prefix;
 343     const size_t real_size = _size + _noaccess_prefix;
 344     if (special()) {
 345       if (_fd_for_heap != -1) {
 346         os::unmap_memory(real_base, real_size);
 347       } else {
 348         os::release_memory_special(real_base, real_size);
 349       }
 350     } else{
 351       os::release_memory(real_base, real_size);
 352     }
 353     clear_members();
 354   }
 355 }
 356 
 357 static size_t noaccess_prefix_size(size_t alignment) {
 358   return lcm(os::vm_page_size(), alignment);
 359 }
 360 
 361 void ReservedHeapSpace::establish_noaccess_prefix() {
 362   assert(_alignment >= os::vm_page_size(), "must be at least page size big");
 363   _noaccess_prefix = noaccess_prefix_size(_alignment);
 364 
 365   if (base() && base() + _size > (char *)OopEncodingHeapMax) {
 366     if (true
 367         WIN64_ONLY(&& !UseLargePages)
 368         AIX_ONLY(&& os::vm_page_size() != 64*K)) {
 369       // Protect memory at the base of the allocated region.
 370       // If special, the page was committed (only matters on windows)
 371       if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, _special)) {
 372         fatal("cannot protect protection page");
 373       }
 374       log_debug(gc, heap, coops)("Protected page at the reserved heap base: "
 375                                  PTR_FORMAT " / " INTX_FORMAT " bytes",
 376                                  p2i(_base),
 377                                  _noaccess_prefix);
 378       assert(CompressedOops::use_implicit_null_checks() == true, "not initialized?");
 379     } else {
 380       CompressedOops::set_use_implicit_null_checks(false);
 381     }
 382   }
 383 
 384   _base += _noaccess_prefix;
 385   _size -= _noaccess_prefix;
 386   assert(((uintptr_t)_base % _alignment == 0), "must be exactly of required alignment");
 387 }
 388 
 389 // Tries to allocate memory of size 'size' at address requested_address with alignment 'alignment'.
 390 // Does not check whether the reserved memory actually is at requested_address, as the memory returned
 391 // might still fulfill the wishes of the caller.
 392 // Assures the memory is aligned to 'alignment'.
 393 // NOTE: If ReservedHeapSpace already points to some reserved memory this is freed, first.
 394 void ReservedHeapSpace::try_reserve_heap(size_t size,
 395                                          size_t alignment,
 396                                          size_t page_size,
 397                                          char* requested_address) {
 398   if (_base != nullptr) {
 399     // We tried before, but we didn't like the address delivered.
 400     release();
 401   }
 402 
 403   // Try to reserve the memory for the heap.
 404   log_trace(gc, heap, coops)("Trying to allocate at address " PTR_FORMAT
 405                              " heap of size " SIZE_FORMAT_X,
 406                              p2i(requested_address),
 407                              size);
 408 
 409   reserve(size, alignment, page_size, requested_address, false);
 410 
 411   // Check alignment constraints.
 412   if (is_reserved() && !is_aligned(_base, _alignment)) {
 413     // Base not aligned, retry.
 414     release();
 415   }
 416 }
 417 
 418 void ReservedHeapSpace::try_reserve_range(char *highest_start,
 419                                           char *lowest_start,
 420                                           size_t attach_point_alignment,
 421                                           char *aligned_heap_base_min_address,
 422                                           char *upper_bound,
 423                                           size_t size,
 424                                           size_t alignment,
 425                                           size_t page_size) {
 426   const size_t attach_range = highest_start - lowest_start;
 427   // Cap num_attempts at possible number.
 428   // At least one is possible even for 0 sized attach range.
 429   const uint64_t num_attempts_possible = (attach_range / attach_point_alignment) + 1;
 430   const uint64_t num_attempts_to_try   = MIN2((uint64_t)HeapSearchSteps, num_attempts_possible);
 431 
 432   const size_t stepsize = (attach_range == 0) ? // Only one try.
 433     (size_t) highest_start : align_up(attach_range / num_attempts_to_try, attach_point_alignment);
 434 
 435   // Try attach points from top to bottom.
 436   char* attach_point = highest_start;
 437   while (attach_point >= lowest_start  &&
 438          attach_point <= highest_start &&  // Avoid wrap around.
 439          ((_base == nullptr) ||
 440           (_base < aligned_heap_base_min_address || _base + size > upper_bound))) {
 441     try_reserve_heap(size, alignment, page_size, attach_point);
 442     attach_point -= stepsize;
 443   }
 444 }
 445 
 446 #define SIZE_64K  ((uint64_t) UCONST64(      0x10000))
 447 #define SIZE_256M ((uint64_t) UCONST64(   0x10000000))
 448 #define SIZE_32G  ((uint64_t) UCONST64(  0x800000000))
 449 
 450 // Helper for heap allocation. Returns an array with addresses
 451 // (OS-specific) which are suited for disjoint base mode. Array is
 452 // null terminated.
 453 static char** get_attach_addresses_for_disjoint_mode() {
 454   static uint64_t addresses[] = {
 455      2 * SIZE_32G,
 456      3 * SIZE_32G,
 457      4 * SIZE_32G,
 458      8 * SIZE_32G,
 459     10 * SIZE_32G,
 460      1 * SIZE_64K * SIZE_32G,
 461      2 * SIZE_64K * SIZE_32G,
 462      3 * SIZE_64K * SIZE_32G,
 463      4 * SIZE_64K * SIZE_32G,
 464     16 * SIZE_64K * SIZE_32G,
 465     32 * SIZE_64K * SIZE_32G,
 466     34 * SIZE_64K * SIZE_32G,
 467     0
 468   };
 469 
 470   // Sort out addresses smaller than HeapBaseMinAddress. This assumes
 471   // the array is sorted.
 472   uint i = 0;
 473   while (addresses[i] != 0 &&
 474          (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) {
 475     i++;
 476   }
 477   uint start = i;
 478 
 479   // Avoid more steps than requested.
 480   i = 0;
 481   while (addresses[start+i] != 0) {
 482     if (i == HeapSearchSteps) {
 483       addresses[start+i] = 0;
 484       break;
 485     }
 486     i++;
 487   }
 488 
 489   return (char**) &addresses[start];
 490 }
 491 
 492 void ReservedHeapSpace::initialize_compressed_heap(const size_t size, size_t alignment, size_t page_size) {
 493   guarantee(size + noaccess_prefix_size(alignment) <= OopEncodingHeapMax,
 494             "can not allocate compressed oop heap for this size");
 495   guarantee(alignment == MAX2(alignment, os::vm_page_size()), "alignment too small");
 496 
 497   const size_t granularity = os::vm_allocation_granularity();
 498   assert((size & (granularity - 1)) == 0,
 499          "size not aligned to os::vm_allocation_granularity()");
 500   assert((alignment & (granularity - 1)) == 0,
 501          "alignment not aligned to os::vm_allocation_granularity()");
 502   assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
 503          "not a power of 2");
 504 
 505   // The necessary attach point alignment for generated wish addresses.
 506   // This is needed to increase the chance of attaching for mmap and shmat.
 507   const size_t os_attach_point_alignment =
 508     AIX_ONLY(SIZE_256M)  // Known shm boundary alignment.
 509     NOT_AIX(os::vm_allocation_granularity());
 510   const size_t attach_point_alignment = lcm(alignment, os_attach_point_alignment);
 511 
 512   char *aligned_heap_base_min_address = (char *)align_up((void *)HeapBaseMinAddress, alignment);
 513   size_t noaccess_prefix = ((aligned_heap_base_min_address + size) > (char*)OopEncodingHeapMax) ?
 514     noaccess_prefix_size(alignment) : 0;
 515 
 516   // Attempt to alloc at user-given address.
 517   if (!FLAG_IS_DEFAULT(HeapBaseMinAddress)) {
 518     try_reserve_heap(size + noaccess_prefix, alignment, page_size, aligned_heap_base_min_address);
 519     if (_base != aligned_heap_base_min_address) { // Enforce this exact address.
 520       release();
 521     }
 522   }
 523 
 524   // Keep heap at HeapBaseMinAddress.
 525   if (_base == nullptr) {
 526 
 527     // Try to allocate the heap at addresses that allow efficient oop compression.
 528     // Different schemes are tried, in order of decreasing optimization potential.
 529     //
 530     // For this, try_reserve_heap() is called with the desired heap base addresses.
 531     // A call into the os layer to allocate at a given address can return memory
 532     // at a different address than requested.  Still, this might be memory at a useful
 533     // address. try_reserve_heap() always returns this allocated memory, as only here
 534     // the criteria for a good heap are checked.
 535 
 536     // Attempt to allocate so that we can run without base and scale (32-Bit unscaled compressed oops).
 537     // Give it several tries from top of range to bottom.
 538     if (aligned_heap_base_min_address + size <= (char *)UnscaledOopHeapMax) {
 539 
 540       // Calc address range within we try to attach (range of possible start addresses).
 541       char* const highest_start = align_down((char *)UnscaledOopHeapMax - size, attach_point_alignment);
 542       char* const lowest_start  = align_up(aligned_heap_base_min_address, attach_point_alignment);
 543       try_reserve_range(highest_start, lowest_start, attach_point_alignment,
 544                         aligned_heap_base_min_address, (char *)UnscaledOopHeapMax, size, alignment, page_size);
 545     }
 546 
 547     // zerobased: Attempt to allocate in the lower 32G.
 548     // But leave room for the compressed class pointers, which is allocated above
 549     // the heap.
 550     char *zerobased_max = (char *)OopEncodingHeapMax;
 551     const size_t class_space = align_up(CompressedClassSpaceSize, alignment);
 552     // For small heaps, save some space for compressed class pointer
 553     // space so it can be decoded with no base.
 554     if (UseCompressedClassPointers && !UseSharedSpaces &&
 555         OopEncodingHeapMax <= KlassEncodingMetaspaceMax &&
 556         (uint64_t)(aligned_heap_base_min_address + size + class_space) <= KlassEncodingMetaspaceMax) {
 557       zerobased_max = (char *)OopEncodingHeapMax - class_space;
 558     }
 559 
 560     // Give it several tries from top of range to bottom.
 561     if (aligned_heap_base_min_address + size <= zerobased_max &&    // Zerobased theoretical possible.
 562         ((_base == nullptr) ||                        // No previous try succeeded.
 563          (_base + size > zerobased_max))) {        // Unscaled delivered an arbitrary address.
 564 
 565       // Calc address range within we try to attach (range of possible start addresses).
 566       char *const highest_start = align_down(zerobased_max - size, attach_point_alignment);
 567       // Need to be careful about size being guaranteed to be less
 568       // than UnscaledOopHeapMax due to type constraints.
 569       char *lowest_start = aligned_heap_base_min_address;
 570       uint64_t unscaled_end = UnscaledOopHeapMax - size;
 571       if (unscaled_end < UnscaledOopHeapMax) { // unscaled_end wrapped if size is large
 572         lowest_start = MAX2(lowest_start, (char*)unscaled_end);
 573       }
 574       lowest_start = align_up(lowest_start, attach_point_alignment);
 575       try_reserve_range(highest_start, lowest_start, attach_point_alignment,
 576                         aligned_heap_base_min_address, zerobased_max, size, alignment, page_size);
 577     }
 578 
 579     // Now we go for heaps with base != 0.  We need a noaccess prefix to efficiently
 580     // implement null checks.
 581     noaccess_prefix = noaccess_prefix_size(alignment);
 582 
 583     // Try to attach at addresses that are aligned to OopEncodingHeapMax. Disjointbase mode.
 584     char** addresses = get_attach_addresses_for_disjoint_mode();
 585     int i = 0;
 586     while (addresses[i] &&                                 // End of array not yet reached.
 587            ((_base == nullptr) ||                             // No previous try succeeded.
 588             (_base + size >  (char *)OopEncodingHeapMax && // Not zerobased or unscaled address.
 589              !CompressedOops::is_disjoint_heap_base_address((address)_base)))) {  // Not disjoint address.
 590       char* const attach_point = addresses[i];
 591       assert(attach_point >= aligned_heap_base_min_address, "Flag support broken");
 592       try_reserve_heap(size + noaccess_prefix, alignment, page_size, attach_point);
 593       i++;
 594     }
 595 
 596     // Last, desperate try without any placement.
 597     if (_base == nullptr) {
 598       log_trace(gc, heap, coops)("Trying to allocate at address nullptr heap of size " SIZE_FORMAT_X, size + noaccess_prefix);
 599       initialize(size + noaccess_prefix, alignment, page_size, nullptr, false);
 600     }
 601   }
 602 }
 603 
 604 ReservedHeapSpace::ReservedHeapSpace(size_t size, size_t alignment, size_t page_size, const char* heap_allocation_directory) : ReservedSpace() {
 605 
 606   if (size == 0) {
 607     return;
 608   }
 609 
 610   if (heap_allocation_directory != nullptr) {
 611     _fd_for_heap = os::create_file_for_heap(heap_allocation_directory);
 612     if (_fd_for_heap == -1) {
 613       vm_exit_during_initialization(
 614         err_msg("Could not create file for Heap at location %s", heap_allocation_directory));
 615     }
 616     // When there is a backing file directory for this space then whether
 617     // large pages are allocated is up to the filesystem of the backing file.
 618     // If requested, let the user know that explicit large pages can't be used.
 619     if (use_explicit_large_pages(page_size) && large_pages_requested()) {
 620       log_debug(gc, heap)("Cannot allocate explicit large pages for Java Heap when AllocateHeapAt option is set.");
 621     }
 622   }
 623 
 624   // Heap size should be aligned to alignment, too.
 625   guarantee(is_aligned(size, alignment), "set by caller");
 626 
 627   if (UseCompressedOops) {
 628     initialize_compressed_heap(size, alignment, page_size);
 629     if (_size > size) {
 630       // We allocated heap with noaccess prefix.
 631       // It can happen we get a zerobased/unscaled heap with noaccess prefix,
 632       // if we had to try at arbitrary address.
 633       establish_noaccess_prefix();
 634     }
 635   } else {
 636     initialize(size, alignment, page_size, nullptr, false);
 637   }
 638 
 639   assert(markWord::encode_pointer_as_mark(_base).decode_pointer() == _base,
 640          "area must be distinguishable from marks for mark-sweep");
 641   assert(markWord::encode_pointer_as_mark(&_base[size]).decode_pointer() == &_base[size],
 642          "area must be distinguishable from marks for mark-sweep");
 643 
 644   if (base() != nullptr) {
 645     MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap);
 646   }
 647 
 648   if (_fd_for_heap != -1) {
 649     ::close(_fd_for_heap);
 650   }
 651 }
 652 
 653 MemRegion ReservedHeapSpace::region() const {
 654   return MemRegion((HeapWord*)base(), (HeapWord*)end());
 655 }
 656 
 657 // Reserve space for code segment.  Same as Java heap only we mark this as
 658 // executable.
 659 ReservedCodeSpace::ReservedCodeSpace(size_t r_size,
 660                                      size_t rs_align,
 661                                      size_t rs_page_size) : ReservedSpace() {
 662   initialize(r_size, rs_align, rs_page_size, /*requested address*/ nullptr, /*executable*/ true);
 663   MemTracker::record_virtual_memory_type((address)base(), mtCode);
 664 }
 665 
 666 // VirtualSpace
 667 
 668 VirtualSpace::VirtualSpace() {
 669   _low_boundary           = nullptr;
 670   _high_boundary          = nullptr;
 671   _low                    = nullptr;
 672   _high                   = nullptr;
 673   _lower_high             = nullptr;
 674   _middle_high            = nullptr;
 675   _upper_high             = nullptr;
 676   _lower_high_boundary    = nullptr;
 677   _middle_high_boundary   = nullptr;
 678   _upper_high_boundary    = nullptr;
 679   _lower_alignment        = 0;
 680   _middle_alignment       = 0;
 681   _upper_alignment        = 0;
 682   _special                = false;
 683   _executable             = false;
 684 }
 685 
 686 
 687 bool VirtualSpace::initialize(ReservedSpace rs, size_t committed_size) {
 688   const size_t max_commit_granularity = os::page_size_for_region_unaligned(rs.size(), 1);
 689   return initialize_with_granularity(rs, committed_size, max_commit_granularity);
 690 }
 691 
 692 bool VirtualSpace::initialize_with_granularity(ReservedSpace rs, size_t committed_size, size_t max_commit_granularity) {
 693   if(!rs.is_reserved()) return false;  // allocation failed.
 694   assert(_low_boundary == nullptr, "VirtualSpace already initialized");
 695   assert(max_commit_granularity > 0, "Granularity must be non-zero.");
 696 
 697   _low_boundary  = rs.base();
 698   _high_boundary = low_boundary() + rs.size();
 699 
 700   _low = low_boundary();
 701   _high = low();
 702 
 703   _special = rs.special();
 704   _executable = rs.executable();
 705 
 706   // When a VirtualSpace begins life at a large size, make all future expansion
 707   // and shrinking occur aligned to a granularity of large pages.  This avoids
 708   // fragmentation of physical addresses that inhibits the use of large pages
 709   // by the OS virtual memory system.  Empirically,  we see that with a 4MB
 710   // page size, the only spaces that get handled this way are codecache and
 711   // the heap itself, both of which provide a substantial performance
 712   // boost in many benchmarks when covered by large pages.
 713   //
 714   // No attempt is made to force large page alignment at the very top and
 715   // bottom of the space if they are not aligned so already.
 716   _lower_alignment  = os::vm_page_size();
 717   _middle_alignment = max_commit_granularity;
 718   _upper_alignment  = os::vm_page_size();
 719 
 720   // End of each region
 721   _lower_high_boundary = align_up(low_boundary(), middle_alignment());
 722   _middle_high_boundary = align_down(high_boundary(), middle_alignment());
 723   _upper_high_boundary = high_boundary();
 724 
 725   // High address of each region
 726   _lower_high = low_boundary();
 727   _middle_high = lower_high_boundary();
 728   _upper_high = middle_high_boundary();
 729 
 730   // commit to initial size
 731   if (committed_size > 0) {
 732     if (!expand_by(committed_size)) {
 733       return false;
 734     }
 735   }
 736   return true;
 737 }
 738 
 739 
 740 VirtualSpace::~VirtualSpace() {
 741   release();
 742 }
 743 
 744 
 745 void VirtualSpace::release() {
 746   // This does not release memory it reserved.
 747   // Caller must release via rs.release();
 748   _low_boundary           = nullptr;
 749   _high_boundary          = nullptr;
 750   _low                    = nullptr;
 751   _high                   = nullptr;
 752   _lower_high             = nullptr;
 753   _middle_high            = nullptr;
 754   _upper_high             = nullptr;
 755   _lower_high_boundary    = nullptr;
 756   _middle_high_boundary   = nullptr;
 757   _upper_high_boundary    = nullptr;
 758   _lower_alignment        = 0;
 759   _middle_alignment       = 0;
 760   _upper_alignment        = 0;
 761   _special                = false;
 762   _executable             = false;
 763 }
 764 
 765 
 766 size_t VirtualSpace::committed_size() const {
 767   return pointer_delta(high(), low(), sizeof(char));
 768 }
 769 
 770 
 771 size_t VirtualSpace::reserved_size() const {
 772   return pointer_delta(high_boundary(), low_boundary(), sizeof(char));
 773 }
 774 
 775 
 776 size_t VirtualSpace::uncommitted_size()  const {
 777   return reserved_size() - committed_size();
 778 }
 779 
 780 size_t VirtualSpace::actual_committed_size() const {
 781   // Special VirtualSpaces commit all reserved space up front.
 782   if (special()) {
 783     return reserved_size();
 784   }
 785 
 786   size_t committed_low    = pointer_delta(_lower_high,  _low_boundary,         sizeof(char));
 787   size_t committed_middle = pointer_delta(_middle_high, _lower_high_boundary,  sizeof(char));
 788   size_t committed_high   = pointer_delta(_upper_high,  _middle_high_boundary, sizeof(char));
 789 
 790 #ifdef ASSERT
 791   size_t lower  = pointer_delta(_lower_high_boundary,  _low_boundary,         sizeof(char));
 792   size_t middle = pointer_delta(_middle_high_boundary, _lower_high_boundary,  sizeof(char));
 793   size_t upper  = pointer_delta(_upper_high_boundary,  _middle_high_boundary, sizeof(char));
 794 
 795   if (committed_high > 0) {
 796     assert(committed_low == lower, "Must be");
 797     assert(committed_middle == middle, "Must be");
 798   }
 799 
 800   if (committed_middle > 0) {
 801     assert(committed_low == lower, "Must be");
 802   }
 803   if (committed_middle < middle) {
 804     assert(committed_high == 0, "Must be");
 805   }
 806 
 807   if (committed_low < lower) {
 808     assert(committed_high == 0, "Must be");
 809     assert(committed_middle == 0, "Must be");
 810   }
 811 #endif
 812 
 813   return committed_low + committed_middle + committed_high;
 814 }
 815 
 816 
 817 bool VirtualSpace::contains(const void* p) const {
 818   return low() <= (const char*) p && (const char*) p < high();
 819 }
 820 
 821 static void pretouch_expanded_memory(void* start, void* end) {
 822   assert(is_aligned(start, os::vm_page_size()), "Unexpected alignment");
 823   assert(is_aligned(end,   os::vm_page_size()), "Unexpected alignment");
 824 
 825   os::pretouch_memory(start, end);
 826 }
 827 
 828 static bool commit_expanded(char* start, size_t size, size_t alignment, bool pre_touch, bool executable) {
 829   if (os::commit_memory(start, size, alignment, executable)) {
 830     if (pre_touch || AlwaysPreTouch) {
 831       pretouch_expanded_memory(start, start + size);
 832     }
 833     return true;
 834   }
 835 
 836   debug_only(warning(
 837       "INFO: os::commit_memory(" PTR_FORMAT ", " PTR_FORMAT
 838       " size=" SIZE_FORMAT ", executable=%d) failed",
 839       p2i(start), p2i(start + size), size, executable);)
 840 
 841   return false;
 842 }
 843 
 844 /*
 845    First we need to determine if a particular virtual space is using large
 846    pages.  This is done at the initialize function and only virtual spaces
 847    that are larger than LargePageSizeInBytes use large pages.  Once we
 848    have determined this, all expand_by and shrink_by calls must grow and
 849    shrink by large page size chunks.  If a particular request
 850    is within the current large page, the call to commit and uncommit memory
 851    can be ignored.  In the case that the low and high boundaries of this
 852    space is not large page aligned, the pages leading to the first large
 853    page address and the pages after the last large page address must be
 854    allocated with default pages.
 855 */
 856 bool VirtualSpace::expand_by(size_t bytes, bool pre_touch) {
 857   if (uncommitted_size() < bytes) {
 858     return false;
 859   }
 860 
 861   if (special()) {
 862     // don't commit memory if the entire space is pinned in memory
 863     _high += bytes;
 864     return true;
 865   }
 866 
 867   char* previous_high = high();
 868   char* unaligned_new_high = high() + bytes;
 869   assert(unaligned_new_high <= high_boundary(), "cannot expand by more than upper boundary");
 870 
 871   // Calculate where the new high for each of the regions should be.  If
 872   // the low_boundary() and high_boundary() are LargePageSizeInBytes aligned
 873   // then the unaligned lower and upper new highs would be the
 874   // lower_high() and upper_high() respectively.
 875   char* unaligned_lower_new_high =  MIN2(unaligned_new_high, lower_high_boundary());
 876   char* unaligned_middle_new_high = MIN2(unaligned_new_high, middle_high_boundary());
 877   char* unaligned_upper_new_high =  MIN2(unaligned_new_high, upper_high_boundary());
 878 
 879   // Align the new highs based on the regions alignment.  lower and upper
 880   // alignment will always be default page size.  middle alignment will be
 881   // LargePageSizeInBytes if the actual size of the virtual space is in
 882   // fact larger than LargePageSizeInBytes.
 883   char* aligned_lower_new_high =  align_up(unaligned_lower_new_high, lower_alignment());
 884   char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
 885   char* aligned_upper_new_high =  align_up(unaligned_upper_new_high, upper_alignment());
 886 
 887   // Determine which regions need to grow in this expand_by call.
 888   // If you are growing in the lower region, high() must be in that
 889   // region so calculate the size based on high().  For the middle and
 890   // upper regions, determine the starting point of growth based on the
 891   // location of high().  By getting the MAX of the region's low address
 892   // (or the previous region's high address) and high(), we can tell if it
 893   // is an intra or inter region growth.
 894   size_t lower_needs = 0;
 895   if (aligned_lower_new_high > lower_high()) {
 896     lower_needs = pointer_delta(aligned_lower_new_high, lower_high(), sizeof(char));
 897   }
 898   size_t middle_needs = 0;
 899   if (aligned_middle_new_high > middle_high()) {
 900     middle_needs = pointer_delta(aligned_middle_new_high, middle_high(), sizeof(char));
 901   }
 902   size_t upper_needs = 0;
 903   if (aligned_upper_new_high > upper_high()) {
 904     upper_needs = pointer_delta(aligned_upper_new_high, upper_high(), sizeof(char));
 905   }
 906 
 907   // Check contiguity.
 908   assert(low_boundary() <= lower_high() && lower_high() <= lower_high_boundary(),
 909          "high address must be contained within the region");
 910   assert(lower_high_boundary() <= middle_high() && middle_high() <= middle_high_boundary(),
 911          "high address must be contained within the region");
 912   assert(middle_high_boundary() <= upper_high() && upper_high() <= upper_high_boundary(),
 913          "high address must be contained within the region");
 914 
 915   // Commit regions
 916   if (lower_needs > 0) {
 917     assert(lower_high() + lower_needs <= lower_high_boundary(), "must not expand beyond region");
 918     if (!commit_expanded(lower_high(), lower_needs, _lower_alignment, pre_touch, _executable)) {
 919       return false;
 920     }
 921     _lower_high += lower_needs;
 922   }
 923 
 924   if (middle_needs > 0) {
 925     assert(middle_high() + middle_needs <= middle_high_boundary(), "must not expand beyond region");
 926     if (!commit_expanded(middle_high(), middle_needs, _middle_alignment, pre_touch, _executable)) {
 927       return false;
 928     }
 929     _middle_high += middle_needs;
 930   }
 931 
 932   if (upper_needs > 0) {
 933     assert(upper_high() + upper_needs <= upper_high_boundary(), "must not expand beyond region");
 934     if (!commit_expanded(upper_high(), upper_needs, _upper_alignment, pre_touch, _executable)) {
 935       return false;
 936     }
 937     _upper_high += upper_needs;
 938   }
 939 
 940   _high += bytes;
 941   return true;
 942 }
 943 
 944 // A page is uncommitted if the contents of the entire page is deemed unusable.
 945 // Continue to decrement the high() pointer until it reaches a page boundary
 946 // in which case that particular page can now be uncommitted.
 947 void VirtualSpace::shrink_by(size_t size) {
 948   if (committed_size() < size)
 949     fatal("Cannot shrink virtual space to negative size");
 950 
 951   if (special()) {
 952     // don't uncommit if the entire space is pinned in memory
 953     _high -= size;
 954     return;
 955   }
 956 
 957   char* unaligned_new_high = high() - size;
 958   assert(unaligned_new_high >= low_boundary(), "cannot shrink past lower boundary");
 959 
 960   // Calculate new unaligned address
 961   char* unaligned_upper_new_high =
 962     MAX2(unaligned_new_high, middle_high_boundary());
 963   char* unaligned_middle_new_high =
 964     MAX2(unaligned_new_high, lower_high_boundary());
 965   char* unaligned_lower_new_high =
 966     MAX2(unaligned_new_high, low_boundary());
 967 
 968   // Align address to region's alignment
 969   char* aligned_upper_new_high =  align_up(unaligned_upper_new_high, upper_alignment());
 970   char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
 971   char* aligned_lower_new_high =  align_up(unaligned_lower_new_high, lower_alignment());
 972 
 973   // Determine which regions need to shrink
 974   size_t upper_needs = 0;
 975   if (aligned_upper_new_high < upper_high()) {
 976     upper_needs =
 977       pointer_delta(upper_high(), aligned_upper_new_high, sizeof(char));
 978   }
 979   size_t middle_needs = 0;
 980   if (aligned_middle_new_high < middle_high()) {
 981     middle_needs =
 982       pointer_delta(middle_high(), aligned_middle_new_high, sizeof(char));
 983   }
 984   size_t lower_needs = 0;
 985   if (aligned_lower_new_high < lower_high()) {
 986     lower_needs =
 987       pointer_delta(lower_high(), aligned_lower_new_high, sizeof(char));
 988   }
 989 
 990   // Check contiguity.
 991   assert(middle_high_boundary() <= upper_high() &&
 992          upper_high() <= upper_high_boundary(),
 993          "high address must be contained within the region");
 994   assert(lower_high_boundary() <= middle_high() &&
 995          middle_high() <= middle_high_boundary(),
 996          "high address must be contained within the region");
 997   assert(low_boundary() <= lower_high() &&
 998          lower_high() <= lower_high_boundary(),
 999          "high address must be contained within the region");
1000 
1001   // Uncommit
1002   if (upper_needs > 0) {
1003     assert(middle_high_boundary() <= aligned_upper_new_high &&
1004            aligned_upper_new_high + upper_needs <= upper_high_boundary(),
1005            "must not shrink beyond region");
1006     if (!os::uncommit_memory(aligned_upper_new_high, upper_needs, _executable)) {
1007       debug_only(warning("os::uncommit_memory failed"));
1008       return;
1009     } else {
1010       _upper_high -= upper_needs;
1011     }
1012   }
1013   if (middle_needs > 0) {
1014     assert(lower_high_boundary() <= aligned_middle_new_high &&
1015            aligned_middle_new_high + middle_needs <= middle_high_boundary(),
1016            "must not shrink beyond region");
1017     if (!os::uncommit_memory(aligned_middle_new_high, middle_needs, _executable)) {
1018       debug_only(warning("os::uncommit_memory failed"));
1019       return;
1020     } else {
1021       _middle_high -= middle_needs;
1022     }
1023   }
1024   if (lower_needs > 0) {
1025     assert(low_boundary() <= aligned_lower_new_high &&
1026            aligned_lower_new_high + lower_needs <= lower_high_boundary(),
1027            "must not shrink beyond region");
1028     if (!os::uncommit_memory(aligned_lower_new_high, lower_needs, _executable)) {
1029       debug_only(warning("os::uncommit_memory failed"));
1030       return;
1031     } else {
1032       _lower_high -= lower_needs;
1033     }
1034   }
1035 
1036   _high -= size;
1037 }
1038 
1039 #ifndef PRODUCT
1040 void VirtualSpace::check_for_contiguity() {
1041   // Check contiguity.
1042   assert(low_boundary() <= lower_high() &&
1043          lower_high() <= lower_high_boundary(),
1044          "high address must be contained within the region");
1045   assert(lower_high_boundary() <= middle_high() &&
1046          middle_high() <= middle_high_boundary(),
1047          "high address must be contained within the region");
1048   assert(middle_high_boundary() <= upper_high() &&
1049          upper_high() <= upper_high_boundary(),
1050          "high address must be contained within the region");
1051   assert(low() >= low_boundary(), "low");
1052   assert(low_boundary() <= lower_high_boundary(), "lower high boundary");
1053   assert(upper_high_boundary() <= high_boundary(), "upper high boundary");
1054   assert(high() <= upper_high(), "upper high");
1055 }
1056 
1057 void VirtualSpace::print_on(outputStream* out) const {
1058   out->print   ("Virtual space:");
1059   if (special()) out->print(" (pinned in memory)");
1060   out->cr();
1061   out->print_cr(" - committed: " SIZE_FORMAT, committed_size());
1062   out->print_cr(" - reserved:  " SIZE_FORMAT, reserved_size());
1063   out->print_cr(" - [low, high]:     [" PTR_FORMAT ", " PTR_FORMAT "]",  p2i(low()), p2i(high()));
1064   out->print_cr(" - [low_b, high_b]: [" PTR_FORMAT ", " PTR_FORMAT "]",  p2i(low_boundary()), p2i(high_boundary()));
1065 }
1066 
1067 void VirtualSpace::print() const {
1068   print_on(tty);
1069 }
1070 
1071 #endif