1 /*
   2  * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "logging/log.hpp"
  27 #include "memory/resourceArea.hpp"
  28 #include "memory/virtualspace.hpp"
  29 #include "nmt/memTracker.hpp"
  30 #include "oops/compressedKlass.hpp"
  31 #include "oops/compressedOops.hpp"
  32 #include "oops/markWord.hpp"
  33 #include "oops/oop.inline.hpp"
  34 #include "runtime/globals_extension.hpp"
  35 #include "runtime/java.hpp"
  36 #include "runtime/os.hpp"
  37 #include "utilities/align.hpp"
  38 #include "utilities/formatBuffer.hpp"
  39 #include "utilities/powerOfTwo.hpp"
  40 
  41 // ReservedSpace
  42 
  43 // Dummy constructor
  44 ReservedSpace::ReservedSpace() : _base(nullptr), _size(0), _noaccess_prefix(0),
  45     _alignment(0), _special(false), _fd_for_heap(-1), _executable(false) {
  46 }
  47 
  48 ReservedSpace::ReservedSpace(size_t size) : _fd_for_heap(-1) {
  49   // Want to use large pages where possible. If the size is
  50   // not large page aligned the mapping will be a mix of
  51   // large and normal pages.
  52   size_t page_size = os::page_size_for_region_unaligned(size, 1);
  53   size_t alignment = os::vm_allocation_granularity();
  54   initialize(size, alignment, page_size, nullptr, false);
  55 }
  56 
  57 ReservedSpace::ReservedSpace(size_t size, size_t preferred_page_size) : _fd_for_heap(-1) {
  58   // When a page size is given we don't want to mix large
  59   // and normal pages. If the size is not a multiple of the
  60   // page size it will be aligned up to achieve this.
  61   size_t alignment = os::vm_allocation_granularity();;
  62   if (preferred_page_size != os::vm_page_size()) {
  63     alignment = MAX2(preferred_page_size, alignment);
  64     size = align_up(size, alignment);
  65   }
  66   initialize(size, alignment, preferred_page_size, nullptr, false);
  67 }
  68 
  69 ReservedSpace::ReservedSpace(size_t size,
  70                              size_t alignment,
  71                              size_t page_size,
  72                              char* requested_address) : _fd_for_heap(-1) {
  73   initialize(size, alignment, page_size, requested_address, false);
  74 }
  75 
  76 ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment, size_t page_size,
  77                              bool special, bool executable) : _fd_for_heap(-1) {
  78   assert((size % os::vm_allocation_granularity()) == 0,
  79          "size not allocation aligned");
  80   initialize_members(base, size, alignment, page_size, special, executable);
  81 }
  82 
  83 // Helper method
  84 static char* attempt_map_or_reserve_memory_at(char* base, size_t size, int fd, bool executable) {
  85   if (fd != -1) {
  86     return os::attempt_map_memory_to_file_at(base, size, fd);
  87   }
  88   return os::attempt_reserve_memory_at(base, size, executable);
  89 }
  90 
  91 // Helper method
  92 static char* map_or_reserve_memory(size_t size, int fd, bool executable) {
  93   if (fd != -1) {
  94     return os::map_memory_to_file(size, fd);
  95   }
  96   return os::reserve_memory(size, executable);
  97 }
  98 
  99 // Helper method
 100 static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int fd, bool executable) {
 101   if (fd != -1) {
 102     return os::map_memory_to_file_aligned(size, alignment, fd);
 103   }
 104   return os::reserve_memory_aligned(size, alignment, executable);
 105 }
 106 
 107 // Helper method
 108 static void unmap_or_release_memory(char* base, size_t size, bool is_file_mapped) {
 109   if (is_file_mapped) {
 110     if (!os::unmap_memory(base, size)) {
 111       fatal("os::unmap_memory failed");
 112     }
 113   } else if (!os::release_memory(base, size)) {
 114     fatal("os::release_memory failed");
 115   }
 116 }
 117 
 118 // Helper method
 119 static bool failed_to_reserve_as_requested(char* base, char* requested_address) {
 120   if (base == requested_address || requested_address == nullptr) {
 121     return false; // did not fail
 122   }
 123 
 124   if (base != nullptr) {
 125     // Different reserve address may be acceptable in other cases
 126     // but for compressed oops heap should be at requested address.
 127     assert(UseCompressedOops, "currently requested address used only for compressed oops");
 128     log_debug(gc, heap, coops)("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, p2i(base), p2i(requested_address));
 129   }
 130   return true;
 131 }
 132 
 133 static bool use_explicit_large_pages(size_t page_size) {
 134   return !os::can_commit_large_page_memory() &&
 135          page_size != os::vm_page_size();
 136 }
 137 
 138 static bool large_pages_requested() {
 139   return UseLargePages &&
 140          (!FLAG_IS_DEFAULT(UseLargePages) || !FLAG_IS_DEFAULT(LargePageSizeInBytes));
 141 }
 142 
 143 static void log_on_large_pages_failure(char* req_addr, size_t bytes) {
 144   if (large_pages_requested()) {
 145     // Compressed oops logging.
 146     log_debug(gc, heap, coops)("Reserve regular memory without large pages");
 147     // JVM style warning that we did not succeed in using large pages.
 148     char msg[128];
 149     jio_snprintf(msg, sizeof(msg), "Failed to reserve and commit memory using large pages. "
 150                                    "req_addr: " PTR_FORMAT " bytes: " SIZE_FORMAT,
 151                                    req_addr, bytes);
 152     warning("%s", msg);
 153   }
 154 }
 155 
 156 static char* reserve_memory(char* requested_address, const size_t size,
 157                             const size_t alignment, int fd, bool exec) {
 158   char* base;
 159   // If the memory was requested at a particular address, use
 160   // os::attempt_reserve_memory_at() to avoid mapping over something
 161   // important.  If the reservation fails, return null.
 162   if (requested_address != 0) {
 163     assert(is_aligned(requested_address, alignment),
 164            "Requested address " PTR_FORMAT " must be aligned to " SIZE_FORMAT,
 165            p2i(requested_address), alignment);
 166     base = attempt_map_or_reserve_memory_at(requested_address, size, fd, exec);
 167   } else {
 168     // Optimistically assume that the OS returns an aligned base pointer.
 169     // When reserving a large address range, most OSes seem to align to at
 170     // least 64K.
 171     base = map_or_reserve_memory(size, fd, exec);
 172     // Check alignment constraints. This is only needed when there is
 173     // no requested address.
 174     if (!is_aligned(base, alignment)) {
 175       // Base not aligned, retry.
 176       unmap_or_release_memory(base, size, fd != -1 /*is_file_mapped*/);
 177       // Map using the requested alignment.
 178       base = map_or_reserve_memory_aligned(size, alignment, fd, exec);
 179     }
 180   }
 181 
 182   return base;
 183 }
 184 
 185 static char* reserve_memory_special(char* requested_address, const size_t size,
 186                                     const size_t alignment, const size_t page_size, bool exec) {
 187 
 188   log_trace(pagesize)("Attempt special mapping: size: " SIZE_FORMAT "%s, "
 189                       "alignment: " SIZE_FORMAT "%s",
 190                       byte_size_in_exact_unit(size), exact_unit_for_byte_size(size),
 191                       byte_size_in_exact_unit(alignment), exact_unit_for_byte_size(alignment));
 192 
 193   char* base = os::reserve_memory_special(size, alignment, page_size, requested_address, exec);
 194   if (base != nullptr) {
 195     // Check alignment constraints.
 196     assert(is_aligned(base, alignment),
 197            "reserve_memory_special() returned an unaligned address, base: " PTR_FORMAT
 198            " alignment: " SIZE_FORMAT_X,
 199            p2i(base), alignment);
 200   }
 201   return base;
 202 }
 203 
 204 void ReservedSpace::clear_members() {
 205   initialize_members(nullptr, 0, 0, 0, false, false);
 206 }
 207 
 208 void ReservedSpace::initialize_members(char* base, size_t size, size_t alignment,
 209                                        size_t page_size, bool special, bool executable) {
 210   _base = base;
 211   _size = size;
 212   _alignment = alignment;
 213   _page_size = page_size;
 214   _special = special;
 215   _executable = executable;
 216   _noaccess_prefix = 0;
 217 }
 218 
 219 void ReservedSpace::reserve(size_t size,
 220                             size_t alignment,
 221                             size_t page_size,
 222                             char* requested_address,
 223                             bool executable) {
 224   assert(is_aligned(size, alignment), "Size must be aligned to the requested alignment");
 225 
 226   // There are basically three different cases that we need to handle below:
 227   // 1. Mapping backed by a file
 228   // 2. Mapping backed by explicit large pages
 229   // 3. Mapping backed by normal pages or transparent huge pages
 230   // The first two have restrictions that requires the whole mapping to be
 231   // committed up front. To record this the ReservedSpace is marked 'special'.
 232 
 233   // == Case 1 ==
 234   if (_fd_for_heap != -1) {
 235     // When there is a backing file directory for this space then whether
 236     // large pages are allocated is up to the filesystem of the backing file.
 237     // So UseLargePages is not taken into account for this reservation.
 238     char* base = reserve_memory(requested_address, size, alignment, _fd_for_heap, executable);
 239     if (base != nullptr) {
 240       initialize_members(base, size, alignment, os::vm_page_size(), true, executable);
 241     }
 242     // Always return, not possible to fall back to reservation not using a file.
 243     return;
 244   }
 245 
 246   // == Case 2 ==
 247   if (use_explicit_large_pages(page_size)) {
 248     // System can't commit large pages i.e. use transparent huge pages and
 249     // the caller requested large pages. To satisfy this request we use
 250     // explicit large pages and these have to be committed up front to ensure
 251     // no reservations are lost.
 252     do {
 253       char* base = reserve_memory_special(requested_address, size, alignment, page_size, executable);
 254       if (base != nullptr) {
 255         // Successful reservation using large pages.
 256         initialize_members(base, size, alignment, page_size, true, executable);
 257         return;
 258       }
 259       page_size = os::page_sizes().next_smaller(page_size);
 260     } while (page_size > os::vm_page_size());
 261 
 262     // Failed to reserve explicit large pages, do proper logging.
 263     log_on_large_pages_failure(requested_address, size);
 264     // Now fall back to normal reservation.
 265     assert(page_size == os::vm_page_size(), "inv");
 266   }
 267 
 268   // == Case 3 ==
 269   char* base = reserve_memory(requested_address, size, alignment, -1, executable);
 270   if (base != nullptr) {
 271     // Successful mapping.
 272     initialize_members(base, size, alignment, page_size, false, executable);
 273   }
 274 }
 275 
 276 void ReservedSpace::initialize(size_t size,
 277                                size_t alignment,
 278                                size_t page_size,
 279                                char* requested_address,
 280                                bool executable) {
 281   const size_t granularity = os::vm_allocation_granularity();
 282   assert((size & (granularity - 1)) == 0,
 283          "size not aligned to os::vm_allocation_granularity()");
 284   assert((alignment & (granularity - 1)) == 0,
 285          "alignment not aligned to os::vm_allocation_granularity()");
 286   assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
 287          "not a power of 2");
 288   assert(page_size >= os::vm_page_size(), "Invalid page size");
 289   assert(is_power_of_2(page_size), "Invalid page size");
 290 
 291   clear_members();
 292 
 293   if (size == 0) {
 294     return;
 295   }
 296 
 297   // Adjust alignment to not be 0.
 298   alignment = MAX2(alignment, os::vm_page_size());
 299 
 300   // Reserve the memory.
 301   reserve(size, alignment, page_size, requested_address, executable);
 302 
 303   // Check that the requested address is used if given.
 304   if (failed_to_reserve_as_requested(_base, requested_address)) {
 305     // OS ignored the requested address, release the reservation.
 306     release();
 307     return;
 308   }
 309 }
 310 
 311 ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment) {
 312   assert(partition_size <= size(), "partition failed");
 313   ReservedSpace result(base(), partition_size, alignment, page_size(), special(), executable());
 314   return result;
 315 }
 316 
 317 ReservedSpace ReservedSpace::last_part(size_t partition_size, size_t alignment) {
 318   assert(partition_size <= size(), "partition failed");
 319   ReservedSpace result(base() + partition_size, size() - partition_size,
 320                        alignment, page_size(), special(), executable());
 321   return result;
 322 }
 323 
 324 ReservedSpace ReservedSpace::partition(size_t offset, size_t partition_size, size_t alignment) {
 325   assert(offset + partition_size <= size(), "partition failed");
 326   ReservedSpace result(base() + offset, partition_size, alignment, page_size(), special(), executable());
 327   return result;
 328 }
 329 
 330 size_t ReservedSpace::page_align_size_up(size_t size) {
 331   return align_up(size, os::vm_page_size());
 332 }
 333 
 334 
 335 size_t ReservedSpace::page_align_size_down(size_t size) {
 336   return align_down(size, os::vm_page_size());
 337 }
 338 
 339 
 340 size_t ReservedSpace::allocation_align_size_up(size_t size) {
 341   return align_up(size, os::vm_allocation_granularity());
 342 }
 343 
 344 void ReservedSpace::release() {
 345   if (is_reserved()) {
 346     char *real_base = _base - _noaccess_prefix;
 347     const size_t real_size = _size + _noaccess_prefix;
 348     if (special()) {
 349       if (_fd_for_heap != -1) {
 350         os::unmap_memory(real_base, real_size);
 351       } else {
 352         os::release_memory_special(real_base, real_size);
 353       }
 354     } else{
 355       os::release_memory(real_base, real_size);
 356     }
 357     clear_members();
 358   }
 359 }
 360 
 361 // Put a ReservedSpace over an existing range
 362 ReservedSpace ReservedSpace::space_for_range(char* base, size_t size, size_t alignment,
 363                                              size_t page_size, bool special, bool executable) {
 364   assert(is_aligned(base, os::vm_allocation_granularity()), "Unaligned base");
 365   assert(is_aligned(size, os::vm_page_size()), "Unaligned size");
 366   assert(os::page_sizes().contains(page_size), "Invalid pagesize");
 367   ReservedSpace space;
 368   space.initialize_members(base, size, alignment, page_size, special, executable);
 369   return space;
 370 }
 371 
 372 static size_t noaccess_prefix_size(size_t alignment) {
 373   return lcm(os::vm_page_size(), alignment);
 374 }
 375 
 376 void ReservedHeapSpace::establish_noaccess_prefix() {
 377   assert(_alignment >= os::vm_page_size(), "must be at least page size big");
 378   _noaccess_prefix = noaccess_prefix_size(_alignment);
 379 
 380   if (base() && base() + _size > (char *)OopEncodingHeapMax) {
 381     if (true
 382         WIN64_ONLY(&& !UseLargePages)
 383         AIX_ONLY(&& os::vm_page_size() != 64*K)) {
 384       // Protect memory at the base of the allocated region.
 385       // If special, the page was committed (only matters on windows)
 386       if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, _special)) {
 387         fatal("cannot protect protection page");
 388       }
 389       log_debug(gc, heap, coops)("Protected page at the reserved heap base: "
 390                                  PTR_FORMAT " / " INTX_FORMAT " bytes",
 391                                  p2i(_base),
 392                                  _noaccess_prefix);
 393       assert(CompressedOops::use_implicit_null_checks() == true, "not initialized?");
 394     } else {
 395       CompressedOops::set_use_implicit_null_checks(false);
 396     }
 397   }
 398 
 399   _base += _noaccess_prefix;
 400   _size -= _noaccess_prefix;
 401   assert(((uintptr_t)_base % _alignment == 0), "must be exactly of required alignment");
 402 }
 403 
 404 // Tries to allocate memory of size 'size' at address requested_address with alignment 'alignment'.
 405 // Does not check whether the reserved memory actually is at requested_address, as the memory returned
 406 // might still fulfill the wishes of the caller.
 407 // Assures the memory is aligned to 'alignment'.
 408 // NOTE: If ReservedHeapSpace already points to some reserved memory this is freed, first.
 409 void ReservedHeapSpace::try_reserve_heap(size_t size,
 410                                          size_t alignment,
 411                                          size_t page_size,
 412                                          char* requested_address) {
 413   if (_base != nullptr) {
 414     // We tried before, but we didn't like the address delivered.
 415     release();
 416   }
 417 
 418   // Try to reserve the memory for the heap.
 419   log_trace(gc, heap, coops)("Trying to allocate at address " PTR_FORMAT
 420                              " heap of size " SIZE_FORMAT_X,
 421                              p2i(requested_address),
 422                              size);
 423 
 424   reserve(size, alignment, page_size, requested_address, false);
 425 
 426   // Check alignment constraints.
 427   if (is_reserved() && !is_aligned(_base, _alignment)) {
 428     // Base not aligned, retry.
 429     release();
 430   }
 431 }
 432 
 433 void ReservedHeapSpace::try_reserve_range(char *highest_start,
 434                                           char *lowest_start,
 435                                           size_t attach_point_alignment,
 436                                           char *aligned_heap_base_min_address,
 437                                           char *upper_bound,
 438                                           size_t size,
 439                                           size_t alignment,
 440                                           size_t page_size) {
 441   const size_t attach_range = highest_start - lowest_start;
 442   // Cap num_attempts at possible number.
 443   // At least one is possible even for 0 sized attach range.
 444   const uint64_t num_attempts_possible = (attach_range / attach_point_alignment) + 1;
 445   const uint64_t num_attempts_to_try   = MIN2((uint64_t)HeapSearchSteps, num_attempts_possible);
 446 
 447   const size_t stepsize = (attach_range == 0) ? // Only one try.
 448     (size_t) highest_start : align_up(attach_range / num_attempts_to_try, attach_point_alignment);
 449 
 450   // Try attach points from top to bottom.
 451   char* attach_point = highest_start;
 452   while (attach_point >= lowest_start  &&
 453          attach_point <= highest_start &&  // Avoid wrap around.
 454          ((_base == nullptr) ||
 455           (_base < aligned_heap_base_min_address || _base + size > upper_bound))) {
 456     try_reserve_heap(size, alignment, page_size, attach_point);
 457     attach_point -= stepsize;
 458   }
 459 }
 460 
 461 #define SIZE_64K  ((uint64_t) UCONST64(      0x10000))
 462 #define SIZE_256M ((uint64_t) UCONST64(   0x10000000))
 463 #define SIZE_32G  ((uint64_t) UCONST64(  0x800000000))
 464 
 465 // Helper for heap allocation. Returns an array with addresses
 466 // (OS-specific) which are suited for disjoint base mode. Array is
 467 // null terminated.
 468 static char** get_attach_addresses_for_disjoint_mode() {
 469   static uint64_t addresses[] = {
 470      2 * SIZE_32G,
 471      3 * SIZE_32G,
 472      4 * SIZE_32G,
 473      8 * SIZE_32G,
 474     10 * SIZE_32G,
 475      1 * SIZE_64K * SIZE_32G,
 476      2 * SIZE_64K * SIZE_32G,
 477      3 * SIZE_64K * SIZE_32G,
 478      4 * SIZE_64K * SIZE_32G,
 479     16 * SIZE_64K * SIZE_32G,
 480     32 * SIZE_64K * SIZE_32G,
 481     34 * SIZE_64K * SIZE_32G,
 482     0
 483   };
 484 
 485   // Sort out addresses smaller than HeapBaseMinAddress. This assumes
 486   // the array is sorted.
 487   uint i = 0;
 488   while (addresses[i] != 0 &&
 489          (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) {
 490     i++;
 491   }
 492   uint start = i;
 493 
 494   // Avoid more steps than requested.
 495   i = 0;
 496   while (addresses[start+i] != 0) {
 497     if (i == HeapSearchSteps) {
 498       addresses[start+i] = 0;
 499       break;
 500     }
 501     i++;
 502   }
 503 
 504   return (char**) &addresses[start];
 505 }
 506 
 507 void ReservedHeapSpace::initialize_compressed_heap(const size_t size, size_t alignment, size_t page_size) {
 508   guarantee(size + noaccess_prefix_size(alignment) <= OopEncodingHeapMax,
 509             "can not allocate compressed oop heap for this size");
 510   guarantee(alignment == MAX2(alignment, os::vm_page_size()), "alignment too small");
 511 
 512   const size_t granularity = os::vm_allocation_granularity();
 513   assert((size & (granularity - 1)) == 0,
 514          "size not aligned to os::vm_allocation_granularity()");
 515   assert((alignment & (granularity - 1)) == 0,
 516          "alignment not aligned to os::vm_allocation_granularity()");
 517   assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
 518          "not a power of 2");
 519 
 520   // The necessary attach point alignment for generated wish addresses.
 521   // This is needed to increase the chance of attaching for mmap and shmat.
 522   // AIX is the only platform that uses System V shm for reserving virtual memory.
 523   // In this case, the required alignment of the allocated size (64K) and the alignment
 524   // of possible start points of the memory region (256M) differ.
 525   // This is not reflected by os_allocation_granularity().
 526   // The logic here is dual to the one in pd_reserve_memory in os_aix.cpp
 527   const size_t os_attach_point_alignment =
 528     AIX_ONLY(os::vm_page_size() == 4*K ? 4*K : 256*M)
 529     NOT_AIX(os::vm_allocation_granularity());
 530 
 531   const size_t attach_point_alignment = lcm(alignment, os_attach_point_alignment);
 532 
 533   char *aligned_heap_base_min_address = (char *)align_up((void *)HeapBaseMinAddress, alignment);
 534   size_t noaccess_prefix = (((aligned_heap_base_min_address + size) > (char*)OopEncodingHeapMax) || UseCompatibleCompressedOops) ?
 535     noaccess_prefix_size(alignment) : 0;
 536 
 537   // Attempt to alloc at user-given address.
 538   if (!FLAG_IS_DEFAULT(HeapBaseMinAddress) || UseCompatibleCompressedOops) {
 539     try_reserve_heap(size + noaccess_prefix, alignment, page_size, aligned_heap_base_min_address);
 540     if (_base != aligned_heap_base_min_address) { // Enforce this exact address.
 541       release();
 542     }
 543   }
 544 
 545   // Keep heap at HeapBaseMinAddress.
 546   if (_base == nullptr) {
 547 
 548     // Try to allocate the heap at addresses that allow efficient oop compression.
 549     // Different schemes are tried, in order of decreasing optimization potential.
 550     //
 551     // For this, try_reserve_heap() is called with the desired heap base addresses.
 552     // A call into the os layer to allocate at a given address can return memory
 553     // at a different address than requested.  Still, this might be memory at a useful
 554     // address. try_reserve_heap() always returns this allocated memory, as only here
 555     // the criteria for a good heap are checked.
 556 
 557     // Attempt to allocate so that we can run without base and scale (32-Bit unscaled compressed oops).
 558     // Give it several tries from top of range to bottom.
 559     if (aligned_heap_base_min_address + size <= (char *)UnscaledOopHeapMax && !UseCompatibleCompressedOops) {
 560 
 561       // Calc address range within we try to attach (range of possible start addresses).
 562       char* const highest_start = align_down((char *)UnscaledOopHeapMax - size, attach_point_alignment);
 563       char* const lowest_start  = align_up(aligned_heap_base_min_address, attach_point_alignment);
 564       try_reserve_range(highest_start, lowest_start, attach_point_alignment,
 565                         aligned_heap_base_min_address, (char *)UnscaledOopHeapMax, size, alignment, page_size);
 566     }
 567 
 568     // zerobased: Attempt to allocate in the lower 32G.
 569     char *zerobased_max = (char *)OopEncodingHeapMax;
 570 
 571     // Give it several tries from top of range to bottom.
 572     if (!UseCompatibleCompressedOops &&
 573         aligned_heap_base_min_address + size <= zerobased_max &&    // Zerobased theoretical possible.
 574         ((_base == nullptr) ||                        // No previous try succeeded.
 575          (_base + size > zerobased_max))) {        // Unscaled delivered an arbitrary address.
 576 
 577       // Calc address range within we try to attach (range of possible start addresses).
 578       char *const highest_start = align_down(zerobased_max - size, attach_point_alignment);
 579       // Need to be careful about size being guaranteed to be less
 580       // than UnscaledOopHeapMax due to type constraints.
 581       char *lowest_start = aligned_heap_base_min_address;
 582       uint64_t unscaled_end = UnscaledOopHeapMax - size;
 583       if (unscaled_end < UnscaledOopHeapMax) { // unscaled_end wrapped if size is large
 584         lowest_start = MAX2(lowest_start, (char*)unscaled_end);
 585       }
 586       lowest_start = align_up(lowest_start, attach_point_alignment);
 587       try_reserve_range(highest_start, lowest_start, attach_point_alignment,
 588                         aligned_heap_base_min_address, zerobased_max, size, alignment, page_size);
 589     }
 590 
 591     // Now we go for heaps with base != 0.  We need a noaccess prefix to efficiently
 592     // implement null checks.
 593     noaccess_prefix = noaccess_prefix_size(alignment);
 594 
 595     // Try to attach at addresses that are aligned to OopEncodingHeapMax. Disjointbase mode.
 596     char** addresses = get_attach_addresses_for_disjoint_mode();
 597     int i = 0;
 598     while (addresses[i] &&                                 // End of array not yet reached.
 599            ((_base == nullptr) ||                             // No previous try succeeded.
 600             (_base + size >  (char *)OopEncodingHeapMax && // Not zerobased or unscaled address.
 601              !CompressedOops::is_disjoint_heap_base_address((address)_base)))) {  // Not disjoint address.
 602       char* const attach_point = addresses[i];
 603       assert(attach_point >= aligned_heap_base_min_address, "Flag support broken");
 604       try_reserve_heap(size + noaccess_prefix, alignment, page_size, attach_point);
 605       i++;
 606     }
 607 
 608     // Last, desperate try without any placement.
 609     if (_base == nullptr) {
 610       log_trace(gc, heap, coops)("Trying to allocate at address null heap of size " SIZE_FORMAT_X, size + noaccess_prefix);
 611       initialize(size + noaccess_prefix, alignment, page_size, nullptr, false);
 612     }
 613   }
 614 }
 615 
 616 ReservedHeapSpace::ReservedHeapSpace(size_t size, size_t alignment, size_t page_size, const char* heap_allocation_directory) : ReservedSpace() {
 617 
 618   if (size == 0) {
 619     return;
 620   }
 621 
 622   if (heap_allocation_directory != nullptr) {
 623     _fd_for_heap = os::create_file_for_heap(heap_allocation_directory);
 624     if (_fd_for_heap == -1) {
 625       vm_exit_during_initialization(
 626         err_msg("Could not create file for Heap at location %s", heap_allocation_directory));
 627     }
 628     // When there is a backing file directory for this space then whether
 629     // large pages are allocated is up to the filesystem of the backing file.
 630     // If requested, let the user know that explicit large pages can't be used.
 631     if (use_explicit_large_pages(page_size) && large_pages_requested()) {
 632       log_debug(gc, heap)("Cannot allocate explicit large pages for Java Heap when AllocateHeapAt option is set.");
 633     }
 634   }
 635 
 636   // Heap size should be aligned to alignment, too.
 637   guarantee(is_aligned(size, alignment), "set by caller");
 638 
 639   if (UseCompressedOops) {
 640     initialize_compressed_heap(size, alignment, page_size);
 641     if (_size > size || UseCompatibleCompressedOops) {
 642       // We allocated heap with noaccess prefix.
 643       // It can happen we get a zerobased/unscaled heap with noaccess prefix,
 644       // if we had to try at arbitrary address.
 645       establish_noaccess_prefix();
 646     }
 647   } else {
 648     initialize(size, alignment, page_size, nullptr, false);
 649   }
 650 
 651   assert(markWord::encode_pointer_as_mark(_base).decode_pointer() == _base,
 652          "area must be distinguishable from marks for mark-sweep");
 653   assert(markWord::encode_pointer_as_mark(&_base[size]).decode_pointer() == &_base[size],
 654          "area must be distinguishable from marks for mark-sweep");
 655 
 656   if (base() != nullptr) {
 657     MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap);
 658   }
 659 
 660   if (_fd_for_heap != -1) {
 661     ::close(_fd_for_heap);
 662   }
 663 }
 664 
 665 MemRegion ReservedHeapSpace::region() const {
 666   return MemRegion((HeapWord*)base(), (HeapWord*)end());
 667 }
 668 
 669 // Reserve space for code segment.  Same as Java heap only we mark this as
 670 // executable.
 671 ReservedCodeSpace::ReservedCodeSpace(size_t r_size,
 672                                      size_t rs_align,
 673                                      size_t rs_page_size) : ReservedSpace() {
 674   initialize(r_size, rs_align, rs_page_size, /*requested address*/ nullptr, /*executable*/ true);
 675   MemTracker::record_virtual_memory_type((address)base(), mtCode);
 676 }
 677 
 678 // VirtualSpace
 679 
 680 VirtualSpace::VirtualSpace() {
 681   _low_boundary           = nullptr;
 682   _high_boundary          = nullptr;
 683   _low                    = nullptr;
 684   _high                   = nullptr;
 685   _lower_high             = nullptr;
 686   _middle_high            = nullptr;
 687   _upper_high             = nullptr;
 688   _lower_high_boundary    = nullptr;
 689   _middle_high_boundary   = nullptr;
 690   _upper_high_boundary    = nullptr;
 691   _lower_alignment        = 0;
 692   _middle_alignment       = 0;
 693   _upper_alignment        = 0;
 694   _special                = false;
 695   _executable             = false;
 696 }
 697 
 698 
 699 bool VirtualSpace::initialize(ReservedSpace rs, size_t committed_size) {
 700   const size_t max_commit_granularity = os::page_size_for_region_unaligned(rs.size(), 1);
 701   return initialize_with_granularity(rs, committed_size, max_commit_granularity);
 702 }
 703 
 704 bool VirtualSpace::initialize_with_granularity(ReservedSpace rs, size_t committed_size, size_t max_commit_granularity) {
 705   if(!rs.is_reserved()) return false;  // allocation failed.
 706   assert(_low_boundary == nullptr, "VirtualSpace already initialized");
 707   assert(max_commit_granularity > 0, "Granularity must be non-zero.");
 708 
 709   _low_boundary  = rs.base();
 710   _high_boundary = low_boundary() + rs.size();
 711 
 712   _low = low_boundary();
 713   _high = low();
 714 
 715   _special = rs.special();
 716   _executable = rs.executable();
 717 
 718   // When a VirtualSpace begins life at a large size, make all future expansion
 719   // and shrinking occur aligned to a granularity of large pages.  This avoids
 720   // fragmentation of physical addresses that inhibits the use of large pages
 721   // by the OS virtual memory system.  Empirically,  we see that with a 4MB
 722   // page size, the only spaces that get handled this way are codecache and
 723   // the heap itself, both of which provide a substantial performance
 724   // boost in many benchmarks when covered by large pages.
 725   //
 726   // No attempt is made to force large page alignment at the very top and
 727   // bottom of the space if they are not aligned so already.
 728   _lower_alignment  = os::vm_page_size();
 729   _middle_alignment = max_commit_granularity;
 730   _upper_alignment  = os::vm_page_size();
 731 
 732   // End of each region
 733   _lower_high_boundary = align_up(low_boundary(), middle_alignment());
 734   _middle_high_boundary = align_down(high_boundary(), middle_alignment());
 735   _upper_high_boundary = high_boundary();
 736 
 737   // High address of each region
 738   _lower_high = low_boundary();
 739   _middle_high = lower_high_boundary();
 740   _upper_high = middle_high_boundary();
 741 
 742   // commit to initial size
 743   if (committed_size > 0) {
 744     if (!expand_by(committed_size)) {
 745       return false;
 746     }
 747   }
 748   return true;
 749 }
 750 
 751 
 752 VirtualSpace::~VirtualSpace() {
 753   release();
 754 }
 755 
 756 
 757 void VirtualSpace::release() {
 758   // This does not release memory it reserved.
 759   // Caller must release via rs.release();
 760   _low_boundary           = nullptr;
 761   _high_boundary          = nullptr;
 762   _low                    = nullptr;
 763   _high                   = nullptr;
 764   _lower_high             = nullptr;
 765   _middle_high            = nullptr;
 766   _upper_high             = nullptr;
 767   _lower_high_boundary    = nullptr;
 768   _middle_high_boundary   = nullptr;
 769   _upper_high_boundary    = nullptr;
 770   _lower_alignment        = 0;
 771   _middle_alignment       = 0;
 772   _upper_alignment        = 0;
 773   _special                = false;
 774   _executable             = false;
 775 }
 776 
 777 
 778 size_t VirtualSpace::committed_size() const {
 779   return pointer_delta(high(), low(), sizeof(char));
 780 }
 781 
 782 
 783 size_t VirtualSpace::reserved_size() const {
 784   return pointer_delta(high_boundary(), low_boundary(), sizeof(char));
 785 }
 786 
 787 
 788 size_t VirtualSpace::uncommitted_size()  const {
 789   return reserved_size() - committed_size();
 790 }
 791 
 792 size_t VirtualSpace::actual_committed_size() const {
 793   // Special VirtualSpaces commit all reserved space up front.
 794   if (special()) {
 795     return reserved_size();
 796   }
 797 
 798   size_t committed_low    = pointer_delta(_lower_high,  _low_boundary,         sizeof(char));
 799   size_t committed_middle = pointer_delta(_middle_high, _lower_high_boundary,  sizeof(char));
 800   size_t committed_high   = pointer_delta(_upper_high,  _middle_high_boundary, sizeof(char));
 801 
 802 #ifdef ASSERT
 803   size_t lower  = pointer_delta(_lower_high_boundary,  _low_boundary,         sizeof(char));
 804   size_t middle = pointer_delta(_middle_high_boundary, _lower_high_boundary,  sizeof(char));
 805   size_t upper  = pointer_delta(_upper_high_boundary,  _middle_high_boundary, sizeof(char));
 806 
 807   if (committed_high > 0) {
 808     assert(committed_low == lower, "Must be");
 809     assert(committed_middle == middle, "Must be");
 810   }
 811 
 812   if (committed_middle > 0) {
 813     assert(committed_low == lower, "Must be");
 814   }
 815   if (committed_middle < middle) {
 816     assert(committed_high == 0, "Must be");
 817   }
 818 
 819   if (committed_low < lower) {
 820     assert(committed_high == 0, "Must be");
 821     assert(committed_middle == 0, "Must be");
 822   }
 823 #endif
 824 
 825   return committed_low + committed_middle + committed_high;
 826 }
 827 
 828 
 829 bool VirtualSpace::contains(const void* p) const {
 830   return low() <= (const char*) p && (const char*) p < high();
 831 }
 832 
 833 static void pretouch_expanded_memory(void* start, void* end) {
 834   assert(is_aligned(start, os::vm_page_size()), "Unexpected alignment");
 835   assert(is_aligned(end,   os::vm_page_size()), "Unexpected alignment");
 836 
 837   os::pretouch_memory(start, end);
 838 }
 839 
 840 static bool commit_expanded(char* start, size_t size, size_t alignment, bool pre_touch, bool executable) {
 841   if (os::commit_memory(start, size, alignment, executable)) {
 842     if (pre_touch || AlwaysPreTouch) {
 843       pretouch_expanded_memory(start, start + size);
 844     }
 845     return true;
 846   }
 847 
 848   debug_only(warning(
 849       "INFO: os::commit_memory(" PTR_FORMAT ", " PTR_FORMAT
 850       " size=" SIZE_FORMAT ", executable=%d) failed",
 851       p2i(start), p2i(start + size), size, executable);)
 852 
 853   return false;
 854 }
 855 
 856 /*
 857    First we need to determine if a particular virtual space is using large
 858    pages.  This is done at the initialize function and only virtual spaces
 859    that are larger than LargePageSizeInBytes use large pages.  Once we
 860    have determined this, all expand_by and shrink_by calls must grow and
 861    shrink by large page size chunks.  If a particular request
 862    is within the current large page, the call to commit and uncommit memory
 863    can be ignored.  In the case that the low and high boundaries of this
 864    space is not large page aligned, the pages leading to the first large
 865    page address and the pages after the last large page address must be
 866    allocated with default pages.
 867 */
 868 bool VirtualSpace::expand_by(size_t bytes, bool pre_touch) {
 869   if (uncommitted_size() < bytes) {
 870     return false;
 871   }
 872 
 873   if (special()) {
 874     // don't commit memory if the entire space is pinned in memory
 875     _high += bytes;
 876     return true;
 877   }
 878 
 879   char* previous_high = high();
 880   char* unaligned_new_high = high() + bytes;
 881   assert(unaligned_new_high <= high_boundary(), "cannot expand by more than upper boundary");
 882 
 883   // Calculate where the new high for each of the regions should be.  If
 884   // the low_boundary() and high_boundary() are LargePageSizeInBytes aligned
 885   // then the unaligned lower and upper new highs would be the
 886   // lower_high() and upper_high() respectively.
 887   char* unaligned_lower_new_high =  MIN2(unaligned_new_high, lower_high_boundary());
 888   char* unaligned_middle_new_high = MIN2(unaligned_new_high, middle_high_boundary());
 889   char* unaligned_upper_new_high =  MIN2(unaligned_new_high, upper_high_boundary());
 890 
 891   // Align the new highs based on the regions alignment.  lower and upper
 892   // alignment will always be default page size.  middle alignment will be
 893   // LargePageSizeInBytes if the actual size of the virtual space is in
 894   // fact larger than LargePageSizeInBytes.
 895   char* aligned_lower_new_high =  align_up(unaligned_lower_new_high, lower_alignment());
 896   char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
 897   char* aligned_upper_new_high =  align_up(unaligned_upper_new_high, upper_alignment());
 898 
 899   // Determine which regions need to grow in this expand_by call.
 900   // If you are growing in the lower region, high() must be in that
 901   // region so calculate the size based on high().  For the middle and
 902   // upper regions, determine the starting point of growth based on the
 903   // location of high().  By getting the MAX of the region's low address
 904   // (or the previous region's high address) and high(), we can tell if it
 905   // is an intra or inter region growth.
 906   size_t lower_needs = 0;
 907   if (aligned_lower_new_high > lower_high()) {
 908     lower_needs = pointer_delta(aligned_lower_new_high, lower_high(), sizeof(char));
 909   }
 910   size_t middle_needs = 0;
 911   if (aligned_middle_new_high > middle_high()) {
 912     middle_needs = pointer_delta(aligned_middle_new_high, middle_high(), sizeof(char));
 913   }
 914   size_t upper_needs = 0;
 915   if (aligned_upper_new_high > upper_high()) {
 916     upper_needs = pointer_delta(aligned_upper_new_high, upper_high(), sizeof(char));
 917   }
 918 
 919   // Check contiguity.
 920   assert(low_boundary() <= lower_high() && lower_high() <= lower_high_boundary(),
 921          "high address must be contained within the region");
 922   assert(lower_high_boundary() <= middle_high() && middle_high() <= middle_high_boundary(),
 923          "high address must be contained within the region");
 924   assert(middle_high_boundary() <= upper_high() && upper_high() <= upper_high_boundary(),
 925          "high address must be contained within the region");
 926 
 927   // Commit regions
 928   if (lower_needs > 0) {
 929     assert(lower_high() + lower_needs <= lower_high_boundary(), "must not expand beyond region");
 930     if (!commit_expanded(lower_high(), lower_needs, _lower_alignment, pre_touch, _executable)) {
 931       return false;
 932     }
 933     _lower_high += lower_needs;
 934   }
 935 
 936   if (middle_needs > 0) {
 937     assert(middle_high() + middle_needs <= middle_high_boundary(), "must not expand beyond region");
 938     if (!commit_expanded(middle_high(), middle_needs, _middle_alignment, pre_touch, _executable)) {
 939       return false;
 940     }
 941     _middle_high += middle_needs;
 942   }
 943 
 944   if (upper_needs > 0) {
 945     assert(upper_high() + upper_needs <= upper_high_boundary(), "must not expand beyond region");
 946     if (!commit_expanded(upper_high(), upper_needs, _upper_alignment, pre_touch, _executable)) {
 947       return false;
 948     }
 949     _upper_high += upper_needs;
 950   }
 951 
 952   _high += bytes;
 953   return true;
 954 }
 955 
 956 // A page is uncommitted if the contents of the entire page is deemed unusable.
 957 // Continue to decrement the high() pointer until it reaches a page boundary
 958 // in which case that particular page can now be uncommitted.
 959 void VirtualSpace::shrink_by(size_t size) {
 960   if (committed_size() < size)
 961     fatal("Cannot shrink virtual space to negative size");
 962 
 963   if (special()) {
 964     // don't uncommit if the entire space is pinned in memory
 965     _high -= size;
 966     return;
 967   }
 968 
 969   char* unaligned_new_high = high() - size;
 970   assert(unaligned_new_high >= low_boundary(), "cannot shrink past lower boundary");
 971 
 972   // Calculate new unaligned address
 973   char* unaligned_upper_new_high =
 974     MAX2(unaligned_new_high, middle_high_boundary());
 975   char* unaligned_middle_new_high =
 976     MAX2(unaligned_new_high, lower_high_boundary());
 977   char* unaligned_lower_new_high =
 978     MAX2(unaligned_new_high, low_boundary());
 979 
 980   // Align address to region's alignment
 981   char* aligned_upper_new_high =  align_up(unaligned_upper_new_high, upper_alignment());
 982   char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
 983   char* aligned_lower_new_high =  align_up(unaligned_lower_new_high, lower_alignment());
 984 
 985   // Determine which regions need to shrink
 986   size_t upper_needs = 0;
 987   if (aligned_upper_new_high < upper_high()) {
 988     upper_needs =
 989       pointer_delta(upper_high(), aligned_upper_new_high, sizeof(char));
 990   }
 991   size_t middle_needs = 0;
 992   if (aligned_middle_new_high < middle_high()) {
 993     middle_needs =
 994       pointer_delta(middle_high(), aligned_middle_new_high, sizeof(char));
 995   }
 996   size_t lower_needs = 0;
 997   if (aligned_lower_new_high < lower_high()) {
 998     lower_needs =
 999       pointer_delta(lower_high(), aligned_lower_new_high, sizeof(char));
1000   }
1001 
1002   // Check contiguity.
1003   assert(middle_high_boundary() <= upper_high() &&
1004          upper_high() <= upper_high_boundary(),
1005          "high address must be contained within the region");
1006   assert(lower_high_boundary() <= middle_high() &&
1007          middle_high() <= middle_high_boundary(),
1008          "high address must be contained within the region");
1009   assert(low_boundary() <= lower_high() &&
1010          lower_high() <= lower_high_boundary(),
1011          "high address must be contained within the region");
1012 
1013   // Uncommit
1014   if (upper_needs > 0) {
1015     assert(middle_high_boundary() <= aligned_upper_new_high &&
1016            aligned_upper_new_high + upper_needs <= upper_high_boundary(),
1017            "must not shrink beyond region");
1018     if (!os::uncommit_memory(aligned_upper_new_high, upper_needs, _executable)) {
1019       debug_only(warning("os::uncommit_memory failed"));
1020       return;
1021     } else {
1022       _upper_high -= upper_needs;
1023     }
1024   }
1025   if (middle_needs > 0) {
1026     assert(lower_high_boundary() <= aligned_middle_new_high &&
1027            aligned_middle_new_high + middle_needs <= middle_high_boundary(),
1028            "must not shrink beyond region");
1029     if (!os::uncommit_memory(aligned_middle_new_high, middle_needs, _executable)) {
1030       debug_only(warning("os::uncommit_memory failed"));
1031       return;
1032     } else {
1033       _middle_high -= middle_needs;
1034     }
1035   }
1036   if (lower_needs > 0) {
1037     assert(low_boundary() <= aligned_lower_new_high &&
1038            aligned_lower_new_high + lower_needs <= lower_high_boundary(),
1039            "must not shrink beyond region");
1040     if (!os::uncommit_memory(aligned_lower_new_high, lower_needs, _executable)) {
1041       debug_only(warning("os::uncommit_memory failed"));
1042       return;
1043     } else {
1044       _lower_high -= lower_needs;
1045     }
1046   }
1047 
1048   _high -= size;
1049 }
1050 
1051 #ifndef PRODUCT
1052 void VirtualSpace::check_for_contiguity() {
1053   // Check contiguity.
1054   assert(low_boundary() <= lower_high() &&
1055          lower_high() <= lower_high_boundary(),
1056          "high address must be contained within the region");
1057   assert(lower_high_boundary() <= middle_high() &&
1058          middle_high() <= middle_high_boundary(),
1059          "high address must be contained within the region");
1060   assert(middle_high_boundary() <= upper_high() &&
1061          upper_high() <= upper_high_boundary(),
1062          "high address must be contained within the region");
1063   assert(low() >= low_boundary(), "low");
1064   assert(low_boundary() <= lower_high_boundary(), "lower high boundary");
1065   assert(upper_high_boundary() <= high_boundary(), "upper high boundary");
1066   assert(high() <= upper_high(), "upper high");
1067 }
1068 
1069 void VirtualSpace::print_on(outputStream* out) const {
1070   out->print   ("Virtual space:");
1071   if (special()) out->print(" (pinned in memory)");
1072   out->cr();
1073   out->print_cr(" - committed: " SIZE_FORMAT, committed_size());
1074   out->print_cr(" - reserved:  " SIZE_FORMAT, reserved_size());
1075   out->print_cr(" - [low, high]:     [" PTR_FORMAT ", " PTR_FORMAT "]",  p2i(low()), p2i(high()));
1076   out->print_cr(" - [low_b, high_b]: [" PTR_FORMAT ", " PTR_FORMAT "]",  p2i(low_boundary()), p2i(high_boundary()));
1077 }
1078 
1079 void VirtualSpace::print() const {
1080   print_on(tty);
1081 }
1082 
1083 #endif