1 /*
   2  * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "logging/log.hpp"
  27 #include "memory/resourceArea.hpp"
  28 #include "memory/virtualspace.hpp"
  29 #include "oops/compressedOops.hpp"
  30 #include "oops/markWord.hpp"
  31 #include "oops/oop.inline.hpp"
  32 #include "runtime/globals_extension.hpp"
  33 #include "runtime/java.hpp"
  34 #include "runtime/os.hpp"
  35 #include "services/memTracker.hpp"
  36 #include "utilities/align.hpp"
  37 #include "utilities/formatBuffer.hpp"
  38 #include "utilities/powerOfTwo.hpp"
  39 
  40 // ReservedSpace
  41 
  42 // Dummy constructor
  43 ReservedSpace::ReservedSpace() : _base(NULL), _size(0), _noaccess_prefix(0),
  44     _alignment(0), _special(false), _fd_for_heap(-1), _executable(false) {
  45 }
  46 
  47 ReservedSpace::ReservedSpace(size_t size) : _fd_for_heap(-1) {
  48   // Want to use large pages where possible. If the size is
  49   // not large page aligned the mapping will be a mix of
  50   // large and normal pages.
  51   size_t page_size = os::page_size_for_region_unaligned(size, 1);
  52   size_t alignment = os::vm_allocation_granularity();
  53   initialize(size, alignment, page_size, NULL, false);
  54 }
  55 
  56 ReservedSpace::ReservedSpace(size_t size, size_t preferred_page_size) : _fd_for_heap(-1) {
  57   // When a page size is given we don't want to mix large
  58   // and normal pages. If the size is not a multiple of the
  59   // page size it will be aligned up to achieve this.
  60   size_t alignment = os::vm_allocation_granularity();;
  61   if (preferred_page_size != (size_t)os::vm_page_size()) {
  62     alignment = MAX2(preferred_page_size, alignment);
  63     size = align_up(size, alignment);
  64   }
  65   initialize(size, alignment, preferred_page_size, NULL, false);
  66 }
  67 
  68 ReservedSpace::ReservedSpace(size_t size,
  69                              size_t alignment,
  70                              size_t page_size,
  71                              char* requested_address) : _fd_for_heap(-1) {
  72   initialize(size, alignment, page_size, requested_address, false);
  73 }
  74 
  75 ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment, size_t page_size,
  76                              bool special, bool executable) : _fd_for_heap(-1) {
  77   assert((size % os::vm_allocation_granularity()) == 0,
  78          "size not allocation aligned");
  79   initialize_members(base, size, alignment, page_size, special, executable);
  80 }
  81 
  82 // Helper method
  83 static char* attempt_map_or_reserve_memory_at(char* base, size_t size, int fd, bool executable) {
  84   if (fd != -1) {
  85     return os::attempt_map_memory_to_file_at(base, size, fd);
  86   }
  87   return os::attempt_reserve_memory_at(base, size, executable);
  88 }
  89 
  90 // Helper method
  91 static char* map_or_reserve_memory(size_t size, int fd, bool executable) {
  92   if (fd != -1) {
  93     return os::map_memory_to_file(size, fd);
  94   }
  95   return os::reserve_memory(size, executable);
  96 }
  97 
  98 // Helper method
  99 static char* map_or_reserve_memory_aligned(size_t size, size_t alignment, int fd, bool executable) {
 100   if (fd != -1) {
 101     return os::map_memory_to_file_aligned(size, alignment, fd);
 102   }
 103   return os::reserve_memory_aligned(size, alignment, executable);
 104 }
 105 
 106 // Helper method
 107 static void unmap_or_release_memory(char* base, size_t size, bool is_file_mapped) {
 108   if (is_file_mapped) {
 109     if (!os::unmap_memory(base, size)) {
 110       fatal("os::unmap_memory failed");
 111     }
 112   } else if (!os::release_memory(base, size)) {
 113     fatal("os::release_memory failed");
 114   }
 115 }
 116 
 117 // Helper method
 118 static bool failed_to_reserve_as_requested(char* base, char* requested_address) {
 119   if (base == requested_address || requested_address == NULL) {
 120     return false; // did not fail
 121   }
 122 
 123   if (base != NULL) {
 124     // Different reserve address may be acceptable in other cases
 125     // but for compressed oops heap should be at requested address.
 126     assert(UseCompressedOops, "currently requested address used only for compressed oops");
 127     log_debug(gc, heap, coops)("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, p2i(base), p2i(requested_address));
 128   }
 129   return true;
 130 }
 131 
 132 static bool use_explicit_large_pages(size_t page_size) {
 133   return !os::can_commit_large_page_memory() &&
 134          page_size != (size_t) os::vm_page_size();
 135 }
 136 
 137 static bool large_pages_requested() {
 138   return UseLargePages &&
 139          (!FLAG_IS_DEFAULT(UseLargePages) || !FLAG_IS_DEFAULT(LargePageSizeInBytes));
 140 }
 141 
 142 static void log_on_large_pages_failure(char* req_addr, size_t bytes) {
 143   if (large_pages_requested()) {
 144     // Compressed oops logging.
 145     log_debug(gc, heap, coops)("Reserve regular memory without large pages");
 146     // JVM style warning that we did not succeed in using large pages.
 147     char msg[128];
 148     jio_snprintf(msg, sizeof(msg), "Failed to reserve and commit memory using large pages. "
 149                                    "req_addr: " PTR_FORMAT " bytes: " SIZE_FORMAT,
 150                                    req_addr, bytes);
 151     warning("%s", msg);
 152   }
 153 }
 154 
 155 static char* reserve_memory(char* requested_address, const size_t size,
 156                             const size_t alignment, int fd, bool exec) {
 157   char* base;
 158   // If the memory was requested at a particular address, use
 159   // os::attempt_reserve_memory_at() to avoid mapping over something
 160   // important.  If the reservation fails, return NULL.
 161   if (requested_address != 0) {
 162     assert(is_aligned(requested_address, alignment),
 163            "Requested address " PTR_FORMAT " must be aligned to " SIZE_FORMAT,
 164            p2i(requested_address), alignment);
 165     base = attempt_map_or_reserve_memory_at(requested_address, size, fd, exec);
 166   } else {
 167     // Optimistically assume that the OS returns an aligned base pointer.
 168     // When reserving a large address range, most OSes seem to align to at
 169     // least 64K.
 170     base = map_or_reserve_memory(size, fd, exec);
 171     // Check alignment constraints. This is only needed when there is
 172     // no requested address.
 173     if (!is_aligned(base, alignment)) {
 174       // Base not aligned, retry.
 175       unmap_or_release_memory(base, size, fd != -1 /*is_file_mapped*/);
 176       // Map using the requested alignment.
 177       base = map_or_reserve_memory_aligned(size, alignment, fd, exec);
 178     }
 179   }
 180 
 181   return base;
 182 }
 183 
 184 static char* reserve_memory_special(char* requested_address, const size_t size,
 185                                     const size_t alignment, const size_t page_size, bool exec) {
 186 
 187   log_trace(pagesize)("Attempt special mapping: size: " SIZE_FORMAT "%s, "
 188                       "alignment: " SIZE_FORMAT "%s",
 189                       byte_size_in_exact_unit(size), exact_unit_for_byte_size(size),
 190                       byte_size_in_exact_unit(alignment), exact_unit_for_byte_size(alignment));
 191 
 192   char* base = os::reserve_memory_special(size, alignment, page_size, requested_address, exec);
 193   if (base != NULL) {
 194     // Check alignment constraints.
 195     assert(is_aligned(base, alignment),
 196            "reserve_memory_special() returned an unaligned address, base: " PTR_FORMAT
 197            " alignment: " SIZE_FORMAT_X,
 198            p2i(base), alignment);
 199   }
 200   return base;
 201 }
 202 
 203 void ReservedSpace::clear_members() {
 204   initialize_members(NULL, 0, 0, 0, false, false);
 205 }
 206 
 207 void ReservedSpace::initialize_members(char* base, size_t size, size_t alignment,
 208                                        size_t page_size, bool special, bool executable) {
 209   _base = base;
 210   _size = size;
 211   _alignment = alignment;
 212   _page_size = page_size;
 213   _special = special;
 214   _executable = executable;
 215   _noaccess_prefix = 0;
 216 }
 217 
 218 void ReservedSpace::reserve(size_t size,
 219                             size_t alignment,
 220                             size_t page_size,
 221                             char* requested_address,
 222                             bool executable) {
 223   assert(is_aligned(size, alignment), "Size must be aligned to the requested alignment");
 224 
 225   // There are basically three different cases that we need to handle below:
 226   // - Mapping backed by a file
 227   // - Mapping backed by explicit large pages
 228   // - Mapping backed by normal pages or transparent huge pages
 229   // The first two have restrictions that requires the whole mapping to be
 230   // committed up front. To record this the ReservedSpace is marked 'special'.
 231 
 232   if (_fd_for_heap != -1) {
 233     // When there is a backing file directory for this space then whether
 234     // large pages are allocated is up to the filesystem of the backing file.
 235     // So UseLargePages is not taken into account for this reservation.
 236     char* base = reserve_memory(requested_address, size, alignment, _fd_for_heap, executable);
 237     if (base != NULL) {
 238       initialize_members(base, size, alignment, os::vm_page_size(), true, executable);
 239     }
 240     // Always return, not possible to fall back to reservation not using a file.
 241     return;
 242   } else if (use_explicit_large_pages(page_size)) {
 243     // System can't commit large pages i.e. use transparent huge pages and
 244     // the caller requested large pages. To satisfy this request we use
 245     // explicit large pages and these have to be committed up front to ensure
 246     // no reservations are lost.
 247     size_t used_page_size = page_size;
 248     char* base = NULL;
 249 
 250     do {
 251       base = reserve_memory_special(requested_address, size, alignment, used_page_size, executable);
 252       if (base != NULL) {
 253         break;
 254       }
 255       used_page_size = os::page_sizes().next_smaller(used_page_size);
 256     } while (used_page_size > (size_t) os::vm_page_size());
 257 
 258     if (base != NULL) {
 259       // Successful reservation using large pages.
 260       initialize_members(base, size, alignment, used_page_size, true, executable);
 261       return;
 262     }
 263     // Failed to reserve explicit large pages, do proper logging.
 264     log_on_large_pages_failure(requested_address, size);
 265     // Now fall back to normal reservation.
 266     page_size = os::vm_page_size();
 267   }
 268 
 269   // Not a 'special' reservation.
 270   char* base = reserve_memory(requested_address, size, alignment, -1, executable);
 271   if (base != NULL) {
 272     // Successful mapping.
 273     initialize_members(base, size, alignment, page_size, false, executable);
 274   }
 275 }
 276 
 277 void ReservedSpace::initialize(size_t size,
 278                                size_t alignment,
 279                                size_t page_size,
 280                                char* requested_address,
 281                                bool executable) {
 282   const size_t granularity = os::vm_allocation_granularity();
 283   assert((size & (granularity - 1)) == 0,
 284          "size not aligned to os::vm_allocation_granularity()");
 285   assert((alignment & (granularity - 1)) == 0,
 286          "alignment not aligned to os::vm_allocation_granularity()");
 287   assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
 288          "not a power of 2");
 289   assert(page_size >= (size_t) os::vm_page_size(), "Invalid page size");
 290   assert(is_power_of_2(page_size), "Invalid page size");
 291 
 292   clear_members();
 293 
 294   if (size == 0) {
 295     return;
 296   }
 297 
 298   // Adjust alignment to not be 0.
 299   alignment = MAX2(alignment, (size_t)os::vm_page_size());
 300 
 301   // Reserve the memory.
 302   reserve(size, alignment, page_size, requested_address, executable);
 303 
 304   // Check that the requested address is used if given.
 305   if (failed_to_reserve_as_requested(_base, requested_address)) {
 306     // OS ignored the requested address, release the reservation.
 307     release();
 308     return;
 309   }
 310 }
 311 
 312 ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment) {
 313   assert(partition_size <= size(), "partition failed");
 314   ReservedSpace result(base(), partition_size, alignment, page_size(), special(), executable());
 315   return result;
 316 }
 317 
 318 
 319 ReservedSpace
 320 ReservedSpace::last_part(size_t partition_size, size_t alignment) {
 321   assert(partition_size <= size(), "partition failed");
 322   ReservedSpace result(base() + partition_size, size() - partition_size,
 323                        alignment, page_size(), special(), executable());
 324   return result;
 325 }
 326 
 327 
 328 size_t ReservedSpace::page_align_size_up(size_t size) {
 329   return align_up(size, os::vm_page_size());
 330 }
 331 
 332 
 333 size_t ReservedSpace::page_align_size_down(size_t size) {
 334   return align_down(size, os::vm_page_size());
 335 }
 336 
 337 
 338 size_t ReservedSpace::allocation_align_size_up(size_t size) {
 339   return align_up(size, os::vm_allocation_granularity());
 340 }
 341 
 342 void ReservedSpace::release() {
 343   if (is_reserved()) {
 344     char *real_base = _base - _noaccess_prefix;
 345     const size_t real_size = _size + _noaccess_prefix;
 346     if (special()) {
 347       if (_fd_for_heap != -1) {
 348         os::unmap_memory(real_base, real_size);
 349       } else {
 350         os::release_memory_special(real_base, real_size);
 351       }
 352     } else{
 353       os::release_memory(real_base, real_size);
 354     }
 355     clear_members();
 356   }
 357 }
 358 
 359 static size_t noaccess_prefix_size(size_t alignment) {
 360   return lcm(os::vm_page_size(), alignment);
 361 }
 362 
 363 void ReservedHeapSpace::establish_noaccess_prefix() {
 364   assert(_alignment >= (size_t)os::vm_page_size(), "must be at least page size big");
 365   _noaccess_prefix = noaccess_prefix_size(_alignment);
 366 
 367   if (base() && base() + _size > (char *)OopEncodingHeapMax) {
 368     if (true
 369         WIN64_ONLY(&& !UseLargePages)
 370         AIX_ONLY(&& os::vm_page_size() != 64*K)) {
 371       // Protect memory at the base of the allocated region.
 372       // If special, the page was committed (only matters on windows)
 373       if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, _special)) {
 374         fatal("cannot protect protection page");
 375       }
 376       log_debug(gc, heap, coops)("Protected page at the reserved heap base: "
 377                                  PTR_FORMAT " / " INTX_FORMAT " bytes",
 378                                  p2i(_base),
 379                                  _noaccess_prefix);
 380       assert(CompressedOops::use_implicit_null_checks() == true, "not initialized?");
 381     } else {
 382       CompressedOops::set_use_implicit_null_checks(false);
 383     }
 384   }
 385 
 386   _base += _noaccess_prefix;
 387   _size -= _noaccess_prefix;
 388   assert(((uintptr_t)_base % _alignment == 0), "must be exactly of required alignment");
 389 }
 390 
 391 // Tries to allocate memory of size 'size' at address requested_address with alignment 'alignment'.
 392 // Does not check whether the reserved memory actually is at requested_address, as the memory returned
 393 // might still fulfill the wishes of the caller.
 394 // Assures the memory is aligned to 'alignment'.
 395 // NOTE: If ReservedHeapSpace already points to some reserved memory this is freed, first.
 396 void ReservedHeapSpace::try_reserve_heap(size_t size,
 397                                          size_t alignment,
 398                                          size_t page_size,
 399                                          char* requested_address) {
 400   if (_base != NULL) {
 401     // We tried before, but we didn't like the address delivered.
 402     release();
 403   }
 404 
 405   // Try to reserve the memory for the heap.
 406   log_trace(gc, heap, coops)("Trying to allocate at address " PTR_FORMAT
 407                              " heap of size " SIZE_FORMAT_X,
 408                              p2i(requested_address),
 409                              size);
 410 
 411   reserve(size, alignment, page_size, requested_address, false);
 412 
 413   // Check alignment constraints.
 414   if (is_reserved() && !is_aligned(_base, _alignment)) {
 415     // Base not aligned, retry.
 416     release();
 417   }
 418 }
 419 
 420 void ReservedHeapSpace::try_reserve_range(char *highest_start,
 421                                           char *lowest_start,
 422                                           size_t attach_point_alignment,
 423                                           char *aligned_heap_base_min_address,
 424                                           char *upper_bound,
 425                                           size_t size,
 426                                           size_t alignment,
 427                                           size_t page_size) {
 428   const size_t attach_range = highest_start - lowest_start;
 429   // Cap num_attempts at possible number.
 430   // At least one is possible even for 0 sized attach range.
 431   const uint64_t num_attempts_possible = (attach_range / attach_point_alignment) + 1;
 432   const uint64_t num_attempts_to_try   = MIN2((uint64_t)HeapSearchSteps, num_attempts_possible);
 433 
 434   const size_t stepsize = (attach_range == 0) ? // Only one try.
 435     (size_t) highest_start : align_up(attach_range / num_attempts_to_try, attach_point_alignment);
 436 
 437   // Try attach points from top to bottom.
 438   char* attach_point = highest_start;
 439   while (attach_point >= lowest_start  &&
 440          attach_point <= highest_start &&  // Avoid wrap around.
 441          ((_base == NULL) ||
 442           (_base < aligned_heap_base_min_address || _base + size > upper_bound))) {
 443     try_reserve_heap(size, alignment, page_size, attach_point);
 444     attach_point -= stepsize;
 445   }
 446 }
 447 
 448 #define SIZE_64K  ((uint64_t) UCONST64(      0x10000))
 449 #define SIZE_256M ((uint64_t) UCONST64(   0x10000000))
 450 #define SIZE_32G  ((uint64_t) UCONST64(  0x800000000))
 451 
 452 // Helper for heap allocation. Returns an array with addresses
 453 // (OS-specific) which are suited for disjoint base mode. Array is
 454 // NULL terminated.
 455 static char** get_attach_addresses_for_disjoint_mode() {
 456   static uint64_t addresses[] = {
 457      2 * SIZE_32G,
 458      3 * SIZE_32G,
 459      4 * SIZE_32G,
 460      8 * SIZE_32G,
 461     10 * SIZE_32G,
 462      1 * SIZE_64K * SIZE_32G,
 463      2 * SIZE_64K * SIZE_32G,
 464      3 * SIZE_64K * SIZE_32G,
 465      4 * SIZE_64K * SIZE_32G,
 466     16 * SIZE_64K * SIZE_32G,
 467     32 * SIZE_64K * SIZE_32G,
 468     34 * SIZE_64K * SIZE_32G,
 469     0
 470   };
 471 
 472   // Sort out addresses smaller than HeapBaseMinAddress. This assumes
 473   // the array is sorted.
 474   uint i = 0;
 475   while (addresses[i] != 0 &&
 476          (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) {
 477     i++;
 478   }
 479   uint start = i;
 480 
 481   // Avoid more steps than requested.
 482   i = 0;
 483   while (addresses[start+i] != 0) {
 484     if (i == HeapSearchSteps) {
 485       addresses[start+i] = 0;
 486       break;
 487     }
 488     i++;
 489   }
 490 
 491   return (char**) &addresses[start];
 492 }
 493 
 494 void ReservedHeapSpace::initialize_compressed_heap(const size_t size, size_t alignment, size_t page_size) {
 495   guarantee(size + noaccess_prefix_size(alignment) <= OopEncodingHeapMax,
 496             "can not allocate compressed oop heap for this size");
 497   guarantee(alignment == MAX2(alignment, (size_t)os::vm_page_size()), "alignment too small");
 498 
 499   const size_t granularity = os::vm_allocation_granularity();
 500   assert((size & (granularity - 1)) == 0,
 501          "size not aligned to os::vm_allocation_granularity()");
 502   assert((alignment & (granularity - 1)) == 0,
 503          "alignment not aligned to os::vm_allocation_granularity()");
 504   assert(alignment == 0 || is_power_of_2((intptr_t)alignment),
 505          "not a power of 2");
 506 
 507   // The necessary attach point alignment for generated wish addresses.
 508   // This is needed to increase the chance of attaching for mmap and shmat.
 509   const size_t os_attach_point_alignment =
 510     AIX_ONLY(SIZE_256M)  // Known shm boundary alignment.
 511     NOT_AIX(os::vm_allocation_granularity());
 512   const size_t attach_point_alignment = lcm(alignment, os_attach_point_alignment);
 513 
 514   char *aligned_heap_base_min_address = (char *)align_up((void *)HeapBaseMinAddress, alignment);
 515   size_t noaccess_prefix = ((aligned_heap_base_min_address + size) > (char*)OopEncodingHeapMax) ?
 516     noaccess_prefix_size(alignment) : 0;
 517 
 518   // Attempt to alloc at user-given address.
 519   if (!FLAG_IS_DEFAULT(HeapBaseMinAddress)) {
 520     try_reserve_heap(size + noaccess_prefix, alignment, page_size, aligned_heap_base_min_address);
 521     if (_base != aligned_heap_base_min_address) { // Enforce this exact address.
 522       release();
 523     }
 524   }
 525 
 526   // Keep heap at HeapBaseMinAddress.
 527   if (_base == NULL) {
 528 
 529     // Try to allocate the heap at addresses that allow efficient oop compression.
 530     // Different schemes are tried, in order of decreasing optimization potential.
 531     //
 532     // For this, try_reserve_heap() is called with the desired heap base addresses.
 533     // A call into the os layer to allocate at a given address can return memory
 534     // at a different address than requested.  Still, this might be memory at a useful
 535     // address. try_reserve_heap() always returns this allocated memory, as only here
 536     // the criteria for a good heap are checked.
 537 
 538     // Attempt to allocate so that we can run without base and scale (32-Bit unscaled compressed oops).
 539     // Give it several tries from top of range to bottom.
 540     if (aligned_heap_base_min_address + size <= (char *)UnscaledOopHeapMax) {
 541 
 542       // Calc address range within we try to attach (range of possible start addresses).
 543       char* const highest_start = align_down((char *)UnscaledOopHeapMax - size, attach_point_alignment);
 544       char* const lowest_start  = align_up(aligned_heap_base_min_address, attach_point_alignment);
 545       try_reserve_range(highest_start, lowest_start, attach_point_alignment,
 546                         aligned_heap_base_min_address, (char *)UnscaledOopHeapMax, size, alignment, page_size);
 547     }
 548 
 549     // zerobased: Attempt to allocate in the lower 32G.
 550     // But leave room for the compressed class pointers, which is allocated above
 551     // the heap.
 552     // Note Lilliput: the advantages of this strategy were questionable before
 553     //  (since CDS=off + Compressed oops + heap large enough to suffocate us out of lower 32g
 554     //  is rare) and with Lilliput the encoding range drastically shrank. We may just do away
 555     //  with this altogether.
 556     char *zerobased_max = (char *)OopEncodingHeapMax;
 557     const size_t class_space = align_up(CompressedClassSpaceSize, alignment);
 558     // For small heaps, save some space for compressed class pointer
 559     // space so it can be decoded with no base.
 560     if (UseCompressedClassPointers && !UseSharedSpaces &&
 561         OopEncodingHeapMax <= KlassEncodingMetaspaceMax &&
 562         (uint64_t)(aligned_heap_base_min_address + size + class_space) <= KlassEncodingMetaspaceMax) {
 563       zerobased_max = (char *)OopEncodingHeapMax - class_space;
 564     }
 565 
 566     // Give it several tries from top of range to bottom.
 567     if (aligned_heap_base_min_address + size <= zerobased_max &&    // Zerobased theoretical possible.
 568         ((_base == NULL) ||                        // No previous try succeeded.
 569          (_base + size > zerobased_max))) {        // Unscaled delivered an arbitrary address.
 570 
 571       // Calc address range within we try to attach (range of possible start addresses).
 572       char *const highest_start = align_down(zerobased_max - size, attach_point_alignment);
 573       // Need to be careful about size being guaranteed to be less
 574       // than UnscaledOopHeapMax due to type constraints.
 575       char *lowest_start = aligned_heap_base_min_address;
 576       uint64_t unscaled_end = UnscaledOopHeapMax - size;
 577       if (unscaled_end < UnscaledOopHeapMax) { // unscaled_end wrapped if size is large
 578         lowest_start = MAX2(lowest_start, (char*)unscaled_end);
 579       }
 580       lowest_start = align_up(lowest_start, attach_point_alignment);
 581       try_reserve_range(highest_start, lowest_start, attach_point_alignment,
 582                         aligned_heap_base_min_address, zerobased_max, size, alignment, page_size);
 583     }
 584 
 585     // Now we go for heaps with base != 0.  We need a noaccess prefix to efficiently
 586     // implement null checks.
 587     noaccess_prefix = noaccess_prefix_size(alignment);
 588 
 589     // Try to attach at addresses that are aligned to OopEncodingHeapMax. Disjointbase mode.
 590     char** addresses = get_attach_addresses_for_disjoint_mode();
 591     int i = 0;
 592     while (addresses[i] &&                                 // End of array not yet reached.
 593            ((_base == NULL) ||                             // No previous try succeeded.
 594             (_base + size >  (char *)OopEncodingHeapMax && // Not zerobased or unscaled address.
 595              !CompressedOops::is_disjoint_heap_base_address((address)_base)))) {  // Not disjoint address.
 596       char* const attach_point = addresses[i];
 597       assert(attach_point >= aligned_heap_base_min_address, "Flag support broken");
 598       try_reserve_heap(size + noaccess_prefix, alignment, page_size, attach_point);
 599       i++;
 600     }
 601 
 602     // Last, desperate try without any placement.
 603     if (_base == NULL) {
 604       log_trace(gc, heap, coops)("Trying to allocate at address NULL heap of size " SIZE_FORMAT_X, size + noaccess_prefix);
 605       initialize(size + noaccess_prefix, alignment, page_size, NULL, false);
 606     }
 607   }
 608 }
 609 
 610 ReservedHeapSpace::ReservedHeapSpace(size_t size, size_t alignment, size_t page_size, const char* heap_allocation_directory) : ReservedSpace() {
 611 
 612   if (size == 0) {
 613     return;
 614   }
 615 
 616   if (heap_allocation_directory != NULL) {
 617     _fd_for_heap = os::create_file_for_heap(heap_allocation_directory);
 618     if (_fd_for_heap == -1) {
 619       vm_exit_during_initialization(
 620         err_msg("Could not create file for Heap at location %s", heap_allocation_directory));
 621     }
 622     // When there is a backing file directory for this space then whether
 623     // large pages are allocated is up to the filesystem of the backing file.
 624     // If requested, let the user know that explicit large pages can't be used.
 625     if (use_explicit_large_pages(page_size) && large_pages_requested()) {
 626       log_debug(gc, heap)("Cannot allocate explicit large pages for Java Heap when AllocateHeapAt option is set.");
 627     }
 628   }
 629 
 630   // Heap size should be aligned to alignment, too.
 631   guarantee(is_aligned(size, alignment), "set by caller");
 632 
 633   if (UseCompressedOops) {
 634     initialize_compressed_heap(size, alignment, page_size);
 635     if (_size > size) {
 636       // We allocated heap with noaccess prefix.
 637       // It can happen we get a zerobased/unscaled heap with noaccess prefix,
 638       // if we had to try at arbitrary address.
 639       establish_noaccess_prefix();
 640     }
 641   } else {
 642     initialize(size, alignment, page_size, NULL, false);
 643   }
 644 
 645   assert(markWord::encode_pointer_as_mark(_base).decode_pointer() == _base,
 646          "area must be distinguishable from marks for mark-sweep");
 647   assert(markWord::encode_pointer_as_mark(&_base[size]).decode_pointer() == &_base[size],
 648          "area must be distinguishable from marks for mark-sweep");
 649 
 650   if (base() != NULL) {
 651     MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap);
 652   }
 653 
 654   if (_fd_for_heap != -1) {
 655     ::close(_fd_for_heap);
 656   }
 657 }
 658 
 659 MemRegion ReservedHeapSpace::region() const {
 660   return MemRegion((HeapWord*)base(), (HeapWord*)end());
 661 }
 662 
 663 // Reserve space for code segment.  Same as Java heap only we mark this as
 664 // executable.
 665 ReservedCodeSpace::ReservedCodeSpace(size_t r_size,
 666                                      size_t rs_align,
 667                                      size_t rs_page_size) : ReservedSpace() {
 668   initialize(r_size, rs_align, rs_page_size, /*requested address*/ NULL, /*executable*/ true);
 669   MemTracker::record_virtual_memory_type((address)base(), mtCode);
 670 }
 671 
 672 // VirtualSpace
 673 
 674 VirtualSpace::VirtualSpace() {
 675   _low_boundary           = NULL;
 676   _high_boundary          = NULL;
 677   _low                    = NULL;
 678   _high                   = NULL;
 679   _lower_high             = NULL;
 680   _middle_high            = NULL;
 681   _upper_high             = NULL;
 682   _lower_high_boundary    = NULL;
 683   _middle_high_boundary   = NULL;
 684   _upper_high_boundary    = NULL;
 685   _lower_alignment        = 0;
 686   _middle_alignment       = 0;
 687   _upper_alignment        = 0;
 688   _special                = false;
 689   _executable             = false;
 690 }
 691 
 692 
 693 bool VirtualSpace::initialize(ReservedSpace rs, size_t committed_size) {
 694   const size_t max_commit_granularity = os::page_size_for_region_unaligned(rs.size(), 1);
 695   return initialize_with_granularity(rs, committed_size, max_commit_granularity);
 696 }
 697 
 698 bool VirtualSpace::initialize_with_granularity(ReservedSpace rs, size_t committed_size, size_t max_commit_granularity) {
 699   if(!rs.is_reserved()) return false;  // allocation failed.
 700   assert(_low_boundary == NULL, "VirtualSpace already initialized");
 701   assert(max_commit_granularity > 0, "Granularity must be non-zero.");
 702 
 703   _low_boundary  = rs.base();
 704   _high_boundary = low_boundary() + rs.size();
 705 
 706   _low = low_boundary();
 707   _high = low();
 708 
 709   _special = rs.special();
 710   _executable = rs.executable();
 711 
 712   // When a VirtualSpace begins life at a large size, make all future expansion
 713   // and shrinking occur aligned to a granularity of large pages.  This avoids
 714   // fragmentation of physical addresses that inhibits the use of large pages
 715   // by the OS virtual memory system.  Empirically,  we see that with a 4MB
 716   // page size, the only spaces that get handled this way are codecache and
 717   // the heap itself, both of which provide a substantial performance
 718   // boost in many benchmarks when covered by large pages.
 719   //
 720   // No attempt is made to force large page alignment at the very top and
 721   // bottom of the space if they are not aligned so already.
 722   _lower_alignment  = os::vm_page_size();
 723   _middle_alignment = max_commit_granularity;
 724   _upper_alignment  = os::vm_page_size();
 725 
 726   // End of each region
 727   _lower_high_boundary = align_up(low_boundary(), middle_alignment());
 728   _middle_high_boundary = align_down(high_boundary(), middle_alignment());
 729   _upper_high_boundary = high_boundary();
 730 
 731   // High address of each region
 732   _lower_high = low_boundary();
 733   _middle_high = lower_high_boundary();
 734   _upper_high = middle_high_boundary();
 735 
 736   // commit to initial size
 737   if (committed_size > 0) {
 738     if (!expand_by(committed_size)) {
 739       return false;
 740     }
 741   }
 742   return true;
 743 }
 744 
 745 
 746 VirtualSpace::~VirtualSpace() {
 747   release();
 748 }
 749 
 750 
 751 void VirtualSpace::release() {
 752   // This does not release memory it reserved.
 753   // Caller must release via rs.release();
 754   _low_boundary           = NULL;
 755   _high_boundary          = NULL;
 756   _low                    = NULL;
 757   _high                   = NULL;
 758   _lower_high             = NULL;
 759   _middle_high            = NULL;
 760   _upper_high             = NULL;
 761   _lower_high_boundary    = NULL;
 762   _middle_high_boundary   = NULL;
 763   _upper_high_boundary    = NULL;
 764   _lower_alignment        = 0;
 765   _middle_alignment       = 0;
 766   _upper_alignment        = 0;
 767   _special                = false;
 768   _executable             = false;
 769 }
 770 
 771 
 772 size_t VirtualSpace::committed_size() const {
 773   return pointer_delta(high(), low(), sizeof(char));
 774 }
 775 
 776 
 777 size_t VirtualSpace::reserved_size() const {
 778   return pointer_delta(high_boundary(), low_boundary(), sizeof(char));
 779 }
 780 
 781 
 782 size_t VirtualSpace::uncommitted_size()  const {
 783   return reserved_size() - committed_size();
 784 }
 785 
 786 size_t VirtualSpace::actual_committed_size() const {
 787   // Special VirtualSpaces commit all reserved space up front.
 788   if (special()) {
 789     return reserved_size();
 790   }
 791 
 792   size_t committed_low    = pointer_delta(_lower_high,  _low_boundary,         sizeof(char));
 793   size_t committed_middle = pointer_delta(_middle_high, _lower_high_boundary,  sizeof(char));
 794   size_t committed_high   = pointer_delta(_upper_high,  _middle_high_boundary, sizeof(char));
 795 
 796 #ifdef ASSERT
 797   size_t lower  = pointer_delta(_lower_high_boundary,  _low_boundary,         sizeof(char));
 798   size_t middle = pointer_delta(_middle_high_boundary, _lower_high_boundary,  sizeof(char));
 799   size_t upper  = pointer_delta(_upper_high_boundary,  _middle_high_boundary, sizeof(char));
 800 
 801   if (committed_high > 0) {
 802     assert(committed_low == lower, "Must be");
 803     assert(committed_middle == middle, "Must be");
 804   }
 805 
 806   if (committed_middle > 0) {
 807     assert(committed_low == lower, "Must be");
 808   }
 809   if (committed_middle < middle) {
 810     assert(committed_high == 0, "Must be");
 811   }
 812 
 813   if (committed_low < lower) {
 814     assert(committed_high == 0, "Must be");
 815     assert(committed_middle == 0, "Must be");
 816   }
 817 #endif
 818 
 819   return committed_low + committed_middle + committed_high;
 820 }
 821 
 822 
 823 bool VirtualSpace::contains(const void* p) const {
 824   return low() <= (const char*) p && (const char*) p < high();
 825 }
 826 
 827 static void pretouch_expanded_memory(void* start, void* end) {
 828   assert(is_aligned(start, os::vm_page_size()), "Unexpected alignment");
 829   assert(is_aligned(end,   os::vm_page_size()), "Unexpected alignment");
 830 
 831   os::pretouch_memory(start, end);
 832 }
 833 
 834 static bool commit_expanded(char* start, size_t size, size_t alignment, bool pre_touch, bool executable) {
 835   if (os::commit_memory(start, size, alignment, executable)) {
 836     if (pre_touch || AlwaysPreTouch) {
 837       pretouch_expanded_memory(start, start + size);
 838     }
 839     return true;
 840   }
 841 
 842   debug_only(warning(
 843       "INFO: os::commit_memory(" PTR_FORMAT ", " PTR_FORMAT
 844       " size=" SIZE_FORMAT ", executable=%d) failed",
 845       p2i(start), p2i(start + size), size, executable);)
 846 
 847   return false;
 848 }
 849 
 850 /*
 851    First we need to determine if a particular virtual space is using large
 852    pages.  This is done at the initialize function and only virtual spaces
 853    that are larger than LargePageSizeInBytes use large pages.  Once we
 854    have determined this, all expand_by and shrink_by calls must grow and
 855    shrink by large page size chunks.  If a particular request
 856    is within the current large page, the call to commit and uncommit memory
 857    can be ignored.  In the case that the low and high boundaries of this
 858    space is not large page aligned, the pages leading to the first large
 859    page address and the pages after the last large page address must be
 860    allocated with default pages.
 861 */
 862 bool VirtualSpace::expand_by(size_t bytes, bool pre_touch) {
 863   if (uncommitted_size() < bytes) {
 864     return false;
 865   }
 866 
 867   if (special()) {
 868     // don't commit memory if the entire space is pinned in memory
 869     _high += bytes;
 870     return true;
 871   }
 872 
 873   char* previous_high = high();
 874   char* unaligned_new_high = high() + bytes;
 875   assert(unaligned_new_high <= high_boundary(), "cannot expand by more than upper boundary");
 876 
 877   // Calculate where the new high for each of the regions should be.  If
 878   // the low_boundary() and high_boundary() are LargePageSizeInBytes aligned
 879   // then the unaligned lower and upper new highs would be the
 880   // lower_high() and upper_high() respectively.
 881   char* unaligned_lower_new_high =  MIN2(unaligned_new_high, lower_high_boundary());
 882   char* unaligned_middle_new_high = MIN2(unaligned_new_high, middle_high_boundary());
 883   char* unaligned_upper_new_high =  MIN2(unaligned_new_high, upper_high_boundary());
 884 
 885   // Align the new highs based on the regions alignment.  lower and upper
 886   // alignment will always be default page size.  middle alignment will be
 887   // LargePageSizeInBytes if the actual size of the virtual space is in
 888   // fact larger than LargePageSizeInBytes.
 889   char* aligned_lower_new_high =  align_up(unaligned_lower_new_high, lower_alignment());
 890   char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
 891   char* aligned_upper_new_high =  align_up(unaligned_upper_new_high, upper_alignment());
 892 
 893   // Determine which regions need to grow in this expand_by call.
 894   // If you are growing in the lower region, high() must be in that
 895   // region so calculate the size based on high().  For the middle and
 896   // upper regions, determine the starting point of growth based on the
 897   // location of high().  By getting the MAX of the region's low address
 898   // (or the previous region's high address) and high(), we can tell if it
 899   // is an intra or inter region growth.
 900   size_t lower_needs = 0;
 901   if (aligned_lower_new_high > lower_high()) {
 902     lower_needs = pointer_delta(aligned_lower_new_high, lower_high(), sizeof(char));
 903   }
 904   size_t middle_needs = 0;
 905   if (aligned_middle_new_high > middle_high()) {
 906     middle_needs = pointer_delta(aligned_middle_new_high, middle_high(), sizeof(char));
 907   }
 908   size_t upper_needs = 0;
 909   if (aligned_upper_new_high > upper_high()) {
 910     upper_needs = pointer_delta(aligned_upper_new_high, upper_high(), sizeof(char));
 911   }
 912 
 913   // Check contiguity.
 914   assert(low_boundary() <= lower_high() && lower_high() <= lower_high_boundary(),
 915          "high address must be contained within the region");
 916   assert(lower_high_boundary() <= middle_high() && middle_high() <= middle_high_boundary(),
 917          "high address must be contained within the region");
 918   assert(middle_high_boundary() <= upper_high() && upper_high() <= upper_high_boundary(),
 919          "high address must be contained within the region");
 920 
 921   // Commit regions
 922   if (lower_needs > 0) {
 923     assert(lower_high() + lower_needs <= lower_high_boundary(), "must not expand beyond region");
 924     if (!commit_expanded(lower_high(), lower_needs, _lower_alignment, pre_touch, _executable)) {
 925       return false;
 926     }
 927     _lower_high += lower_needs;
 928   }
 929 
 930   if (middle_needs > 0) {
 931     assert(middle_high() + middle_needs <= middle_high_boundary(), "must not expand beyond region");
 932     if (!commit_expanded(middle_high(), middle_needs, _middle_alignment, pre_touch, _executable)) {
 933       return false;
 934     }
 935     _middle_high += middle_needs;
 936   }
 937 
 938   if (upper_needs > 0) {
 939     assert(upper_high() + upper_needs <= upper_high_boundary(), "must not expand beyond region");
 940     if (!commit_expanded(upper_high(), upper_needs, _upper_alignment, pre_touch, _executable)) {
 941       return false;
 942     }
 943     _upper_high += upper_needs;
 944   }
 945 
 946   _high += bytes;
 947   return true;
 948 }
 949 
 950 // A page is uncommitted if the contents of the entire page is deemed unusable.
 951 // Continue to decrement the high() pointer until it reaches a page boundary
 952 // in which case that particular page can now be uncommitted.
 953 void VirtualSpace::shrink_by(size_t size) {
 954   if (committed_size() < size)
 955     fatal("Cannot shrink virtual space to negative size");
 956 
 957   if (special()) {
 958     // don't uncommit if the entire space is pinned in memory
 959     _high -= size;
 960     return;
 961   }
 962 
 963   char* unaligned_new_high = high() - size;
 964   assert(unaligned_new_high >= low_boundary(), "cannot shrink past lower boundary");
 965 
 966   // Calculate new unaligned address
 967   char* unaligned_upper_new_high =
 968     MAX2(unaligned_new_high, middle_high_boundary());
 969   char* unaligned_middle_new_high =
 970     MAX2(unaligned_new_high, lower_high_boundary());
 971   char* unaligned_lower_new_high =
 972     MAX2(unaligned_new_high, low_boundary());
 973 
 974   // Align address to region's alignment
 975   char* aligned_upper_new_high =  align_up(unaligned_upper_new_high, upper_alignment());
 976   char* aligned_middle_new_high = align_up(unaligned_middle_new_high, middle_alignment());
 977   char* aligned_lower_new_high =  align_up(unaligned_lower_new_high, lower_alignment());
 978 
 979   // Determine which regions need to shrink
 980   size_t upper_needs = 0;
 981   if (aligned_upper_new_high < upper_high()) {
 982     upper_needs =
 983       pointer_delta(upper_high(), aligned_upper_new_high, sizeof(char));
 984   }
 985   size_t middle_needs = 0;
 986   if (aligned_middle_new_high < middle_high()) {
 987     middle_needs =
 988       pointer_delta(middle_high(), aligned_middle_new_high, sizeof(char));
 989   }
 990   size_t lower_needs = 0;
 991   if (aligned_lower_new_high < lower_high()) {
 992     lower_needs =
 993       pointer_delta(lower_high(), aligned_lower_new_high, sizeof(char));
 994   }
 995 
 996   // Check contiguity.
 997   assert(middle_high_boundary() <= upper_high() &&
 998          upper_high() <= upper_high_boundary(),
 999          "high address must be contained within the region");
1000   assert(lower_high_boundary() <= middle_high() &&
1001          middle_high() <= middle_high_boundary(),
1002          "high address must be contained within the region");
1003   assert(low_boundary() <= lower_high() &&
1004          lower_high() <= lower_high_boundary(),
1005          "high address must be contained within the region");
1006 
1007   // Uncommit
1008   if (upper_needs > 0) {
1009     assert(middle_high_boundary() <= aligned_upper_new_high &&
1010            aligned_upper_new_high + upper_needs <= upper_high_boundary(),
1011            "must not shrink beyond region");
1012     if (!os::uncommit_memory(aligned_upper_new_high, upper_needs, _executable)) {
1013       debug_only(warning("os::uncommit_memory failed"));
1014       return;
1015     } else {
1016       _upper_high -= upper_needs;
1017     }
1018   }
1019   if (middle_needs > 0) {
1020     assert(lower_high_boundary() <= aligned_middle_new_high &&
1021            aligned_middle_new_high + middle_needs <= middle_high_boundary(),
1022            "must not shrink beyond region");
1023     if (!os::uncommit_memory(aligned_middle_new_high, middle_needs, _executable)) {
1024       debug_only(warning("os::uncommit_memory failed"));
1025       return;
1026     } else {
1027       _middle_high -= middle_needs;
1028     }
1029   }
1030   if (lower_needs > 0) {
1031     assert(low_boundary() <= aligned_lower_new_high &&
1032            aligned_lower_new_high + lower_needs <= lower_high_boundary(),
1033            "must not shrink beyond region");
1034     if (!os::uncommit_memory(aligned_lower_new_high, lower_needs, _executable)) {
1035       debug_only(warning("os::uncommit_memory failed"));
1036       return;
1037     } else {
1038       _lower_high -= lower_needs;
1039     }
1040   }
1041 
1042   _high -= size;
1043 }
1044 
1045 #ifndef PRODUCT
1046 void VirtualSpace::check_for_contiguity() {
1047   // Check contiguity.
1048   assert(low_boundary() <= lower_high() &&
1049          lower_high() <= lower_high_boundary(),
1050          "high address must be contained within the region");
1051   assert(lower_high_boundary() <= middle_high() &&
1052          middle_high() <= middle_high_boundary(),
1053          "high address must be contained within the region");
1054   assert(middle_high_boundary() <= upper_high() &&
1055          upper_high() <= upper_high_boundary(),
1056          "high address must be contained within the region");
1057   assert(low() >= low_boundary(), "low");
1058   assert(low_boundary() <= lower_high_boundary(), "lower high boundary");
1059   assert(upper_high_boundary() <= high_boundary(), "upper high boundary");
1060   assert(high() <= upper_high(), "upper high");
1061 }
1062 
1063 void VirtualSpace::print_on(outputStream* out) const {
1064   out->print   ("Virtual space:");
1065   if (special()) out->print(" (pinned in memory)");
1066   out->cr();
1067   out->print_cr(" - committed: " SIZE_FORMAT, committed_size());
1068   out->print_cr(" - reserved:  " SIZE_FORMAT, reserved_size());
1069   out->print_cr(" - [low, high]:     [" PTR_FORMAT ", " PTR_FORMAT "]",  p2i(low()), p2i(high()));
1070   out->print_cr(" - [low_b, high_b]: [" PTR_FORMAT ", " PTR_FORMAT "]",  p2i(low_boundary()), p2i(high_boundary()));
1071 }
1072 
1073 void VirtualSpace::print() const {
1074   print_on(tty);
1075 }
1076 
1077 #endif