1 /*
   2  * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "gc/shared/gcArguments.hpp"
  26 #include "gc/shared/oopStorage.hpp"
  27 #include "gc/z/zAddress.hpp"
  28 #include "gc/z/zGlobals.hpp"
  29 #include "gc/z/zHeap.inline.hpp"
  30 #include "gc/z/zHeapIterator.hpp"
  31 #include "gc/z/zList.inline.hpp"
  32 #include "gc/z/zLock.inline.hpp"
  33 #include "gc/z/zMark.inline.hpp"
  34 #include "gc/z/zOopClosures.inline.hpp"
  35 #include "gc/z/zPage.inline.hpp"
  36 #include "gc/z/zPageTable.inline.hpp"
  37 #include "gc/z/zRelocationSet.inline.hpp"
  38 #include "gc/z/zResurrection.hpp"
  39 #include "gc/z/zRootsIterator.hpp"
  40 #include "gc/z/zStat.hpp"
  41 #include "gc/z/zTask.hpp"
  42 #include "gc/z/zThread.hpp"
  43 #include "gc/z/zTracer.inline.hpp"
  44 #include "gc/z/zVirtualMemory.inline.hpp"
  45 #include "gc/z/zWorkers.inline.hpp"
  46 #include "logging/log.hpp"
  47 #include "memory/resourceArea.hpp"
  48 #include "oops/oop.inline.hpp"
  49 #include "runtime/arguments.hpp"
  50 #include "runtime/safepoint.hpp"
  51 #include "runtime/thread.hpp"
  52 #include "utilities/align.hpp"
  53 #include "utilities/debug.hpp"
  54 
  55 static const ZStatSampler ZSamplerHeapUsedBeforeMark("Memory", "Heap Used Before Mark", ZStatUnitBytes);
  56 static const ZStatSampler ZSamplerHeapUsedAfterMark("Memory", "Heap Used After Mark", ZStatUnitBytes);
  57 static const ZStatSampler ZSamplerHeapUsedBeforeRelocation("Memory", "Heap Used Before Relocation", ZStatUnitBytes);
  58 static const ZStatSampler ZSamplerHeapUsedAfterRelocation("Memory", "Heap Used After Relocation", ZStatUnitBytes);
  59 static const ZStatCounter ZCounterUndoPageAllocation("Memory", "Undo Page Allocation", ZStatUnitOpsPerSecond);
  60 static const ZStatCounter ZCounterOutOfMemory("Memory", "Out Of Memory", ZStatUnitOpsPerSecond);
  61 
  62 ZHeap* ZHeap::_heap = NULL;
  63 
  64 ZHeap::ZHeap() :
  65     _workers(),
  66     _object_allocator(_workers.nworkers()),
  67     _page_allocator(heap_min_size(), heap_initial_size(), heap_max_size(), heap_max_reserve_size()),
  68     _page_table(),
  69     _forwarding_table(),
  70     _mark(&_workers, &_page_table),
  71     _reference_processor(&_workers),
  72     _weak_roots_processor(&_workers),
  73     _relocate(&_workers),
  74     _relocation_set(),
  75     _unload(&_workers),
  76     _serviceability(heap_min_size(), heap_max_size()) {
  77   // Install global heap instance
  78   assert(_heap == NULL, "Already initialized");
  79   _heap = this;
  80 
  81   // Update statistics
  82   ZStatHeap::set_at_initialize(heap_min_size(), heap_max_size(), heap_max_reserve_size());
  83 }
  84 
  85 size_t ZHeap::heap_min_size() const {
  86   return MinHeapSize;
  87 }
  88 
  89 size_t ZHeap::heap_initial_size() const {
  90   return InitialHeapSize;
  91 }
  92 
  93 size_t ZHeap::heap_max_size() const {
  94   return MaxHeapSize;
  95 }
  96 
  97 size_t ZHeap::heap_max_reserve_size() const {
  98   // Reserve one small page per worker plus one shared medium page. This is still just
  99   // an estimate and doesn't guarantee that we can't run out of memory during relocation.
 100   const size_t max_reserve_size = (_workers.nworkers() * ZPageSizeSmall) + ZPageSizeMedium;
 101   return MIN2(max_reserve_size, heap_max_size());
 102 }
 103 
 104 bool ZHeap::is_initialized() const {
 105   return _page_allocator.is_initialized() && _mark.is_initialized();
 106 }
 107 
 108 size_t ZHeap::min_capacity() const {
 109   return _page_allocator.min_capacity();
 110 }
 111 
 112 size_t ZHeap::max_capacity() const {
 113   return _page_allocator.max_capacity();
 114 }
 115 
 116 size_t ZHeap::soft_max_capacity() const {
 117   return _page_allocator.soft_max_capacity();
 118 }
 119 
 120 size_t ZHeap::capacity() const {
 121   return _page_allocator.capacity();
 122 }
 123 
 124 size_t ZHeap::max_reserve() const {
 125   return _page_allocator.max_reserve();
 126 }
 127 
 128 size_t ZHeap::used_high() const {
 129   return _page_allocator.used_high();
 130 }
 131 
 132 size_t ZHeap::used_low() const {
 133   return _page_allocator.used_low();
 134 }
 135 
 136 size_t ZHeap::used() const {
 137   return _page_allocator.used();
 138 }
 139 
 140 size_t ZHeap::unused() const {
 141   return _page_allocator.unused();
 142 }
 143 
 144 size_t ZHeap::allocated() const {
 145   return _page_allocator.allocated();
 146 }
 147 
 148 size_t ZHeap::reclaimed() const {
 149   return _page_allocator.reclaimed();
 150 }
 151 
 152 size_t ZHeap::tlab_capacity() const {
 153   return capacity();
 154 }
 155 
 156 size_t ZHeap::tlab_used() const {
 157   return _object_allocator.used();
 158 }
 159 
 160 size_t ZHeap::max_tlab_size() const {
 161   return ZObjectSizeLimitSmall;
 162 }
 163 
 164 size_t ZHeap::unsafe_max_tlab_alloc() const {
 165   size_t size = _object_allocator.remaining();
 166 
 167   if (size < MinTLABSize) {
 168     // The remaining space in the allocator is not enough to
 169     // fit the smallest possible TLAB. This means that the next
 170     // TLAB allocation will force the allocator to get a new
 171     // backing page anyway, which in turn means that we can then
 172     // fit the largest possible TLAB.
 173     size = max_tlab_size();
 174   }
 175 
 176   return MIN2(size, max_tlab_size());
 177 }
 178 
 179 bool ZHeap::is_in(uintptr_t addr) const {
 180   // An address is considered to be "in the heap" if it points into
 181   // the allocated part of a pages, regardless of which heap view is
 182   // used. Note that an address with the finalizable metadata bit set
 183   // is not pointing into a heap view, and therefore not considered
 184   // to be "in the heap".
 185 
 186   if (ZAddress::is_in(addr)) {
 187     const ZPage* const page = _page_table.get(addr);
 188     if (page != NULL) {
 189       return page->is_in(addr);
 190     }
 191   }
 192 
 193   return false;
 194 }
 195 
 196 uintptr_t ZHeap::block_start(uintptr_t addr) const {
 197   const ZPage* const page = _page_table.get(addr);
 198   return page->block_start(addr);
 199 }
 200 
 201 bool ZHeap::block_is_obj(uintptr_t addr) const {
 202   const ZPage* const page = _page_table.get(addr);
 203   return page->block_is_obj(addr);
 204 }
 205 
 206 uint ZHeap::nconcurrent_worker_threads() const {
 207   return _workers.nconcurrent();
 208 }
 209 
 210 uint ZHeap::nconcurrent_no_boost_worker_threads() const {
 211   return _workers.nconcurrent_no_boost();
 212 }
 213 
 214 void ZHeap::set_boost_worker_threads(bool boost) {
 215   _workers.set_boost(boost);
 216 }
 217 
 218 void ZHeap::worker_threads_do(ThreadClosure* tc) const {
 219   _workers.threads_do(tc);
 220 }
 221 
 222 void ZHeap::print_worker_threads_on(outputStream* st) const {
 223   _workers.print_threads_on(st);
 224 }
 225 
 226 void ZHeap::out_of_memory() {
 227   ResourceMark rm;
 228 
 229   ZStatInc(ZCounterOutOfMemory);
 230   log_info(gc)("Out Of Memory (%s)", Thread::current()->name());
 231 }
 232 
 233 ZPage* ZHeap::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {
 234   ZPage* const page = _page_allocator.alloc_page(type, size, flags);
 235   if (page != NULL) {
 236     // Insert page table entry
 237     _page_table.insert(page);
 238   }
 239 
 240   return page;
 241 }
 242 
 243 void ZHeap::undo_alloc_page(ZPage* page) {
 244   assert(page->is_allocating(), "Invalid page state");
 245 
 246   ZStatInc(ZCounterUndoPageAllocation);
 247   log_trace(gc)("Undo page allocation, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT ", size: " SIZE_FORMAT,
 248                 ZThread::id(), ZThread::name(), p2i(page), page->size());
 249 
 250   free_page(page, false /* reclaimed */);
 251 }
 252 
 253 void ZHeap::free_page(ZPage* page, bool reclaimed) {
 254   // Remove page table entry
 255   _page_table.remove(page);
 256 
 257   // Free page
 258   _page_allocator.free_page(page, reclaimed);
 259 }
 260 
 261 uint64_t ZHeap::uncommit(uint64_t delay) {
 262   return _page_allocator.uncommit(delay);
 263 }
 264 
 265 void ZHeap::before_flip() {
 266   if (ZVerifyViews) {
 267     // Unmap all pages
 268     _page_allocator.debug_unmap_all_pages();
 269   }
 270 }
 271 
 272 void ZHeap::after_flip() {
 273   if (ZVerifyViews) {
 274     // Map all pages
 275     ZPageTableIterator iter(&_page_table);
 276     for (ZPage* page; iter.next(&page);) {
 277       _page_allocator.debug_map_page(page);
 278     }
 279     _page_allocator.debug_map_cached_pages();
 280   }
 281 }
 282 
 283 void ZHeap::flip_to_marked() {
 284   before_flip();
 285   ZAddress::flip_to_marked();
 286   after_flip();
 287 }
 288 
 289 void ZHeap::flip_to_remapped() {
 290   before_flip();
 291   ZAddress::flip_to_remapped();
 292   after_flip();
 293 }
 294 
 295 void ZHeap::mark_start() {
 296   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 297 
 298   // Update statistics
 299   ZStatSample(ZSamplerHeapUsedBeforeMark, used());
 300 
 301   // Flip address view
 302   flip_to_marked();
 303 
 304   // Retire allocating pages
 305   _object_allocator.retire_pages();
 306 
 307   // Reset allocated/reclaimed/used statistics
 308   _page_allocator.reset_statistics();
 309 
 310   // Reset encountered/dropped/enqueued statistics
 311   _reference_processor.reset_statistics();
 312 
 313   // Enter mark phase
 314   ZGlobalPhase = ZPhaseMark;
 315 
 316   // Reset marking information and mark roots
 317   _mark.start();
 318 
 319   // Update statistics
 320   ZStatHeap::set_at_mark_start(soft_max_capacity(), capacity(), used());
 321 }
 322 
 323 void ZHeap::mark(bool initial) {
 324   _mark.mark(initial);
 325 }
 326 
 327 void ZHeap::mark_flush_and_free(Thread* thread) {
 328   _mark.flush_and_free(thread);
 329 }
 330 
 331 bool ZHeap::mark_end() {
 332   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 333 
 334   // Try end marking
 335   if (!_mark.end()) {
 336     // Marking not completed, continue concurrent mark
 337     return false;
 338   }
 339 
 340   // Enter mark completed phase
 341   ZGlobalPhase = ZPhaseMarkCompleted;
 342 
 343   // Update statistics
 344   ZStatSample(ZSamplerHeapUsedAfterMark, used());
 345   ZStatHeap::set_at_mark_end(capacity(), allocated(), used());
 346 
 347   // Block resurrection of weak/phantom references
 348   ZResurrection::block();
 349 
 350   // Process weak roots
 351   _weak_roots_processor.process_weak_roots();
 352 
 353   // Prepare to unload unused classes and code
 354   _unload.prepare();
 355 
 356   return true;
 357 }
 358 
 359 void ZHeap::set_soft_reference_policy(bool clear) {
 360   _reference_processor.set_soft_reference_policy(clear);
 361 }
 362 
 363 void ZHeap::process_non_strong_references() {
 364   // Process Soft/Weak/Final/PhantomReferences
 365   _reference_processor.process_references();
 366 
 367   // Process concurrent weak roots
 368   _weak_roots_processor.process_concurrent_weak_roots();
 369 
 370   // Unload unused classes and code
 371   _unload.unload();
 372 
 373   // Unblock resurrection of weak/phantom references
 374   ZResurrection::unblock();
 375 
 376   // Enqueue Soft/Weak/Final/PhantomReferences. Note that this
 377   // must be done after unblocking resurrection. Otherwise the
 378   // Finalizer thread could call Reference.get() on the Finalizers
 379   // that were just enqueued, which would incorrectly return null
 380   // during the resurrection block window, since such referents
 381   // are only Finalizable marked.
 382   _reference_processor.enqueue_references();
 383 }
 384 
 385 void ZHeap::select_relocation_set() {
 386   // Do not allow pages to be deleted
 387   _page_allocator.enable_deferred_delete();
 388 
 389   // Register relocatable pages with selector
 390   ZRelocationSetSelector selector;
 391   ZPageTableIterator pt_iter(&_page_table);
 392   for (ZPage* page; pt_iter.next(&page);) {
 393     if (!page->is_relocatable()) {
 394       // Not relocatable, don't register
 395       continue;
 396     }
 397 
 398     if (page->is_marked()) {
 399       // Register live page
 400       selector.register_live_page(page);
 401     } else {
 402       // Register garbage page
 403       selector.register_garbage_page(page);
 404 
 405       // Reclaim page immediately
 406       free_page(page, true /* reclaimed */);
 407     }
 408   }
 409 
 410   // Allow pages to be deleted
 411   _page_allocator.disable_deferred_delete();
 412 
 413   // Select pages to relocate
 414   selector.select(&_relocation_set);
 415 
 416   // Setup forwarding table
 417   ZRelocationSetIterator rs_iter(&_relocation_set);
 418   for (ZForwarding* forwarding; rs_iter.next(&forwarding);) {
 419     _forwarding_table.insert(forwarding);
 420   }
 421 
 422   // Update statistics
 423   ZStatRelocation::set_at_select_relocation_set(selector.relocating());
 424   ZStatHeap::set_at_select_relocation_set(selector.live(),
 425                                           selector.garbage(),
 426                                           reclaimed());
 427 }
 428 
 429 void ZHeap::reset_relocation_set() {
 430   // Reset forwarding table
 431   ZRelocationSetIterator iter(&_relocation_set);
 432   for (ZForwarding* forwarding; iter.next(&forwarding);) {
 433     _forwarding_table.remove(forwarding);
 434   }
 435 
 436   // Reset relocation set
 437   _relocation_set.reset();
 438 }
 439 
 440 void ZHeap::relocate_start() {
 441   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 442 
 443   // Finish unloading of classes and code
 444   _unload.finish();
 445 
 446   // Flip address view
 447   flip_to_remapped();
 448 
 449   // Enter relocate phase
 450   ZGlobalPhase = ZPhaseRelocate;
 451 
 452   // Update statistics
 453   ZStatSample(ZSamplerHeapUsedBeforeRelocation, used());
 454   ZStatHeap::set_at_relocate_start(capacity(), allocated(), used());
 455 
 456   // Remap/Relocate roots
 457   _relocate.start();
 458 }
 459 
 460 void ZHeap::relocate() {
 461   // Relocate relocation set
 462   const bool success = _relocate.relocate(&_relocation_set);
 463 
 464   // Update statistics
 465   ZStatSample(ZSamplerHeapUsedAfterRelocation, used());
 466   ZStatRelocation::set_at_relocate_end(success);
 467   ZStatHeap::set_at_relocate_end(capacity(), allocated(), reclaimed(),
 468                                  used(), used_high(), used_low());
 469 }
 470 
 471 void ZHeap::object_iterate(ObjectClosure* cl, bool visit_referents) {
 472   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 473 
 474   ZHeapIterator iter;
 475   iter.objects_do(cl, visit_referents);
 476 }
 477 
 478 void ZHeap::serviceability_initialize() {
 479   _serviceability.initialize();
 480 }
 481 
 482 GCMemoryManager* ZHeap::serviceability_memory_manager() {
 483   return _serviceability.memory_manager();
 484 }
 485 
 486 MemoryPool* ZHeap::serviceability_memory_pool() {
 487   return _serviceability.memory_pool();
 488 }
 489 
 490 ZServiceabilityCounters* ZHeap::serviceability_counters() {
 491   return _serviceability.counters();
 492 }
 493 
 494 void ZHeap::print_on(outputStream* st) const {
 495   st->print_cr(" ZHeap           used " SIZE_FORMAT "M, capacity " SIZE_FORMAT "M, max capacity " SIZE_FORMAT "M",
 496                used() / M,
 497                capacity() / M,
 498                max_capacity() / M);
 499   MetaspaceUtils::print_on(st);
 500 }
 501 
 502 void ZHeap::print_extended_on(outputStream* st) const {
 503   print_on(st);
 504   st->cr();
 505 
 506   // Do not allow pages to be deleted
 507   _page_allocator.enable_deferred_delete();
 508 
 509   // Print all pages
 510   ZPageTableIterator iter(&_page_table);
 511   for (ZPage* page; iter.next(&page);) {
 512     page->print_on(st);
 513   }
 514 
 515   // Allow pages to be deleted
 516   _page_allocator.enable_deferred_delete();
 517 
 518   st->cr();
 519 }
 520 
 521 class ZVerifyRootsTask : public ZTask {
 522 private:
 523   ZStatTimerDisable  _disable;
 524   ZRootsIterator     _strong_roots;
 525   ZWeakRootsIterator _weak_roots;
 526 
 527 public:
 528   ZVerifyRootsTask() :
 529       ZTask("ZVerifyRootsTask"),
 530       _disable(),
 531       _strong_roots(),
 532       _weak_roots() {}
 533 
 534   virtual void work() {
 535     ZStatTimerDisable disable;
 536     ZVerifyOopClosure cl;
 537     _strong_roots.oops_do(&cl);
 538     _weak_roots.oops_do(&cl);
 539   }
 540 };
 541 
 542 void ZHeap::verify() {
 543   // Heap verification can only be done between mark end and
 544   // relocate start. This is the only window where all oop are
 545   // good and the whole heap is in a consistent state.
 546   guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase");
 547 
 548   {
 549     ZVerifyRootsTask task;
 550     _workers.run_parallel(&task);
 551   }
 552 
 553   {
 554     ZVerifyObjectClosure cl;
 555     object_iterate(&cl, false /* visit_referents */);
 556   }
 557 }