< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp

Print this page

  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 
  27 #include "compiler/oopMap.hpp"
  28 #include "gc/shared/continuationGCSupport.hpp"
  29 #include "gc/shared/gcTraceTime.inline.hpp"
  30 #include "gc/shared/preservedMarks.inline.hpp"

  31 #include "gc/shared/tlab_globals.hpp"
  32 #include "gc/shared/workerThread.hpp"
  33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
  35 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  36 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  37 #include "gc/shenandoah/shenandoahFullGC.hpp"
  38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  39 #include "gc/shenandoah/shenandoahMark.inline.hpp"
  40 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  41 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  42 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  43 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  44 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  45 #include "gc/shenandoah/shenandoahMetrics.hpp"
  46 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  47 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  48 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  49 #include "gc/shenandoah/shenandoahSTWMark.hpp"
  50 #include "gc/shenandoah/shenandoahUtils.hpp"

 204   // Coming out of Full GC, we would not have any forwarded objects.
 205   // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
 206   heap->set_has_forwarded_objects(false);
 207 
 208   heap->set_full_gc_move_in_progress(true);
 209 
 210   // Setup workers for the rest
 211   OrderAccess::fence();
 212 
 213   // Initialize worker slices
 214   ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
 215   for (uint i = 0; i < heap->max_workers(); i++) {
 216     worker_slices[i] = new ShenandoahHeapRegionSet();
 217   }
 218 
 219   {
 220     // The rest of code performs region moves, where region status is undefined
 221     // until all phases run together.
 222     ShenandoahHeapLocker lock(heap->lock());
 223 

 224     phase2_calculate_target_addresses(worker_slices);
 225 
 226     OrderAccess::fence();
 227 
 228     phase3_update_references();
 229 
 230     phase4_compact_objects(worker_slices);
 231   }
 232 
 233   {
 234     // Epilogue

 235     _preserved_marks->restore(heap->workers());
 236     _preserved_marks->reclaim();
 237   }
 238 
 239   // Resize metaspace
 240   MetaspaceGC::compute_new_size();
 241 
 242   // Free worker slices
 243   for (uint i = 0; i < heap->max_workers(); i++) {
 244     delete worker_slices[i];
 245   }
 246   FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
 247 
 248   heap->set_full_gc_move_in_progress(false);
 249   heap->set_full_gc_in_progress(false);
 250 
 251   if (ShenandoahVerify) {
 252     heap->verifier()->verify_after_fullgc();
 253   }
 254 

 346       // Object doesn't fit. Pick next empty region and start compacting there.
 347       ShenandoahHeapRegion* new_to_region;
 348       if (_empty_regions_pos < _empty_regions.length()) {
 349         new_to_region = _empty_regions.at(_empty_regions_pos);
 350         _empty_regions_pos++;
 351       } else {
 352         // Out of empty region? Compact within the same region.
 353         new_to_region = _from_region;
 354       }
 355 
 356       assert(new_to_region != _to_region, "must not reuse same to-region");
 357       assert(new_to_region != nullptr, "must not be null");
 358       _to_region = new_to_region;
 359       _compact_point = _to_region->bottom();
 360     }
 361 
 362     // Object fits into current region, record new location:
 363     assert(_compact_point + obj_size <= _to_region->end(), "must fit");
 364     shenandoah_assert_not_forwarded(nullptr, p);
 365     _preserved_marks->push_if_necessary(p, p->mark());
 366     p->forward_to(cast_to_oop(_compact_point));
 367     _compact_point += obj_size;
 368   }
 369 };
 370 
 371 class ShenandoahPrepareForCompactionTask : public WorkerTask {
 372 private:
 373   PreservedMarksSet*        const _preserved_marks;
 374   ShenandoahHeap*           const _heap;
 375   ShenandoahHeapRegionSet** const _worker_slices;
 376 
 377 public:
 378   ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices) :
 379     WorkerTask("Shenandoah Prepare For Compaction"),
 380     _preserved_marks(preserved_marks),
 381     _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
 382   }
 383 
 384   static bool is_candidate_region(ShenandoahHeapRegion* r) {
 385     // Empty region: get it into the slice to defragment the slice itself.
 386     // We could have skipped this without violating correctness, but we really

 399     ShenandoahHeapRegionSetIterator it(slice);
 400     ShenandoahHeapRegion* from_region = it.next();
 401     // No work?
 402     if (from_region == nullptr) {
 403        return;
 404     }
 405 
 406     // Sliding compaction. Walk all regions in the slice, and compact them.
 407     // Remember empty regions and reuse them as needed.
 408     ResourceMark rm;
 409 
 410     GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
 411 
 412     ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
 413 
 414     while (from_region != nullptr) {
 415       assert(is_candidate_region(from_region), "Sanity");
 416 
 417       cl.set_from_region(from_region);
 418       if (from_region->has_live()) {

 419         _heap->marked_object_iterate(from_region, &cl);
 420       }
 421 
 422       // Compacted the region to somewhere else? From-region is empty then.
 423       if (!cl.is_compact_same_region()) {
 424         empty_regions.append(from_region);
 425       }
 426       from_region = it.next();
 427     }
 428     cl.finish_region();
 429 
 430     // Mark all remaining regions as empty
 431     for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
 432       ShenandoahHeapRegion* r = empty_regions.at(pos);
 433       r->set_new_top(r->bottom());
 434     }
 435   }
 436 };
 437 
 438 void ShenandoahFullGC::calculate_target_humongous_objects() {

 454 
 455   for (size_t c = heap->num_regions(); c > 0; c--) {
 456     ShenandoahHeapRegion *r = heap->get_region(c - 1);
 457     if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
 458       // To-region candidate: record this, and continue scan
 459       to_begin = r->index();
 460       continue;
 461     }
 462 
 463     if (r->is_humongous_start() && r->is_stw_move_allowed()) {
 464       // From-region candidate: movable humongous region
 465       oop old_obj = cast_to_oop(r->bottom());
 466       size_t words_size = old_obj->size();
 467       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 468 
 469       size_t start = to_end - num_regions;
 470 
 471       if (start >= to_begin && start != r->index()) {
 472         // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
 473         _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
 474         old_obj->forward_to(cast_to_oop(heap->get_region(start)->bottom()));
 475         to_end = start;
 476         continue;
 477       }
 478     }
 479 
 480     // Failed to fit. Scan starting from current region.
 481     to_begin = r->index();
 482     to_end = r->index();
 483   }
 484 }
 485 
 486 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
 487 private:
 488   ShenandoahHeap* const _heap;
 489 
 490 public:
 491   ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
 492   void heap_region_do(ShenandoahHeapRegion* r) {
 493     if (r->is_trash()) {
 494       r->recycle();

 714   }
 715 
 716   // Compute the new addresses for humongous objects
 717   {
 718     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_humong);
 719     calculate_target_humongous_objects();
 720   }
 721 }
 722 
 723 class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure {
 724 private:
 725   ShenandoahHeap* const _heap;
 726   ShenandoahMarkingContext* const _ctx;
 727 
 728   template <class T>
 729   inline void do_oop_work(T* p) {
 730     T o = RawAccess<>::oop_load(p);
 731     if (!CompressedOops::is_null(o)) {
 732       oop obj = CompressedOops::decode_not_null(o);
 733       assert(_ctx->is_marked(obj), "must be marked");
 734       if (obj->is_forwarded()) {
 735         oop forw = obj->forwardee();
 736         RawAccess<IS_NOT_NULL>::oop_store(p, forw);
 737       }
 738     }
 739   }
 740 
 741 public:
 742   ShenandoahAdjustPointersClosure() :
 743     _heap(ShenandoahHeap::heap()),
 744     _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
 745 
 746   void do_oop(oop* p)       { do_oop_work(p); }
 747   void do_oop(narrowOop* p) { do_oop_work(p); }
 748   void do_method(Method* m) {}
 749   void do_nmethod(nmethod* nm) {}
 750 };
 751 
 752 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
 753 private:
 754   ShenandoahHeap* const _heap;
 755   ShenandoahAdjustPointersClosure _cl;

 825     DerivedPointerTable::update_pointers();
 826 #endif
 827   }
 828 
 829   ShenandoahAdjustPointersTask adjust_pointers_task;
 830   workers->run_task(&adjust_pointers_task);
 831 }
 832 
 833 class ShenandoahCompactObjectsClosure : public ObjectClosure {
 834 private:
 835   ShenandoahHeap* const _heap;
 836   uint            const _worker_id;
 837 
 838 public:
 839   ShenandoahCompactObjectsClosure(uint worker_id) :
 840     _heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
 841 
 842   void do_object(oop p) {
 843     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
 844     size_t size = p->size();
 845     if (p->is_forwarded()) {
 846       HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
 847       HeapWord* compact_to = cast_from_oop<HeapWord*>(p->forwardee());
 848       Copy::aligned_conjoint_words(compact_from, compact_to, size);
 849       oop new_obj = cast_to_oop(compact_to);
 850 
 851       ContinuationGCSupport::relativize_stack_chunk(new_obj);
 852       new_obj->init_mark();
 853     }
 854   }
 855 };
 856 
 857 class ShenandoahCompactObjectsTask : public WorkerTask {
 858 private:
 859   ShenandoahHeap* const _heap;
 860   ShenandoahHeapRegionSet** const _worker_slices;
 861 
 862 public:
 863   ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
 864     WorkerTask("Shenandoah Compact Objects"),
 865     _heap(ShenandoahHeap::heap()),
 866     _worker_slices(worker_slices) {
 867   }

 929   }
 930 
 931   size_t get_live() {
 932     return _live;
 933   }
 934 };
 935 
 936 void ShenandoahFullGC::compact_humongous_objects() {
 937   // Compact humongous regions, based on their fwdptr objects.
 938   //
 939   // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
 940   // humongous regions are already compacted, and do not require further moves, which alleviates
 941   // sliding costs. We may consider doing this in parallel in future.
 942 
 943   ShenandoahHeap* heap = ShenandoahHeap::heap();
 944 
 945   for (size_t c = heap->num_regions(); c > 0; c--) {
 946     ShenandoahHeapRegion* r = heap->get_region(c - 1);
 947     if (r->is_humongous_start()) {
 948       oop old_obj = cast_to_oop(r->bottom());
 949       if (!old_obj->is_forwarded()) {
 950         // No need to move the object, it stays at the same slot
 951         continue;
 952       }
 953       size_t words_size = old_obj->size();
 954       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 955 
 956       size_t old_start = r->index();
 957       size_t old_end   = old_start + num_regions - 1;
 958       size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
 959       size_t new_end   = new_start + num_regions - 1;
 960       assert(old_start != new_start, "must be real move");
 961       assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index());
 962 
 963       Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size);
 964       ContinuationGCSupport::relativize_stack_chunk(cast_to_oop<HeapWord*>(r->bottom()));
 965 
 966       oop new_obj = cast_to_oop(heap->get_region(new_start)->bottom());
 967       new_obj->init_mark();
 968 
 969       {
 970         for (size_t c = old_start; c <= old_end; c++) {
 971           ShenandoahHeapRegion* r = heap->get_region(c);
 972           r->make_regular_bypass();
 973           r->set_top(r->bottom());
 974         }
 975 
 976         for (size_t c = new_start; c <= new_end; c++) {
 977           ShenandoahHeapRegion* r = heap->get_region(c);
 978           if (c == new_start) {

  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 
  27 #include "compiler/oopMap.hpp"
  28 #include "gc/shared/continuationGCSupport.hpp"
  29 #include "gc/shared/gcTraceTime.inline.hpp"
  30 #include "gc/shared/preservedMarks.inline.hpp"
  31 #include "gc/shared/gcForwarding.inline.hpp"
  32 #include "gc/shared/tlab_globals.hpp"
  33 #include "gc/shared/workerThread.hpp"
  34 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  35 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
  36 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  37 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  38 #include "gc/shenandoah/shenandoahFullGC.hpp"
  39 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  40 #include "gc/shenandoah/shenandoahMark.inline.hpp"
  41 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  42 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  43 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  44 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  45 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  46 #include "gc/shenandoah/shenandoahMetrics.hpp"
  47 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  48 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  49 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  50 #include "gc/shenandoah/shenandoahSTWMark.hpp"
  51 #include "gc/shenandoah/shenandoahUtils.hpp"

 205   // Coming out of Full GC, we would not have any forwarded objects.
 206   // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
 207   heap->set_has_forwarded_objects(false);
 208 
 209   heap->set_full_gc_move_in_progress(true);
 210 
 211   // Setup workers for the rest
 212   OrderAccess::fence();
 213 
 214   // Initialize worker slices
 215   ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
 216   for (uint i = 0; i < heap->max_workers(); i++) {
 217     worker_slices[i] = new ShenandoahHeapRegionSet();
 218   }
 219 
 220   {
 221     // The rest of code performs region moves, where region status is undefined
 222     // until all phases run together.
 223     ShenandoahHeapLocker lock(heap->lock());
 224 
 225     GCForwarding::begin();
 226     phase2_calculate_target_addresses(worker_slices);
 227 
 228     OrderAccess::fence();
 229 
 230     phase3_update_references();
 231 
 232     phase4_compact_objects(worker_slices);
 233   }
 234 
 235   {
 236     // Epilogue
 237     GCForwarding::end();
 238     _preserved_marks->restore(heap->workers());
 239     _preserved_marks->reclaim();
 240   }
 241 
 242   // Resize metaspace
 243   MetaspaceGC::compute_new_size();
 244 
 245   // Free worker slices
 246   for (uint i = 0; i < heap->max_workers(); i++) {
 247     delete worker_slices[i];
 248   }
 249   FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
 250 
 251   heap->set_full_gc_move_in_progress(false);
 252   heap->set_full_gc_in_progress(false);
 253 
 254   if (ShenandoahVerify) {
 255     heap->verifier()->verify_after_fullgc();
 256   }
 257 

 349       // Object doesn't fit. Pick next empty region and start compacting there.
 350       ShenandoahHeapRegion* new_to_region;
 351       if (_empty_regions_pos < _empty_regions.length()) {
 352         new_to_region = _empty_regions.at(_empty_regions_pos);
 353         _empty_regions_pos++;
 354       } else {
 355         // Out of empty region? Compact within the same region.
 356         new_to_region = _from_region;
 357       }
 358 
 359       assert(new_to_region != _to_region, "must not reuse same to-region");
 360       assert(new_to_region != nullptr, "must not be null");
 361       _to_region = new_to_region;
 362       _compact_point = _to_region->bottom();
 363     }
 364 
 365     // Object fits into current region, record new location:
 366     assert(_compact_point + obj_size <= _to_region->end(), "must fit");
 367     shenandoah_assert_not_forwarded(nullptr, p);
 368     _preserved_marks->push_if_necessary(p, p->mark());
 369     GCForwarding::forward_to(p, cast_to_oop(_compact_point));
 370     _compact_point += obj_size;
 371   }
 372 };
 373 
 374 class ShenandoahPrepareForCompactionTask : public WorkerTask {
 375 private:
 376   PreservedMarksSet*        const _preserved_marks;
 377   ShenandoahHeap*           const _heap;
 378   ShenandoahHeapRegionSet** const _worker_slices;
 379 
 380 public:
 381   ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices) :
 382     WorkerTask("Shenandoah Prepare For Compaction"),
 383     _preserved_marks(preserved_marks),
 384     _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
 385   }
 386 
 387   static bool is_candidate_region(ShenandoahHeapRegion* r) {
 388     // Empty region: get it into the slice to defragment the slice itself.
 389     // We could have skipped this without violating correctness, but we really

 402     ShenandoahHeapRegionSetIterator it(slice);
 403     ShenandoahHeapRegion* from_region = it.next();
 404     // No work?
 405     if (from_region == nullptr) {
 406        return;
 407     }
 408 
 409     // Sliding compaction. Walk all regions in the slice, and compact them.
 410     // Remember empty regions and reuse them as needed.
 411     ResourceMark rm;
 412 
 413     GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
 414 
 415     ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
 416 
 417     while (from_region != nullptr) {
 418       assert(is_candidate_region(from_region), "Sanity");
 419 
 420       cl.set_from_region(from_region);
 421       if (from_region->has_live()) {
 422         size_t num_marked = _heap->complete_marking_context()->count_marked(MemRegion(from_region->bottom(), from_region->top()));
 423         _heap->marked_object_iterate(from_region, &cl);
 424       }
 425 
 426       // Compacted the region to somewhere else? From-region is empty then.
 427       if (!cl.is_compact_same_region()) {
 428         empty_regions.append(from_region);
 429       }
 430       from_region = it.next();
 431     }
 432     cl.finish_region();
 433 
 434     // Mark all remaining regions as empty
 435     for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
 436       ShenandoahHeapRegion* r = empty_regions.at(pos);
 437       r->set_new_top(r->bottom());
 438     }
 439   }
 440 };
 441 
 442 void ShenandoahFullGC::calculate_target_humongous_objects() {

 458 
 459   for (size_t c = heap->num_regions(); c > 0; c--) {
 460     ShenandoahHeapRegion *r = heap->get_region(c - 1);
 461     if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
 462       // To-region candidate: record this, and continue scan
 463       to_begin = r->index();
 464       continue;
 465     }
 466 
 467     if (r->is_humongous_start() && r->is_stw_move_allowed()) {
 468       // From-region candidate: movable humongous region
 469       oop old_obj = cast_to_oop(r->bottom());
 470       size_t words_size = old_obj->size();
 471       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 472 
 473       size_t start = to_end - num_regions;
 474 
 475       if (start >= to_begin && start != r->index()) {
 476         // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
 477         _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
 478         GCForwarding::forward_to(old_obj, cast_to_oop(heap->get_region(start)->bottom()));
 479         to_end = start;
 480         continue;
 481       }
 482     }
 483 
 484     // Failed to fit. Scan starting from current region.
 485     to_begin = r->index();
 486     to_end = r->index();
 487   }
 488 }
 489 
 490 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
 491 private:
 492   ShenandoahHeap* const _heap;
 493 
 494 public:
 495   ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
 496   void heap_region_do(ShenandoahHeapRegion* r) {
 497     if (r->is_trash()) {
 498       r->recycle();

 718   }
 719 
 720   // Compute the new addresses for humongous objects
 721   {
 722     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_humong);
 723     calculate_target_humongous_objects();
 724   }
 725 }
 726 
 727 class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure {
 728 private:
 729   ShenandoahHeap* const _heap;
 730   ShenandoahMarkingContext* const _ctx;
 731 
 732   template <class T>
 733   inline void do_oop_work(T* p) {
 734     T o = RawAccess<>::oop_load(p);
 735     if (!CompressedOops::is_null(o)) {
 736       oop obj = CompressedOops::decode_not_null(o);
 737       assert(_ctx->is_marked(obj), "must be marked");
 738       if (GCForwarding::is_forwarded(obj)) {
 739         oop forw = GCForwarding::forwardee(obj);
 740         RawAccess<IS_NOT_NULL>::oop_store(p, forw);
 741       }
 742     }
 743   }
 744 
 745 public:
 746   ShenandoahAdjustPointersClosure() :
 747     _heap(ShenandoahHeap::heap()),
 748     _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
 749 
 750   void do_oop(oop* p)       { do_oop_work(p); }
 751   void do_oop(narrowOop* p) { do_oop_work(p); }
 752   void do_method(Method* m) {}
 753   void do_nmethod(nmethod* nm) {}
 754 };
 755 
 756 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
 757 private:
 758   ShenandoahHeap* const _heap;
 759   ShenandoahAdjustPointersClosure _cl;

 829     DerivedPointerTable::update_pointers();
 830 #endif
 831   }
 832 
 833   ShenandoahAdjustPointersTask adjust_pointers_task;
 834   workers->run_task(&adjust_pointers_task);
 835 }
 836 
 837 class ShenandoahCompactObjectsClosure : public ObjectClosure {
 838 private:
 839   ShenandoahHeap* const _heap;
 840   uint            const _worker_id;
 841 
 842 public:
 843   ShenandoahCompactObjectsClosure(uint worker_id) :
 844     _heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
 845 
 846   void do_object(oop p) {
 847     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
 848     size_t size = p->size();
 849     if (GCForwarding::is_forwarded(p)) {
 850       HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
 851       HeapWord* compact_to = cast_from_oop<HeapWord*>(GCForwarding::forwardee(p));
 852       Copy::aligned_conjoint_words(compact_from, compact_to, size);
 853       oop new_obj = cast_to_oop(compact_to);
 854 
 855       ContinuationGCSupport::relativize_stack_chunk(new_obj);
 856       new_obj->init_mark();
 857     }
 858   }
 859 };
 860 
 861 class ShenandoahCompactObjectsTask : public WorkerTask {
 862 private:
 863   ShenandoahHeap* const _heap;
 864   ShenandoahHeapRegionSet** const _worker_slices;
 865 
 866 public:
 867   ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
 868     WorkerTask("Shenandoah Compact Objects"),
 869     _heap(ShenandoahHeap::heap()),
 870     _worker_slices(worker_slices) {
 871   }

 933   }
 934 
 935   size_t get_live() {
 936     return _live;
 937   }
 938 };
 939 
 940 void ShenandoahFullGC::compact_humongous_objects() {
 941   // Compact humongous regions, based on their fwdptr objects.
 942   //
 943   // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
 944   // humongous regions are already compacted, and do not require further moves, which alleviates
 945   // sliding costs. We may consider doing this in parallel in future.
 946 
 947   ShenandoahHeap* heap = ShenandoahHeap::heap();
 948 
 949   for (size_t c = heap->num_regions(); c > 0; c--) {
 950     ShenandoahHeapRegion* r = heap->get_region(c - 1);
 951     if (r->is_humongous_start()) {
 952       oop old_obj = cast_to_oop(r->bottom());
 953       if (GCForwarding::is_not_forwarded(old_obj)) {
 954         // No need to move the object, it stays at the same slot
 955         continue;
 956       }
 957       size_t words_size = old_obj->size();
 958       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 959 
 960       size_t old_start = r->index();
 961       size_t old_end   = old_start + num_regions - 1;
 962       size_t new_start = heap->heap_region_index_containing(GCForwarding::forwardee(old_obj));
 963       size_t new_end   = new_start + num_regions - 1;
 964       assert(old_start != new_start, "must be real move");
 965       assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index());
 966 
 967       Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size);
 968       ContinuationGCSupport::relativize_stack_chunk(cast_to_oop<HeapWord*>(r->bottom()));
 969 
 970       oop new_obj = cast_to_oop(heap->get_region(new_start)->bottom());
 971       new_obj->init_mark();
 972 
 973       {
 974         for (size_t c = old_start; c <= old_end; c++) {
 975           ShenandoahHeapRegion* r = heap->get_region(c);
 976           r->make_regular_bypass();
 977           r->set_top(r->bottom());
 978         }
 979 
 980         for (size_t c = new_start; c <= new_end; c++) {
 981           ShenandoahHeapRegion* r = heap->get_region(c);
 982           if (c == new_start) {
< prev index next >