1 /*
   2  * Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 
  26 #include "code/codeCache.hpp"
  27 #include "gc_implementation/shenandoah/shenandoahGCTraceTime.hpp"
  28 #include "gc_implementation/shared/gcTimer.hpp"
  29 #include "gc_implementation/shenandoah/preservedMarks.inline.hpp"
  30 #include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
  31 #include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
  32 #include "gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp"
  33 #include "gc_implementation/shenandoah/shenandoahCollectionSet.hpp"
  34 #include "gc_implementation/shenandoah/shenandoahFreeSet.hpp"
  35 #include "gc_implementation/shenandoah/shenandoahPhaseTimings.hpp"
  36 #include "gc_implementation/shenandoah/shenandoahMarkCompact.hpp"
  37 #include "gc_implementation/shenandoah/shenandoahHeapRegionSet.hpp"
  38 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
  39 #include "gc_implementation/shenandoah/shenandoahHeuristics.hpp"
  40 #include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp"
  41 #include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp"
  42 #include "gc_implementation/shenandoah/shenandoahTraversalGC.hpp"
  43 #include "gc_implementation/shenandoah/shenandoahTaskqueue.inline.hpp"
  44 #include "gc_implementation/shenandoah/shenandoahUtils.hpp"
  45 #include "gc_implementation/shenandoah/shenandoahVerifier.hpp"
  46 #include "gc_implementation/shenandoah/shenandoahVMOperations.hpp"
  47 #include "gc_implementation/shenandoah/shenandoahWorkGroup.hpp"
  48 #include "gc_implementation/shenandoah/shenandoahWorkerPolicy.hpp"
  49 #include "memory/metaspace.hpp"
  50 #include "oops/oop.inline.hpp"
  51 #include "runtime/biasedLocking.hpp"
  52 #include "runtime/thread.hpp"
  53 #include "utilities/copy.hpp"
  54 #include "utilities/growableArray.hpp"
  55 #include "utilities/workgroup.hpp"
  56 
  57 ShenandoahMarkCompact::ShenandoahMarkCompact() :
  58   _gc_timer(NULL),
  59   _preserved_marks(new PreservedMarksSet(true)) {}
  60 
  61 void ShenandoahMarkCompact::initialize(GCTimer* gc_timer) {
  62   _gc_timer = gc_timer;
  63 }
  64 
  65 void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
  66   ShenandoahHeap* heap = ShenandoahHeap::heap();
  67 
  68   if (ShenandoahVerify) {
  69     heap->verifier()->verify_before_fullgc();
  70   }
  71 
  72   if (VerifyBeforeGC) {
  73     Universe::verify();
  74   }
  75 
  76   heap->set_full_gc_in_progress(true);
  77 
  78   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
  79   assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
  80 
  81   {
  82     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
  83     heap->pre_full_gc_dump(_gc_timer);
  84   }
  85 
  86   {
  87     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_prepare);
  88     // Full GC is supposed to recover from any GC state:
  89 
  90     // a0. Remember if we have forwarded objects
  91     bool has_forwarded_objects = heap->has_forwarded_objects();
  92 
  93     // a1. Cancel evacuation, if in progress
  94     if (heap->is_evacuation_in_progress()) {
  95       heap->set_evacuation_in_progress(false);
  96     }
  97     assert(!heap->is_evacuation_in_progress(), "sanity");
  98 
  99     // a2. Cancel update-refs, if in progress
 100     if (heap->is_update_refs_in_progress()) {
 101       heap->set_update_refs_in_progress(false);
 102     }
 103     assert(!heap->is_update_refs_in_progress(), "sanity");
 104 
 105     // a3. Cancel concurrent traversal GC, if in progress
 106     if (heap->is_concurrent_traversal_in_progress()) {
 107       heap->traversal_gc()->reset();
 108       heap->set_concurrent_traversal_in_progress(false);
 109     }
 110 
 111     // b. Cancel concurrent mark, if in progress
 112     if (heap->is_concurrent_mark_in_progress()) {
 113       heap->concurrent_mark()->cancel();
 114       heap->stop_concurrent_marking();
 115     }
 116     assert(!heap->is_concurrent_mark_in_progress(), "sanity");
 117 
 118     // c. Reset the bitmaps for new marking
 119     heap->reset_mark_bitmap();
 120     assert(heap->marking_context()->is_bitmap_clear(), "sanity");
 121     assert(!heap->marking_context()->is_complete(), "sanity");
 122 
 123     // d. Abandon reference discovery and clear all discovered references.
 124     ReferenceProcessor *rp = heap->ref_processor();
 125     rp->disable_discovery();
 126     rp->abandon_partial_discovery();
 127     rp->verify_no_references_recorded();
 128 
 129     // e. Set back forwarded objects bit back, in case some steps above dropped it.
 130     heap->set_has_forwarded_objects(has_forwarded_objects);
 131 
 132     // f. Sync pinned region status from the CP marks
 133     heap->sync_pinned_region_status();
 134 
 135     // The rest of prologue:
 136     BiasedLocking::preserve_marks();
 137     _preserved_marks->init(heap->workers()->active_workers());
 138   }
 139 
 140   heap->make_parsable(true);
 141 
 142   CodeCache::gc_prologue();
 143 
 144   OrderAccess::fence();
 145 
 146   phase1_mark_heap();
 147 
 148   // Once marking is done, which may have fixed up forwarded objects, we can drop it.
 149   // Coming out of Full GC, we would not have any forwarded objects.
 150   // This also prevents read barrier from kicking in while adjusting pointers in phase3.
 151   heap->set_has_forwarded_objects(false);
 152 
 153   heap->set_full_gc_move_in_progress(true);
 154 
 155   // Setup workers for the rest
 156   OrderAccess::fence();
 157 
 158   // Initialize worker slices
 159   ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
 160   for (uint i = 0; i < heap->max_workers(); i++) {
 161     worker_slices[i] = new ShenandoahHeapRegionSet();
 162   }
 163 
 164   {
 165     // The rest of code performs region moves, where region status is undefined
 166     // until all phases run together.
 167     ShenandoahHeapLocker lock(heap->lock());
 168 
 169     phase2_calculate_target_addresses(worker_slices);
 170 
 171     OrderAccess::fence();
 172 
 173     phase3_update_references();
 174 
 175     phase4_compact_objects(worker_slices);
 176   }
 177 
 178   {
 179     // Epilogue
 180     SharedRestorePreservedMarksTaskExecutor exec(heap->workers());
 181     _preserved_marks->restore(&exec);
 182     BiasedLocking::restore_marks();
 183     _preserved_marks->reclaim();
 184 
 185     JvmtiExport::gc_epilogue();
 186   }
 187 
 188   // Resize metaspace
 189   MetaspaceGC::compute_new_size();
 190 
 191   // Free worker slices
 192   for (uint i = 0; i < heap->max_workers(); i++) {
 193     delete worker_slices[i];
 194   }
 195   FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices, mtGC);
 196 
 197   CodeCache::gc_epilogue();
 198   JvmtiExport::gc_epilogue();
 199 
 200   heap->set_full_gc_move_in_progress(false);
 201   heap->set_full_gc_in_progress(false);
 202 
 203   if (ShenandoahVerify) {
 204     heap->verifier()->verify_after_fullgc();
 205   }
 206 
 207   if (VerifyAfterGC) {
 208     Universe::verify();
 209   }
 210 
 211   {
 212     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
 213     heap->post_full_gc_dump(_gc_timer);
 214   }
 215 
 216   if (UseTLAB) {
 217     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
 218     heap->resize_all_tlabs();
 219   }
 220 }
 221 
 222 class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
 223 private:
 224   ShenandoahMarkingContext* const _ctx;
 225 
 226 public:
 227   ShenandoahPrepareForMarkClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
 228 
 229   void heap_region_do(ShenandoahHeapRegion *r) {
 230     _ctx->capture_top_at_mark_start(r);
 231     r->clear_live_data();
 232     r->set_concurrent_iteration_safe_limit(r->top());
 233   }
 234 };
 235 
 236 void ShenandoahMarkCompact::phase1_mark_heap() {
 237   ShenandoahHeap* heap = ShenandoahHeap::heap();
 238   GCTraceTime time("Phase 1: Mark live objects", ShenandoahLogDebug, _gc_timer, heap->tracer()->gc_id());
 239   ShenandoahGCPhase mark_phase(ShenandoahPhaseTimings::full_gc_mark);
 240 
 241   ShenandoahPrepareForMarkClosure cl;
 242   heap->heap_region_iterate(&cl);
 243 
 244   ShenandoahConcurrentMark* cm = heap->concurrent_mark();
 245 
 246   heap->set_process_references(heap->heuristics()->can_process_references());
 247   heap->set_unload_classes(heap->heuristics()->can_unload_classes());
 248 
 249   ReferenceProcessor* rp = heap->ref_processor();
 250   // enable ("weak") refs discovery
 251   rp->enable_discovery(true /*verify_no_refs*/, true);
 252   rp->setup_policy(true); // forcefully purge all soft references
 253   rp->set_active_mt_degree(heap->workers()->active_workers());
 254 
 255   cm->update_roots(ShenandoahPhaseTimings::full_gc_roots);
 256   cm->mark_roots(ShenandoahPhaseTimings::full_gc_roots);
 257   cm->finish_mark_from_roots(/* full_gc = */ true);
 258 
 259   heap->mark_complete_marking_context();
 260 }
 261 
 262 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
 263 private:
 264   PreservedMarks*          const _preserved_marks;
 265   ShenandoahHeap*          const _heap;
 266   GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
 267   int _empty_regions_pos;
 268   ShenandoahHeapRegion*          _to_region;
 269   ShenandoahHeapRegion*          _from_region;
 270   HeapWord* _compact_point;
 271 
 272 public:
 273   ShenandoahPrepareForCompactionObjectClosure(PreservedMarks* preserved_marks,
 274                                               GrowableArray<ShenandoahHeapRegion*>& empty_regions,
 275                                               ShenandoahHeapRegion* to_region) :
 276     _preserved_marks(preserved_marks),
 277     _heap(ShenandoahHeap::heap()),
 278     _empty_regions(empty_regions),
 279     _empty_regions_pos(0),
 280     _to_region(to_region),
 281     _from_region(NULL),
 282     _compact_point(to_region->bottom()) {}
 283 
 284   void set_from_region(ShenandoahHeapRegion* from_region) {
 285     _from_region = from_region;
 286   }
 287 
 288   void finish_region() {
 289     assert(_to_region != NULL, "should not happen");
 290     _to_region->set_new_top(_compact_point);
 291   }
 292 
 293   bool is_compact_same_region() {
 294     return _from_region == _to_region;
 295   }
 296 
 297   int empty_regions_pos() {
 298     return _empty_regions_pos;
 299   }
 300 
 301   void do_object(oop p) {
 302     assert(_from_region != NULL, "must set before work");
 303     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
 304     assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
 305 
 306     size_t obj_size = p->size();
 307     if (_compact_point + obj_size > _to_region->end()) {
 308       finish_region();
 309 
 310       // Object doesn't fit. Pick next empty region and start compacting there.
 311       ShenandoahHeapRegion* new_to_region;
 312       if (_empty_regions_pos < _empty_regions.length()) {
 313         new_to_region = _empty_regions.at(_empty_regions_pos);
 314         _empty_regions_pos++;
 315       } else {
 316         // Out of empty region? Compact within the same region.
 317         new_to_region = _from_region;
 318       }
 319 
 320       assert(new_to_region != _to_region, "must not reuse same to-region");
 321       assert(new_to_region != NULL, "must not be NULL");
 322       _to_region = new_to_region;
 323       _compact_point = _to_region->bottom();
 324     }
 325 
 326     // Object fits into current region, record new location:
 327     assert(_compact_point + obj_size <= _to_region->end(), "must fit");
 328     shenandoah_assert_not_forwarded(NULL, p);
 329     _preserved_marks->push_if_necessary(p, p->mark());
 330     p->forward_to(oop(_compact_point));
 331     _compact_point += obj_size;
 332   }
 333 };
 334 
 335 class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
 336 private:
 337   PreservedMarksSet*        const _preserved_marks;
 338   ShenandoahHeap*           const _heap;
 339   ShenandoahHeapRegionSet** const _worker_slices;
 340   ShenandoahRegionIterator        _heap_regions;
 341 
 342   ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* slice) {
 343     ShenandoahHeapRegion* from_region = _heap_regions.next();
 344 
 345     // Look for next candidate for this slice:
 346     while (from_region != NULL) {
 347       // Empty region: get it into the slice to defragment the slice itself.
 348       // We could have skipped this without violating correctness, but we really
 349       // want to compact all live regions to the start of the heap, which sometimes
 350       // means moving them into the fully empty regions.
 351       if (from_region->is_empty()) break;
 352 
 353       // Can move the region, and this is not the humongous region. Humongous
 354       // moves are special cased here, because their moves are handled separately.
 355       if (from_region->is_stw_move_allowed() && !from_region->is_humongous()) break;
 356 
 357       from_region = _heap_regions.next();
 358     }
 359 
 360     if (from_region != NULL) {
 361       assert(slice != NULL, "sanity");
 362       assert(!from_region->is_humongous(), "this path cannot handle humongous regions");
 363       assert(from_region->is_empty() || from_region->is_stw_move_allowed(), "only regions that can be moved in mark-compact");
 364       slice->add_region(from_region);
 365     }
 366 
 367     return from_region;
 368   }
 369 
 370 public:
 371   ShenandoahPrepareForCompactionTask(PreservedMarksSet* preserved_marks, ShenandoahHeapRegionSet** worker_slices) :
 372     AbstractGangTask("Shenandoah Prepare For Compaction Task"),
 373     _preserved_marks(preserved_marks),
 374     _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
 375   }
 376 
 377   void work(uint worker_id) {
 378     ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
 379     ShenandoahHeapRegion* from_region = next_from_region(slice);
 380 
 381     // No work?
 382     if (from_region == NULL) {
 383       return;
 384     }
 385 
 386     // Sliding compaction. Walk all regions in the slice, and compact them.
 387     // Remember empty regions and reuse them as needed.
 388     ResourceMark rm;
 389     GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
 390     ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
 391     while (from_region != NULL) {
 392       cl.set_from_region(from_region);
 393       if (from_region->has_live()) {
 394         _heap->marked_object_iterate(from_region, &cl);
 395       }
 396 
 397       // Compacted the region to somewhere else? From-region is empty then.
 398       if (!cl.is_compact_same_region()) {
 399         empty_regions.append(from_region);
 400       }
 401       from_region = next_from_region(slice);
 402     }
 403     cl.finish_region();
 404 
 405     // Mark all remaining regions as empty
 406     for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
 407       ShenandoahHeapRegion* r = empty_regions.at(pos);
 408       r->set_new_top(r->bottom());
 409     }
 410   }
 411 };
 412 
 413 void ShenandoahMarkCompact::calculate_target_humongous_objects() {
 414   ShenandoahHeap* heap = ShenandoahHeap::heap();
 415 
 416   // Compute the new addresses for humongous objects. We need to do this after addresses
 417   // for regular objects are calculated, and we know what regions in heap suffix are
 418   // available for humongous moves.
 419   //
 420   // Scan the heap backwards, because we are compacting humongous regions towards the end.
 421   // Maintain the contiguous compaction window in [to_begin; to_end), so that we can slide
 422   // humongous start there.
 423   //
 424   // The complication is potential non-movable regions during the scan. If such region is
 425   // detected, then sliding restarts towards that non-movable region.
 426 
 427   size_t to_begin = heap->num_regions();
 428   size_t to_end = heap->num_regions();
 429 
 430   for (size_t c = heap->num_regions(); c > 0; c--) {
 431     ShenandoahHeapRegion *r = heap->get_region(c - 1);
 432     if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
 433       // To-region candidate: record this, and continue scan
 434       to_begin = r->region_number();
 435       continue;
 436     }
 437 
 438     if (r->is_humongous_start() && r->is_stw_move_allowed()) {
 439       // From-region candidate: movable humongous region
 440       oop old_obj = oop(r->bottom());
 441       size_t words_size = old_obj->size();
 442       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 443 
 444       size_t start = to_end - num_regions;
 445 
 446       if (start >= to_begin && start != r->region_number()) {
 447         // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
 448         _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
 449         old_obj->forward_to(oop(heap->get_region(start)->bottom()));
 450         to_end = start;
 451         continue;
 452       }
 453     }
 454 
 455     // Failed to fit. Scan starting from current region.
 456     to_begin = r->region_number();
 457     to_end = r->region_number();
 458   }
 459 }
 460 
 461 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
 462 private:
 463   ShenandoahHeap* const _heap;
 464 
 465 public:
 466   ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
 467   void heap_region_do(ShenandoahHeapRegion* r) {
 468     if (r->is_trash()) {
 469       r->recycle();
 470     }
 471     if (r->is_cset()) {
 472       r->make_regular_bypass();
 473     }
 474     if (r->is_empty_uncommitted()) {
 475       r->make_committed_bypass();
 476     }
 477     assert (r->is_committed(), err_msg("only committed regions in heap now, see region " SIZE_FORMAT, r->region_number()));
 478 
 479     // Record current region occupancy: this communicates empty regions are free
 480     // to the rest of Full GC code.
 481     r->set_new_top(r->top());
 482   }
 483 };
 484 
 485 class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure {
 486 private:
 487   ShenandoahHeap* const _heap;
 488   ShenandoahMarkingContext* const _ctx;
 489 
 490 public:
 491   ShenandoahTrashImmediateGarbageClosure() :
 492     _heap(ShenandoahHeap::heap()),
 493     _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
 494 
 495   void heap_region_do(ShenandoahHeapRegion* r) {
 496     if (r->is_humongous_start()) {
 497       oop humongous_obj = oop(r->bottom());
 498       if (!_ctx->is_marked(humongous_obj)) {
 499         assert(!r->has_live(),
 500                err_msg("Region " SIZE_FORMAT " is not marked, should not have live", r->region_number()));
 501         _heap->trash_humongous_region_at(r);
 502       } else {
 503         assert(r->has_live(),
 504                err_msg("Region " SIZE_FORMAT " should have live", r->region_number()));
 505       }
 506     } else if (r->is_humongous_continuation()) {
 507       // If we hit continuation, the non-live humongous starts should have been trashed already
 508       assert(r->humongous_start_region()->has_live(),
 509              err_msg("Region " SIZE_FORMAT " should have live", r->region_number()));
 510     } else if (r->is_regular()) {
 511       if (!r->has_live()) {
 512         r->make_trash_immediate();
 513       }
 514     }
 515   }
 516 };
 517 
 518 void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
 519   ShenandoahHeap* heap = ShenandoahHeap::heap();
 520   GCTraceTime time("Phase 2: Compute new object addresses", ShenandoahLogDebug, _gc_timer, heap->tracer()->gc_id());
 521   ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
 522 
 523   // About to figure out which regions can be compacted, make sure pinning status
 524   // had been updated in GC prologue.
 525   heap->assert_pinned_region_status();
 526 
 527   {
 528     // Trash the immediately collectible regions before computing addresses
 529     ShenandoahTrashImmediateGarbageClosure tigcl;
 530     heap->heap_region_iterate(&tigcl);
 531 
 532     // Make sure regions are in good state: committed, active, clean.
 533     // This is needed because we are potentially sliding the data through them.
 534     ShenandoahEnsureHeapActiveClosure ecl;
 535     heap->heap_region_iterate(&ecl);
 536   }
 537 
 538   // Compute the new addresses for regular objects
 539   {
 540     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
 541     ShenandoahPrepareForCompactionTask prepare_task(_preserved_marks, worker_slices);
 542     heap->workers()->run_task(&prepare_task);
 543   }
 544 
 545   // Compute the new addresses for humongous objects
 546   {
 547     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_humong);
 548     calculate_target_humongous_objects();
 549   }
 550 }
 551 
 552 class ShenandoahAdjustPointersClosure : public MetadataAwareOopClosure {
 553 private:
 554   ShenandoahHeap* const _heap;
 555   ShenandoahMarkingContext* const _ctx;
 556 
 557   template <class T>
 558   inline void do_oop_work(T* p) {
 559     T o = oopDesc::load_heap_oop(p);
 560     if (! oopDesc::is_null(o)) {
 561       oop obj = oopDesc::decode_heap_oop_not_null(o);
 562       assert(_ctx->is_marked(obj), "must be marked");
 563       if (obj->is_forwarded()) {
 564         oop forw = obj->forwardee();
 565         oopDesc::encode_store_heap_oop(p, forw);
 566       }
 567    }
 568   }
 569 
 570 public:
 571   ShenandoahAdjustPointersClosure() :
 572     _heap(ShenandoahHeap::heap()),
 573     _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
 574 
 575   void do_oop(oop* p)       { do_oop_work(p); }
 576   void do_oop(narrowOop* p) { do_oop_work(p); }
 577 };
 578 
 579 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
 580 private:
 581   ShenandoahHeap* const _heap;
 582   ShenandoahAdjustPointersClosure _cl;
 583 
 584 public:
 585   ShenandoahAdjustPointersObjectClosure() :
 586     _heap(ShenandoahHeap::heap()) {
 587   }
 588   void do_object(oop p) {
 589     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
 590     p->oop_iterate(&_cl);
 591   }
 592 };
 593 
 594 class ShenandoahAdjustPointersTask : public AbstractGangTask {
 595 private:
 596   ShenandoahHeap*          const _heap;
 597   ShenandoahRegionIterator       _regions;
 598 
 599 public:
 600   ShenandoahAdjustPointersTask() :
 601     AbstractGangTask("Shenandoah Adjust Pointers Task"),
 602     _heap(ShenandoahHeap::heap()) {
 603   }
 604 
 605   void work(uint worker_id) {
 606     ShenandoahAdjustPointersObjectClosure obj_cl;
 607     ShenandoahHeapRegion* r = _regions.next();
 608     while (r != NULL) {
 609       if (!r->is_humongous_continuation() && r->has_live()) {
 610         _heap->marked_object_iterate(r, &obj_cl);
 611       }
 612       r = _regions.next();
 613     }
 614   }
 615 };
 616 
 617 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
 618 private:
 619   ShenandoahRootProcessor* _rp;
 620   PreservedMarksSet* _preserved_marks;
 621 public:
 622   ShenandoahAdjustRootPointersTask(ShenandoahRootProcessor* rp, PreservedMarksSet* preserved_marks) :
 623     AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
 624     _rp(rp),
 625     _preserved_marks(preserved_marks) {}
 626 
 627   void work(uint worker_id) {
 628     ShenandoahAdjustPointersClosure cl;
 629     CLDToOopClosure adjust_cld_closure(&cl, true);
 630     MarkingCodeBlobClosure adjust_code_closure(&cl,
 631                                              CodeBlobToOopClosure::FixRelocations);
 632 
 633     _rp->process_all_roots(&cl,
 634                            &adjust_cld_closure,
 635                            &adjust_code_closure, NULL, worker_id);
 636     _preserved_marks->get(worker_id)->adjust_during_full_gc();
 637   }
 638 };
 639 
 640 void ShenandoahMarkCompact::phase3_update_references() {
 641   ShenandoahHeap* heap = ShenandoahHeap::heap();
 642   GCTraceTime time("Phase 3: Adjust pointers", ShenandoahLogDebug, _gc_timer, heap->tracer()->gc_id());
 643   ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
 644 
 645   WorkGang* workers = heap->workers();
 646   uint nworkers = workers->active_workers();
 647   {
 648     COMPILER2_PRESENT(DerivedPointerTable::clear());
 649     ShenandoahRootProcessor rp(heap, nworkers, ShenandoahPhaseTimings::full_gc_roots);
 650     ShenandoahAdjustRootPointersTask task(&rp, _preserved_marks);
 651     workers->run_task(&task);
 652     COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
 653   }
 654 
 655   ShenandoahAdjustPointersTask adjust_pointers_task;
 656   workers->run_task(&adjust_pointers_task);
 657 }
 658 
 659 class ShenandoahCompactObjectsClosure : public ObjectClosure {
 660 private:
 661   ShenandoahHeap* const _heap;
 662 
 663 public:
 664   ShenandoahCompactObjectsClosure() : _heap(ShenandoahHeap::heap()) {}
 665 
 666   void do_object(oop p) {
 667     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
 668     size_t size = (size_t)p->size();
 669     if (p->is_forwarded()) {
 670       HeapWord* compact_from = (HeapWord*) p;
 671       HeapWord* compact_to = (HeapWord*) p->forwardee();
 672       Copy::aligned_conjoint_words(compact_from, compact_to, size);
 673       oop new_obj = oop(compact_to);
 674       new_obj->init_mark();
 675     }
 676   }
 677 };
 678 
 679 class ShenandoahCompactObjectsTask : public AbstractGangTask {
 680 private:
 681   ShenandoahHeap* const _heap;
 682   ShenandoahHeapRegionSet** const _worker_slices;
 683 
 684 public:
 685   ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
 686     AbstractGangTask("Shenandoah Compact Objects Task"),
 687     _heap(ShenandoahHeap::heap()),
 688     _worker_slices(worker_slices) {
 689   }
 690 
 691   void work(uint worker_id) {
 692     ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
 693 
 694     ShenandoahCompactObjectsClosure cl;
 695     ShenandoahHeapRegion* r = slice.next();
 696     while (r != NULL) {
 697       assert(!r->is_humongous(), "must not get humongous regions here");
 698       if (r->has_live()) {
 699         _heap->marked_object_iterate(r, &cl);
 700       }
 701       r->set_top(r->new_top());
 702       r = slice.next();
 703     }
 704   }
 705 };
 706 
 707 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
 708 private:
 709   ShenandoahHeap* const _heap;
 710   size_t _live;
 711 
 712 public:
 713   ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0) {
 714     _heap->free_set()->clear();
 715   }
 716 
 717   void heap_region_do(ShenandoahHeapRegion* r) {
 718     assert (!r->is_cset(), "cset regions should have been demoted already");
 719 
 720     // Need to reset the complete-top-at-mark-start pointer here because
 721     // the complete marking bitmap is no longer valid. This ensures
 722     // size-based iteration in marked_object_iterate().
 723     // NOTE: See blurb at ShenandoahMCResetCompleteBitmapTask on why we need to skip
 724     // pinned regions.
 725     if (!r->is_pinned()) {
 726       _heap->complete_marking_context()->reset_top_at_mark_start(r);
 727     }
 728 
 729     size_t live = r->used();
 730 
 731     // Make empty regions that have been allocated into regular
 732     if (r->is_empty() && live > 0) {
 733       r->make_regular_bypass();
 734     }
 735 
 736     // Reclaim regular regions that became empty
 737     if (r->is_regular() && live == 0) {
 738       r->make_trash();
 739     }
 740 
 741     // Recycle all trash regions
 742     if (r->is_trash()) {
 743       live = 0;
 744       r->recycle();
 745     }
 746 
 747     r->set_live_data(live);
 748     r->reset_alloc_metadata_to_shared();
 749     _live += live;
 750   }
 751 
 752   size_t get_live() {
 753     return _live;
 754   }
 755 };
 756 
 757 void ShenandoahMarkCompact::compact_humongous_objects() {
 758   // Compact humongous regions, based on their fwdptr objects.
 759   //
 760   // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
 761   // humongous regions are already compacted, and do not require further moves, which alleviates
 762   // sliding costs. We may consider doing this in parallel in future.
 763 
 764   ShenandoahHeap* heap = ShenandoahHeap::heap();
 765 
 766   for (size_t c = heap->num_regions(); c > 0; c--) {
 767     ShenandoahHeapRegion* r = heap->get_region(c - 1);
 768     if (r->is_humongous_start()) {
 769       oop old_obj = oop(r->bottom());
 770       if (!old_obj->is_forwarded()) {
 771         // No need to move the object, it stays at the same slot
 772         continue;
 773       }
 774       size_t words_size = old_obj->size();
 775       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 776 
 777       size_t old_start = r->region_number();
 778       size_t old_end   = old_start + num_regions - 1;
 779       size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
 780       size_t new_end   = new_start + num_regions - 1;
 781       assert(old_start != new_start, "must be real move");
 782       assert(r->is_stw_move_allowed(), err_msg("Region " SIZE_FORMAT " should be movable", r->region_number()));
 783 
 784       Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
 785                                    heap->get_region(new_start)->bottom(),
 786                                    ShenandoahHeapRegion::region_size_words()*num_regions);
 787 
 788       oop new_obj = oop(heap->get_region(new_start)->bottom());
 789       new_obj->init_mark();
 790 
 791       {
 792         for (size_t c = old_start; c <= old_end; c++) {
 793           ShenandoahHeapRegion* r = heap->get_region(c);
 794           r->make_regular_bypass();
 795           r->set_top(r->bottom());
 796         }
 797 
 798         for (size_t c = new_start; c <= new_end; c++) {
 799           ShenandoahHeapRegion* r = heap->get_region(c);
 800           if (c == new_start) {
 801             r->make_humongous_start_bypass();
 802           } else {
 803             r->make_humongous_cont_bypass();
 804           }
 805 
 806           // Trailing region may be non-full, record the remainder there
 807           size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask();
 808           if ((c == new_end) && (remainder != 0)) {
 809             r->set_top(r->bottom() + remainder);
 810           } else {
 811             r->set_top(r->end());
 812           }
 813 
 814           r->reset_alloc_metadata_to_shared();
 815         }
 816       }
 817     }
 818   }
 819 }
 820 
 821 // This is slightly different to ShHeap::reset_next_mark_bitmap:
 822 // we need to remain able to walk pinned regions.
 823 // Since pinned region do not move and don't get compacted, we will get holes with
 824 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
 825 // cannot be iterated over using oop->size(). The only way to safely iterate over those is using
 826 // a valid marking bitmap and valid TAMS pointer. This class only resets marking
 827 // bitmaps for un-pinned regions, and later we only reset TAMS for unpinned regions.
 828 class ShenandoahMCResetCompleteBitmapTask : public AbstractGangTask {
 829 private:
 830   ShenandoahRegionIterator _regions;
 831 
 832 public:
 833   ShenandoahMCResetCompleteBitmapTask() :
 834     AbstractGangTask("Parallel Reset Bitmap Task") {
 835   }
 836 
 837   void work(uint worker_id) {
 838     ShenandoahHeapRegion* region = _regions.next();
 839     ShenandoahHeap* heap = ShenandoahHeap::heap();
 840     ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
 841     while (region != NULL) {
 842       if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
 843         ctx->clear_bitmap(region);
 844       }
 845       region = _regions.next();
 846     }
 847   }
 848 };
 849 
 850 void ShenandoahMarkCompact::phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices) {
 851   ShenandoahHeap* heap = ShenandoahHeap::heap();
 852   GCTraceTime time("Phase 4: Move objects", ShenandoahLogDebug, _gc_timer, heap->tracer()->gc_id());
 853   ShenandoahGCPhase compaction_phase(ShenandoahPhaseTimings::full_gc_copy_objects);
 854 
 855   // Compact regular objects first
 856   {
 857     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_regular);
 858     ShenandoahCompactObjectsTask compact_task(worker_slices);
 859     heap->workers()->run_task(&compact_task);
 860   }
 861 
 862   // Compact humongous objects after regular object moves
 863   {
 864     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
 865     compact_humongous_objects();
 866   }
 867 
 868   // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
 869   // and must ensure the bitmap is in sync.
 870   {
 871     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_reset_complete);
 872     ShenandoahMCResetCompleteBitmapTask task;
 873     heap->workers()->run_task(&task);
 874   }
 875 
 876   // Bring regions in proper states after the collection, and set heap properties.
 877   {
 878     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
 879 
 880     ShenandoahPostCompactClosure post_compact;
 881     heap->heap_region_iterate(&post_compact);
 882     heap->set_used(post_compact.get_live());
 883 
 884     heap->collection_set()->clear();
 885     heap->free_set()->rebuild();
 886   }
 887 
 888   heap->clear_cancelled_gc();
 889 }