233 vmop_entry_init_update_refs();
234 }
235
236 entry_update_refs();
237 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
238 return false;
239 }
240
241 // Concurrent update thread roots
242 entry_update_thread_roots();
243 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
244 return false;
245 }
246
247 vmop_entry_final_update_refs();
248
249 // Update references freed up collection set, kick the cleanup to reclaim the space.
250 entry_cleanup_complete();
251 } else {
252 _abbreviated = true;
253 if (!entry_final_roots()) {
254 assert(_degen_point != _degenerated_unset, "Need to know where to start degenerated cycle");
255 return false;
256 }
257
258 // In normal cycle, final-update-refs would verify at the end of the cycle.
259 // In abbreviated cycle, we need to verify separately.
260 if (ShenandoahVerify) {
261 vmop_entry_final_verify();
262 }
263 }
264
265 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
266 // abbreviated cycle.
267 if (heap->mode()->is_generational()) {
268 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
269 }
270
271 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
272 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
273 // reducing the likelihood that GC will degenerate.
274 entry_reset_after_collect();
275
276 return true;
277 }
278
279 bool ShenandoahConcurrentGC::complete_abbreviated_cycle() {
280 shenandoah_assert_generational();
281
282 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
329 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
330 ShenandoahHeap* const heap = ShenandoahHeap::heap();
331 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
332 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
333
334 heap->try_inject_alloc_failure();
335 VM_ShenandoahInitUpdateRefs op(this);
336 VMThread::execute(&op);
337 }
338
339 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
340 ShenandoahHeap* const heap = ShenandoahHeap::heap();
341 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
342 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
343
344 heap->try_inject_alloc_failure();
345 VM_ShenandoahFinalUpdateRefs op(this);
346 VMThread::execute(&op);
347 }
348
349 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
350 ShenandoahHeap* const heap = ShenandoahHeap::heap();
351 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
352 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
353
354 // This phase does not use workers, no need for setup
355 heap->try_inject_alloc_failure();
356 VM_ShenandoahFinalVerify op(this);
357 VMThread::execute(&op);
358 }
359
360 void ShenandoahConcurrentGC::entry_init_mark() {
361 const char* msg = init_mark_event_message();
362 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
363 EventMark em("%s", msg);
364
365 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
366 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
367 "init marking");
368
369 op_init_mark();
370 }
371
372 void ShenandoahConcurrentGC::entry_final_mark() {
373 const char* msg = final_mark_event_message();
374 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
375 EventMark em("%s", msg);
376
385 static const char* msg = "Pause Init Update Refs";
386 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
387 EventMark em("%s", msg);
388
389 // No workers used in this phase, no setup required
390 op_init_update_refs();
391 }
392
393 void ShenandoahConcurrentGC::entry_final_update_refs() {
394 static const char* msg = "Pause Final Update Refs";
395 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
396 EventMark em("%s", msg);
397
398 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
399 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
400 "final reference update");
401
402 op_final_update_refs();
403 }
404
405 void ShenandoahConcurrentGC::entry_final_verify() {
406 const char* msg = verify_final_event_message();
407 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_verify);
408 EventMark em("%s", msg);
409
410 op_verify_final();
411 }
412
413 void ShenandoahConcurrentGC::entry_reset() {
414 ShenandoahHeap* const heap = ShenandoahHeap::heap();
415 heap->try_inject_alloc_failure();
416
417 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
418 {
419 const char* msg = conc_reset_event_message();
420 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
421 EventMark em("%s", msg);
422
423 ShenandoahWorkerScope scope(heap->workers(),
424 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
425 msg);
426 op_reset();
427 }
428 }
429
430 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
431 if (_generation->is_young()) {
432 ShenandoahHeap* const heap = ShenandoahHeap::heap();
680 "Region %zu should already have correct TAMS", r->index());
681 }
682 }
683
684 bool is_thread_safe() { return true; }
685 };
686
687 void ShenandoahConcurrentGC::start_mark() {
688 _mark.start_mark();
689 }
690
691 void ShenandoahConcurrentGC::op_init_mark() {
692 ShenandoahHeap* const heap = ShenandoahHeap::heap();
693 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
694 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
695
696 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
697 assert(!_generation->is_mark_complete(), "should not be complete");
698 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
699
700 if (heap->mode()->is_generational()) {
701 if (_generation->is_global()) {
702 heap->old_generation()->cancel_gc();
703 }
704
705 {
706 // After we swap card table below, the write-table is all clean, and the read table holds
707 // cards dirty prior to the start of GC. Young and bootstrap collection will update
708 // the write card table as a side effect of remembered set scanning. Global collection will
709 // update the card table as a side effect of global marking of old objects.
710 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
711 _generation->swap_card_tables();
712 }
713 }
714
715 if (ShenandoahVerify) {
716 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
717 heap->verifier()->verify_before_concmark(_generation);
718 }
719
787
788 // Has to be done after cset selection
789 heap->prepare_concurrent_roots();
790
791 if (!heap->collection_set()->is_empty()) {
792 LogTarget(Debug, gc, cset) lt;
793 if (lt.is_enabled()) {
794 ResourceMark rm;
795 LogStream ls(lt);
796 heap->collection_set()->print_on(&ls);
797 }
798
799 if (ShenandoahVerify) {
800 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
801 heap->verifier()->verify_before_evacuation(_generation);
802 }
803
804 heap->set_evacuation_in_progress(true);
805 // From here on, we need to update references.
806 heap->set_has_forwarded_objects(true);
807
808 // Arm nmethods/stack for concurrent processing
809 ShenandoahCodeRoots::arm_nmethods();
810 ShenandoahStackWatermark::change_epoch_id();
811
812 } else {
813 if (ShenandoahVerify) {
814 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
815 if (has_in_place_promotions(heap)) {
816 heap->verifier()->verify_after_concmark_with_promotions(_generation);
817 } else {
818 heap->verifier()->verify_after_concmark(_generation);
819 }
820 }
821 }
822 }
823
824 {
825 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
826 heap->propagate_gc_state_to_all_threads();
827 }
828 }
829
830 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
831 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
832 }
833
834 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
835 private:
836 OopClosure* const _oops;
837 public:
838 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
839
840 void do_thread(Thread* thread) override {
841 JavaThread* const jt = JavaThread::cast(thread);
842 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
843 }
1024
1025 void ShenandoahConcurrentGC::op_class_unloading() {
1026 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1027 assert (heap->is_concurrent_weak_root_in_progress() &&
1028 heap->unload_classes(),
1029 "Checked by caller");
1030 heap->do_class_unloading();
1031 }
1032
1033 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1034 private:
1035 ShenandoahEvacuateUpdateMetadataClosure _cl;
1036
1037 public:
1038 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1039
1040 void do_nmethod(nmethod* n) {
1041 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1042 ShenandoahNMethodLocker locker(data->lock());
1043 data->oops_do(&_cl, /* fix_relocations = */ true);
1044 ShenandoahNMethod::disarm_nmethod(n);
1045 }
1046 };
1047
1048 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1049 private:
1050 ShenandoahPhaseTimings::Phase _phase;
1051 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1052 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1053 _cld_roots;
1054 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1055
1056 public:
1057 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1058 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1059 _phase(phase),
1060 _vm_roots(phase),
1061 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1062 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1063
1064 void work(uint worker_id) {
1211 heap->set_has_forwarded_objects(false);
1212
1213 if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) {
1214 // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for
1215 // entire regions. Both of these relevant operations occur before final update refs.
1216 ShenandoahGenerationalHeap::heap()->set_aging_cycle(false);
1217 }
1218
1219 if (ShenandoahVerify) {
1220 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1221 heap->verifier()->verify_after_update_refs(_generation);
1222 }
1223
1224 if (VerifyAfterGC) {
1225 Universe::verify();
1226 }
1227
1228 heap->rebuild_free_set(true /*concurrent*/);
1229 _generation->heuristics()->start_idle_span();
1230
1231 {
1232 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1233 heap->propagate_gc_state_to_all_threads();
1234 }
1235 }
1236
1237 bool ShenandoahConcurrentGC::entry_final_roots() {
1238 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1239 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
1240
1241
1242 const char* msg = conc_final_roots_event_message();
1243 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_final_roots);
1244 EventMark em("%s", msg);
1245 ShenandoahWorkerScope scope(heap->workers(),
1246 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
1247 msg);
1248
1249 if (heap->mode()->is_generational()) {
1250 if (!complete_abbreviated_cycle()) {
1251 return false;
1252 }
1253 }
1254
1255 heap->concurrent_final_roots();
1256 return true;
1257 }
1258
1259 void ShenandoahConcurrentGC::op_verify_final() {
1260 assert(ShenandoahVerify, "Should have been checked before");
1261 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1262 heap->verifier()->verify_after_gc(_generation);
1263 }
1264
1265 void ShenandoahConcurrentGC::op_cleanup_complete() {
1266 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1267 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1268 "cleanup complete.");
1269 ShenandoahHeap::heap()->recycle_trash();
1270 }
1271
1272 void ShenandoahConcurrentGC::op_reset_after_collect() {
1273 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1274 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1275 "reset after collection.");
1276
1277 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1278 if (heap->mode()->is_generational()) {
1279 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1280 // the young generation intact. In particular, reference processing in the old generation may potentially
1281 // need the reachability of a young generation referent of a Reference object in the old generation.
1282 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1283 heap->young_generation()->reset_mark_bitmap<false>();
1284 }
1285 } else {
1286 _generation->reset_mark_bitmap<false>();
1287 }
1288 }
1289
1290 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1291 if (ShenandoahHeap::heap()->cancelled_gc()) {
1292 _degen_point = point;
1293 return true;
1294 }
1295 return false;
1296 }
1327 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
1328 }
1329 }
1330
1331 const char* ShenandoahConcurrentGC::conc_reset_event_message() const {
1332 if (ShenandoahHeap::heap()->unload_classes()) {
1333 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)");
1334 } else {
1335 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", "");
1336 }
1337 }
1338
1339 const char* ShenandoahConcurrentGC::conc_reset_after_collect_event_message() const {
1340 if (ShenandoahHeap::heap()->unload_classes()) {
1341 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", " (unload classes)");
1342 } else {
1343 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", "");
1344 }
1345 }
1346
1347 const char* ShenandoahConcurrentGC::verify_final_event_message() const {
1348 if (ShenandoahHeap::heap()->unload_classes()) {
1349 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", " (unload classes)");
1350 } else {
1351 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", "");
1352 }
1353 }
1354
1355 const char* ShenandoahConcurrentGC::conc_final_roots_event_message() const {
1356 if (ShenandoahHeap::heap()->unload_classes()) {
1357 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", " (unload classes)");
1358 } else {
1359 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", "");
1360 }
1361 }
1362
1363 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
1364 if (ShenandoahHeap::heap()->unload_classes()) {
1365 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)");
1366 } else {
1367 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", "");
1368 }
1369 }
1370
1371 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const {
1372 if (ShenandoahHeap::heap()->unload_classes()) {
1373 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)");
1374 } else {
1375 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", "");
1376 }
1377 }
1378
1379 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
|
233 vmop_entry_init_update_refs();
234 }
235
236 entry_update_refs();
237 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
238 return false;
239 }
240
241 // Concurrent update thread roots
242 entry_update_thread_roots();
243 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
244 return false;
245 }
246
247 vmop_entry_final_update_refs();
248
249 // Update references freed up collection set, kick the cleanup to reclaim the space.
250 entry_cleanup_complete();
251 } else {
252 _abbreviated = true;
253
254 if (heap->mode()->is_generational()) {
255 if (!complete_abbreviated_cycle()) {
256 assert(_degen_point != _degenerated_unset, "Need to know where to start degenerated cycle");
257 return false;
258 }
259 }
260
261 // In normal cycle, final-update-refs would verify at the end of the cycle.
262 // In abbreviated cycle, we need to verify separately.
263 // This is now also puts the barriers down at the end of the cycle. TODO: Refine.
264 vmop_entry_final_roots();
265 }
266
267 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
268 // abbreviated cycle.
269 if (heap->mode()->is_generational()) {
270 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
271 }
272
273 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
274 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
275 // reducing the likelihood that GC will degenerate.
276 entry_reset_after_collect();
277
278 return true;
279 }
280
281 bool ShenandoahConcurrentGC::complete_abbreviated_cycle() {
282 shenandoah_assert_generational();
283
284 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
331 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
332 ShenandoahHeap* const heap = ShenandoahHeap::heap();
333 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
334 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
335
336 heap->try_inject_alloc_failure();
337 VM_ShenandoahInitUpdateRefs op(this);
338 VMThread::execute(&op);
339 }
340
341 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
342 ShenandoahHeap* const heap = ShenandoahHeap::heap();
343 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
344 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
345
346 heap->try_inject_alloc_failure();
347 VM_ShenandoahFinalUpdateRefs op(this);
348 VMThread::execute(&op);
349 }
350
351 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
352 ShenandoahHeap* const heap = ShenandoahHeap::heap();
353 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
354 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
355
356 // This phase does not use workers, no need for setup
357 heap->try_inject_alloc_failure();
358 VM_ShenandoahFinalRoots op(this);
359 VMThread::execute(&op);
360 }
361
362 void ShenandoahConcurrentGC::entry_init_mark() {
363 const char* msg = init_mark_event_message();
364 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
365 EventMark em("%s", msg);
366
367 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
368 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
369 "init marking");
370
371 op_init_mark();
372 }
373
374 void ShenandoahConcurrentGC::entry_final_mark() {
375 const char* msg = final_mark_event_message();
376 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
377 EventMark em("%s", msg);
378
387 static const char* msg = "Pause Init Update Refs";
388 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
389 EventMark em("%s", msg);
390
391 // No workers used in this phase, no setup required
392 op_init_update_refs();
393 }
394
395 void ShenandoahConcurrentGC::entry_final_update_refs() {
396 static const char* msg = "Pause Final Update Refs";
397 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
398 EventMark em("%s", msg);
399
400 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
401 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
402 "final reference update");
403
404 op_final_update_refs();
405 }
406
407 void ShenandoahConcurrentGC::entry_reset() {
408 ShenandoahHeap* const heap = ShenandoahHeap::heap();
409 heap->try_inject_alloc_failure();
410
411 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
412 {
413 const char* msg = conc_reset_event_message();
414 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
415 EventMark em("%s", msg);
416
417 ShenandoahWorkerScope scope(heap->workers(),
418 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
419 msg);
420 op_reset();
421 }
422 }
423
424 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
425 if (_generation->is_young()) {
426 ShenandoahHeap* const heap = ShenandoahHeap::heap();
674 "Region %zu should already have correct TAMS", r->index());
675 }
676 }
677
678 bool is_thread_safe() { return true; }
679 };
680
681 void ShenandoahConcurrentGC::start_mark() {
682 _mark.start_mark();
683 }
684
685 void ShenandoahConcurrentGC::op_init_mark() {
686 ShenandoahHeap* const heap = ShenandoahHeap::heap();
687 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
688 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
689
690 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
691 assert(!_generation->is_mark_complete(), "should not be complete");
692 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
693
694 // First pause in cycle, check that barriers were not left enabled.
695 ShenandoahCodeRoots::check_barriers();
696
697 if (heap->mode()->is_generational()) {
698 if (_generation->is_global()) {
699 heap->old_generation()->cancel_gc();
700 }
701
702 {
703 // After we swap card table below, the write-table is all clean, and the read table holds
704 // cards dirty prior to the start of GC. Young and bootstrap collection will update
705 // the write card table as a side effect of remembered set scanning. Global collection will
706 // update the card table as a side effect of global marking of old objects.
707 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
708 _generation->swap_card_tables();
709 }
710 }
711
712 if (ShenandoahVerify) {
713 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
714 heap->verifier()->verify_before_concmark(_generation);
715 }
716
784
785 // Has to be done after cset selection
786 heap->prepare_concurrent_roots();
787
788 if (!heap->collection_set()->is_empty()) {
789 LogTarget(Debug, gc, cset) lt;
790 if (lt.is_enabled()) {
791 ResourceMark rm;
792 LogStream ls(lt);
793 heap->collection_set()->print_on(&ls);
794 }
795
796 if (ShenandoahVerify) {
797 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
798 heap->verifier()->verify_before_evacuation(_generation);
799 }
800
801 heap->set_evacuation_in_progress(true);
802 // From here on, we need to update references.
803 heap->set_has_forwarded_objects(true);
804 } else {
805 if (ShenandoahVerify) {
806 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
807 if (has_in_place_promotions(heap)) {
808 heap->verifier()->verify_after_concmark_with_promotions(_generation);
809 } else {
810 heap->verifier()->verify_after_concmark(_generation);
811 }
812 }
813 }
814 }
815
816 // Arm nmethods/stack for concurrent processing
817 ShenandoahCodeRoots::arm_nmethods();
818 ShenandoahStackWatermark::change_epoch_id();
819
820 {
821 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
822 heap->propagate_gc_state_to_all_threads();
823 }
824 }
825
826 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
827 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
828 }
829
830 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
831 private:
832 OopClosure* const _oops;
833 public:
834 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
835
836 void do_thread(Thread* thread) override {
837 JavaThread* const jt = JavaThread::cast(thread);
838 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
839 }
1020
1021 void ShenandoahConcurrentGC::op_class_unloading() {
1022 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1023 assert (heap->is_concurrent_weak_root_in_progress() &&
1024 heap->unload_classes(),
1025 "Checked by caller");
1026 heap->do_class_unloading();
1027 }
1028
1029 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1030 private:
1031 ShenandoahEvacuateUpdateMetadataClosure _cl;
1032
1033 public:
1034 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1035
1036 void do_nmethod(nmethod* n) {
1037 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1038 ShenandoahNMethodLocker locker(data->lock());
1039 data->oops_do(&_cl, /* fix_relocations = */ true);
1040 ShenandoahNMethod::disarm_nmethod_unlocked(n);
1041 }
1042 };
1043
1044 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1045 private:
1046 ShenandoahPhaseTimings::Phase _phase;
1047 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1048 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1049 _cld_roots;
1050 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1051
1052 public:
1053 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1054 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1055 _phase(phase),
1056 _vm_roots(phase),
1057 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1058 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1059
1060 void work(uint worker_id) {
1207 heap->set_has_forwarded_objects(false);
1208
1209 if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) {
1210 // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for
1211 // entire regions. Both of these relevant operations occur before final update refs.
1212 ShenandoahGenerationalHeap::heap()->set_aging_cycle(false);
1213 }
1214
1215 if (ShenandoahVerify) {
1216 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1217 heap->verifier()->verify_after_update_refs(_generation);
1218 }
1219
1220 if (VerifyAfterGC) {
1221 Universe::verify();
1222 }
1223
1224 heap->rebuild_free_set(true /*concurrent*/);
1225 _generation->heuristics()->start_idle_span();
1226
1227 {
1228 // Final pause: update GC barriers to idle state.
1229 ShenandoahCodeRoots::arm_nmethods();
1230 ShenandoahStackWatermark::change_epoch_id();
1231 }
1232
1233 {
1234 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1235 heap->propagate_gc_state_to_all_threads();
1236 }
1237 }
1238
1239 void ShenandoahConcurrentGC::entry_final_roots() {
1240 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1241 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
1242
1243 const char* msg = final_roots_event_message();
1244 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
1245 EventMark em("%s", msg);
1246 ShenandoahWorkerScope scope(heap->workers(),
1247 ParallelGCThreads,
1248 msg);
1249
1250 heap->op_final_roots();
1251 }
1252
1253 void ShenandoahConcurrentGC::op_cleanup_complete() {
1254 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1255 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1256 "cleanup complete.");
1257 ShenandoahHeap::heap()->recycle_trash();
1258 }
1259
1260 void ShenandoahConcurrentGC::op_reset_after_collect() {
1261 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1262 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1263 "reset after collection.");
1264
1265 // Final concurrent phase: complete disabling all barriers.
1266 ShenandoahCodeRoots::disarm_nmethods();
1267
1268 // Check that barriers were not left enabled.
1269 ShenandoahCodeRoots::check_barriers();
1270
1271 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1272 if (heap->mode()->is_generational()) {
1273 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1274 // the young generation intact. In particular, reference processing in the old generation may potentially
1275 // need the reachability of a young generation referent of a Reference object in the old generation.
1276 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1277 heap->young_generation()->reset_mark_bitmap<false>();
1278 }
1279 } else {
1280 _generation->reset_mark_bitmap<false>();
1281 }
1282 }
1283
1284 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1285 if (ShenandoahHeap::heap()->cancelled_gc()) {
1286 _degen_point = point;
1287 return true;
1288 }
1289 return false;
1290 }
1321 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
1322 }
1323 }
1324
1325 const char* ShenandoahConcurrentGC::conc_reset_event_message() const {
1326 if (ShenandoahHeap::heap()->unload_classes()) {
1327 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)");
1328 } else {
1329 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", "");
1330 }
1331 }
1332
1333 const char* ShenandoahConcurrentGC::conc_reset_after_collect_event_message() const {
1334 if (ShenandoahHeap::heap()->unload_classes()) {
1335 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", " (unload classes)");
1336 } else {
1337 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", "");
1338 }
1339 }
1340
1341 const char* ShenandoahConcurrentGC::final_roots_event_message() const {
1342 if (ShenandoahHeap::heap()->unload_classes()) {
1343 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", " (unload classes)");
1344 } else {
1345 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", "");
1346 }
1347 }
1348
1349 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
1350 if (ShenandoahHeap::heap()->unload_classes()) {
1351 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)");
1352 } else {
1353 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", "");
1354 }
1355 }
1356
1357 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const {
1358 if (ShenandoahHeap::heap()->unload_classes()) {
1359 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)");
1360 } else {
1361 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", "");
1362 }
1363 }
1364
1365 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
|