199 }
200
201 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
202 // the space. This would be the last action if there is nothing to evacuate. Note that
203 // we will not age young-gen objects in the case that we skip evacuation.
204 entry_cleanup_early();
205
206 heap->free_set()->log_status_under_lock();
207
208 // Processing strong roots
209 // This may be skipped if there is nothing to update/evacuate.
210 // If so, strong_root_in_progress would be unset.
211 if (heap->is_concurrent_strong_root_in_progress()) {
212 entry_strong_roots();
213 }
214
215 // Continue the cycle with evacuation and optional update-refs.
216 // This may be skipped if there is nothing to evacuate.
217 // If so, evac_in_progress would be unset by collection set preparation code.
218 if (heap->is_evacuation_in_progress()) {
219 // Concurrently evacuate
220 entry_evacuate();
221 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
222 return false;
223 }
224
225 // Perform update-refs phase.
226 entry_concurrent_update_refs_prepare(heap);
227
228 if (ShenandoahHeap::heap()->mode()->is_generational()) {
229 entry_update_card_table();
230 }
231
232 if (ShenandoahVerify) {
233 vmop_entry_init_update_refs();
234 }
235
236 entry_update_refs();
237 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
238 return false;
239 }
240
241 // Concurrent update thread roots
242 entry_update_thread_roots();
243 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
244 return false;
245 }
246
247 vmop_entry_final_update_refs();
248
249 // Update references freed up collection set, kick the cleanup to reclaim the space.
250 entry_cleanup_complete();
251 } else {
252 _abbreviated = true;
253 if (!entry_final_roots()) {
254 assert(_degen_point != _degenerated_unset, "Need to know where to start degenerated cycle");
255 return false;
256 }
257
258 // In normal cycle, final-update-refs would verify at the end of the cycle.
259 // In abbreviated cycle, we need to verify separately.
260 if (ShenandoahVerify) {
261 vmop_entry_final_verify();
262 }
263 }
264
265 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
266 // abbreviated cycle.
267 if (heap->mode()->is_generational()) {
268 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
269 }
270
271 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
272 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
273 // reducing the likelihood that GC will degenerate.
274 entry_reset_after_collect();
275
276 return true;
277 }
278
279 bool ShenandoahConcurrentGC::complete_abbreviated_cycle() {
280 shenandoah_assert_generational();
281
282 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
283
284 // We chose not to evacuate because we found sufficient immediate garbage.
285 // However, there may still be regions to promote in place, so do that now.
286 if (heap->old_generation()->has_in_place_promotions()) {
287 entry_promote_in_place();
288
289 // If the promote-in-place operation was cancelled, we can have the degenerated
290 // cycle complete the operation. It will see that no evacuations are in progress,
291 // and that there are regions wanting promotion. The risk with not handling the
292 // cancellation would be failing to restore top for these regions and leaving
293 // them unable to serve allocations for the old generation.This will leave the weak
294 // roots flag set (the degenerated cycle will unset it).
295 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
296 return false;
297 }
298 }
299
300 // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
301 // the control thread will detect it on its next iteration and run a degenerated young cycle.
302 if (!_generation->is_old()) {
303 heap->update_region_ages(_generation->complete_marking_context());
304 }
305
306 return true;
307 }
308
309 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
310 ShenandoahHeap* const heap = ShenandoahHeap::heap();
311 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
312 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
313
314 heap->try_inject_alloc_failure();
315 VM_ShenandoahInitMark op(this);
316 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
317 }
318
319 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
320 ShenandoahHeap* const heap = ShenandoahHeap::heap();
321 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
322 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
323
324 heap->try_inject_alloc_failure();
325 VM_ShenandoahFinalMarkStartEvac op(this);
326 VMThread::execute(&op); // jump to entry_final_mark under safepoint
329 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
330 ShenandoahHeap* const heap = ShenandoahHeap::heap();
331 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
332 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
333
334 heap->try_inject_alloc_failure();
335 VM_ShenandoahInitUpdateRefs op(this);
336 VMThread::execute(&op);
337 }
338
339 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
340 ShenandoahHeap* const heap = ShenandoahHeap::heap();
341 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
342 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
343
344 heap->try_inject_alloc_failure();
345 VM_ShenandoahFinalUpdateRefs op(this);
346 VMThread::execute(&op);
347 }
348
349 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
350 ShenandoahHeap* const heap = ShenandoahHeap::heap();
351 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
352 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
353
354 // This phase does not use workers, no need for setup
355 heap->try_inject_alloc_failure();
356 VM_ShenandoahFinalVerify op(this);
357 VMThread::execute(&op);
358 }
359
360 void ShenandoahConcurrentGC::entry_init_mark() {
361 const char* msg = init_mark_event_message();
362 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
363 EventMark em("%s", msg);
364
365 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
366 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
367 "init marking");
368
369 op_init_mark();
370 }
371
372 void ShenandoahConcurrentGC::entry_final_mark() {
373 const char* msg = final_mark_event_message();
374 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
375 EventMark em("%s", msg);
376
385 static const char* msg = "Pause Init Update Refs";
386 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
387 EventMark em("%s", msg);
388
389 // No workers used in this phase, no setup required
390 op_init_update_refs();
391 }
392
393 void ShenandoahConcurrentGC::entry_final_update_refs() {
394 static const char* msg = "Pause Final Update Refs";
395 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
396 EventMark em("%s", msg);
397
398 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
399 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
400 "final reference update");
401
402 op_final_update_refs();
403 }
404
405 void ShenandoahConcurrentGC::entry_final_verify() {
406 const char* msg = verify_final_event_message();
407 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_verify);
408 EventMark em("%s", msg);
409
410 op_verify_final();
411 }
412
413 void ShenandoahConcurrentGC::entry_reset() {
414 ShenandoahHeap* const heap = ShenandoahHeap::heap();
415 heap->try_inject_alloc_failure();
416
417 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
418 {
419 const char* msg = conc_reset_event_message();
420 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
421 EventMark em("%s", msg);
422
423 ShenandoahWorkerScope scope(heap->workers(),
424 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
425 msg);
426 op_reset();
427 }
428 }
429
430 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
431 if (_generation->is_young()) {
432 ShenandoahHeap* const heap = ShenandoahHeap::heap();
566 heap->rebuild_free_set(true /*concurrent*/);
567 }
568 }
569
570 void ShenandoahConcurrentGC::entry_evacuate() {
571 ShenandoahHeap* const heap = ShenandoahHeap::heap();
572 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
573
574 static const char* msg = "Concurrent evacuation";
575 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
576 EventMark em("%s", msg);
577
578 ShenandoahWorkerScope scope(heap->workers(),
579 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
580 "concurrent evacuation");
581
582 heap->try_inject_alloc_failure();
583 op_evacuate();
584 }
585
586 void ShenandoahConcurrentGC::entry_promote_in_place() const {
587 shenandoah_assert_generational();
588
589 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::promote_in_place);
590 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::promote_in_place);
591 EventMark em("%s", "Promote in place");
592
593 ShenandoahGenerationalHeap::heap()->promote_regions_in_place(_generation, true);
594 }
595
596 void ShenandoahConcurrentGC::entry_update_thread_roots() {
597 ShenandoahHeap* const heap = ShenandoahHeap::heap();
598 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
599
600 static const char* msg = "Concurrent update thread roots";
601 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
602 EventMark em("%s", msg);
603
604 // No workers used in this phase, no setup required
605 heap->try_inject_alloc_failure();
606 op_update_thread_roots();
607 }
608
609 void ShenandoahConcurrentGC::entry_update_refs() {
610 ShenandoahHeap* const heap = ShenandoahHeap::heap();
611 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
612 static const char* msg = "Concurrent update references";
613 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
614 EventMark em("%s", msg);
615
680 "Region %zu should already have correct TAMS", r->index());
681 }
682 }
683
684 bool is_thread_safe() { return true; }
685 };
686
687 void ShenandoahConcurrentGC::start_mark() {
688 _mark.start_mark();
689 }
690
691 void ShenandoahConcurrentGC::op_init_mark() {
692 ShenandoahHeap* const heap = ShenandoahHeap::heap();
693 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
694 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
695
696 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
697 assert(!_generation->is_mark_complete(), "should not be complete");
698 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
699
700 if (heap->mode()->is_generational()) {
701 if (_generation->is_global()) {
702 heap->old_generation()->cancel_gc();
703 }
704
705 {
706 // After we swap card table below, the write-table is all clean, and the read table holds
707 // cards dirty prior to the start of GC. Young and bootstrap collection will update
708 // the write card table as a side effect of remembered set scanning. Global collection will
709 // update the card table as a side effect of global marking of old objects.
710 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
711 _generation->swap_card_tables();
712 }
713 }
714
715 if (ShenandoahVerify) {
716 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
717 heap->verifier()->verify_before_concmark(_generation);
718 }
719
787
788 // Has to be done after cset selection
789 heap->prepare_concurrent_roots();
790
791 if (!heap->collection_set()->is_empty()) {
792 LogTarget(Debug, gc, cset) lt;
793 if (lt.is_enabled()) {
794 ResourceMark rm;
795 LogStream ls(lt);
796 heap->collection_set()->print_on(&ls);
797 }
798
799 if (ShenandoahVerify) {
800 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
801 heap->verifier()->verify_before_evacuation(_generation);
802 }
803
804 heap->set_evacuation_in_progress(true);
805 // From here on, we need to update references.
806 heap->set_has_forwarded_objects(true);
807
808 // Arm nmethods/stack for concurrent processing
809 ShenandoahCodeRoots::arm_nmethods();
810 ShenandoahStackWatermark::change_epoch_id();
811
812 } else {
813 if (ShenandoahVerify) {
814 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
815 if (has_in_place_promotions(heap)) {
816 heap->verifier()->verify_after_concmark_with_promotions(_generation);
817 } else {
818 heap->verifier()->verify_after_concmark(_generation);
819 }
820 }
821 }
822 }
823
824 {
825 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
826 heap->propagate_gc_state_to_all_threads();
827 }
828 }
829
830 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
831 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
832 }
833
834 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
835 private:
836 OopClosure* const _oops;
837 public:
838 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
839
840 void do_thread(Thread* thread) override {
841 JavaThread* const jt = JavaThread::cast(thread);
842 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
843 }
1024
1025 void ShenandoahConcurrentGC::op_class_unloading() {
1026 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1027 assert (heap->is_concurrent_weak_root_in_progress() &&
1028 heap->unload_classes(),
1029 "Checked by caller");
1030 heap->do_class_unloading();
1031 }
1032
1033 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1034 private:
1035 ShenandoahEvacuateUpdateMetadataClosure _cl;
1036
1037 public:
1038 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1039
1040 void do_nmethod(nmethod* n) {
1041 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1042 ShenandoahNMethodLocker locker(data->lock());
1043 data->oops_do(&_cl, /* fix_relocations = */ true);
1044 ShenandoahNMethod::disarm_nmethod(n);
1045 }
1046 };
1047
1048 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1049 private:
1050 ShenandoahPhaseTimings::Phase _phase;
1051 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1052 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1053 _cld_roots;
1054 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1055
1056 public:
1057 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1058 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1059 _phase(phase),
1060 _vm_roots(phase),
1061 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1062 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1063
1064 void work(uint worker_id) {
1205
1206 // If we are running in generational mode, this will also age active regions that
1207 // haven't been used for allocation.
1208 heap->update_heap_region_states(true /*concurrent*/);
1209
1210 heap->set_update_refs_in_progress(false);
1211 heap->set_has_forwarded_objects(false);
1212
1213 if (ShenandoahVerify) {
1214 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1215 heap->verifier()->verify_after_update_refs(_generation);
1216 }
1217
1218 if (VerifyAfterGC) {
1219 Universe::verify();
1220 }
1221
1222 heap->rebuild_free_set(true /*concurrent*/);
1223 _generation->heuristics()->start_idle_span();
1224
1225 {
1226 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1227 heap->propagate_gc_state_to_all_threads();
1228 }
1229 }
1230
1231 bool ShenandoahConcurrentGC::entry_final_roots() {
1232 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1233 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
1234
1235
1236 const char* msg = conc_final_roots_event_message();
1237 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_final_roots);
1238 EventMark em("%s", msg);
1239 ShenandoahWorkerScope scope(heap->workers(),
1240 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
1241 msg);
1242
1243 if (heap->mode()->is_generational()) {
1244 if (!complete_abbreviated_cycle()) {
1245 return false;
1246 }
1247 }
1248
1249 heap->concurrent_final_roots();
1250 return true;
1251 }
1252
1253 void ShenandoahConcurrentGC::op_verify_final() {
1254 assert(ShenandoahVerify, "Should have been checked before");
1255 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1256 heap->verifier()->verify_after_gc(_generation);
1257 }
1258
1259 void ShenandoahConcurrentGC::op_cleanup_complete() {
1260 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1261 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1262 "cleanup complete.");
1263 ShenandoahHeap::heap()->recycle_trash();
1264 }
1265
1266 void ShenandoahConcurrentGC::op_reset_after_collect() {
1267 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1268 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1269 "reset after collection.");
1270
1271 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1272 if (heap->mode()->is_generational()) {
1273 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1274 // the young generation intact. In particular, reference processing in the old generation may potentially
1275 // need the reachability of a young generation referent of a Reference object in the old generation.
1276 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1277 heap->young_generation()->reset_mark_bitmap<false>();
1278 }
1279 } else {
1280 _generation->reset_mark_bitmap<false>();
1281 }
1282 }
1283
1284 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1285 if (ShenandoahHeap::heap()->cancelled_gc()) {
1286 _degen_point = point;
1287 return true;
1288 }
1289 return false;
1290 }
1321 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
1322 }
1323 }
1324
1325 const char* ShenandoahConcurrentGC::conc_reset_event_message() const {
1326 if (ShenandoahHeap::heap()->unload_classes()) {
1327 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)");
1328 } else {
1329 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", "");
1330 }
1331 }
1332
1333 const char* ShenandoahConcurrentGC::conc_reset_after_collect_event_message() const {
1334 if (ShenandoahHeap::heap()->unload_classes()) {
1335 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", " (unload classes)");
1336 } else {
1337 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", "");
1338 }
1339 }
1340
1341 const char* ShenandoahConcurrentGC::verify_final_event_message() const {
1342 if (ShenandoahHeap::heap()->unload_classes()) {
1343 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", " (unload classes)");
1344 } else {
1345 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final", "");
1346 }
1347 }
1348
1349 const char* ShenandoahConcurrentGC::conc_final_roots_event_message() const {
1350 if (ShenandoahHeap::heap()->unload_classes()) {
1351 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", " (unload classes)");
1352 } else {
1353 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", "");
1354 }
1355 }
1356
1357 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
1358 if (ShenandoahHeap::heap()->unload_classes()) {
1359 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)");
1360 } else {
1361 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", "");
1362 }
1363 }
1364
1365 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const {
1366 if (ShenandoahHeap::heap()->unload_classes()) {
1367 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)");
1368 } else {
1369 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", "");
1370 }
1371 }
1372
1373 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
|
199 }
200
201 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
202 // the space. This would be the last action if there is nothing to evacuate. Note that
203 // we will not age young-gen objects in the case that we skip evacuation.
204 entry_cleanup_early();
205
206 heap->free_set()->log_status_under_lock();
207
208 // Processing strong roots
209 // This may be skipped if there is nothing to update/evacuate.
210 // If so, strong_root_in_progress would be unset.
211 if (heap->is_concurrent_strong_root_in_progress()) {
212 entry_strong_roots();
213 }
214
215 // Continue the cycle with evacuation and optional update-refs.
216 // This may be skipped if there is nothing to evacuate.
217 // If so, evac_in_progress would be unset by collection set preparation code.
218 if (heap->is_evacuation_in_progress()) {
219 // Roots processing is complete, put the weak roots/ref flags down.
220 vmop_entry_final_roots(false);
221
222 // Concurrently evacuate
223 entry_evacuate();
224 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
225 return false;
226 }
227
228 // Perform update-refs phase.
229 entry_concurrent_update_refs_prepare(heap);
230
231 if (ShenandoahHeap::heap()->mode()->is_generational()) {
232 entry_update_card_table();
233 }
234
235 if (ShenandoahVerify) {
236 vmop_entry_init_update_refs();
237 }
238
239 entry_update_refs();
240 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
241 return false;
242 }
243
244 // Concurrent update thread roots
245 entry_update_thread_roots();
246 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_update_refs)) {
247 return false;
248 }
249
250 vmop_entry_final_update_refs();
251
252 // Update references freed up collection set, kick the cleanup to reclaim the space.
253 entry_cleanup_complete();
254 } else {
255 _abbreviated = true;
256
257 if (heap->mode()->is_generational()) {
258 entry_complete_abbreviated_cycle();
259
260 // If the promote-in-place operation was cancelled, we can have the degenerated
261 // cycle complete the operation. It will see that no evacuations are in progress,
262 // and that there are regions wanting promotion. The risk with not handling the
263 // cancellation would be failing to restore top for these regions and leaving
264 // them unable to serve allocations for the old generation.
265 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
266 return false;
267 }
268 }
269
270 // In normal cycle, final-update-refs would verify at the end of the cycle.
271 // In abbreviated cycle, we need to verify separately.
272 // This is now also puts the barriers down at the end of the cycle. TODO: Refine.
273 vmop_entry_final_roots(true);
274 }
275
276 // We defer generation resizing actions until after cset regions have been recycled. We do this even following an
277 // abbreviated cycle.
278 if (heap->mode()->is_generational()) {
279 ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
280 }
281
282 // Instead of always resetting immediately before the start of a new GC, we can often reset at the end of the
283 // previous GC. This allows us to start the next GC cycle more quickly after a trigger condition is detected,
284 // reducing the likelihood that GC will degenerate.
285 entry_reset_after_collect();
286
287 return true;
288 }
289
290 void ShenandoahConcurrentGC::entry_complete_abbreviated_cycle() {
291 shenandoah_assert_generational();
292
293 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
294
295 static const char* msg = "Concurrent complete abbreviated cycle";
296 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::complete_abbreviated);
297 EventMark em("%s", msg);
298 ShenandoahWorkerScope scope(heap->workers(),
299 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
300 "complete abbreviated");
301
302 // We chose not to evacuate because we found sufficient immediate garbage.
303 // However, there may still be regions to promote in place, so do that now.
304 if (heap->old_generation()->has_in_place_promotions()) {
305 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
306 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::complete_abbreviated_promote_in_place);
307 heap->promote_regions_in_place(_generation, true);
308 }
309
310 // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
311 // the control thread will detect it on its next iteration and run a degenerated young cycle.
312 if (!heap->cancelled_gc() && !_generation->is_old()) {
313 ShenandoahTimingsTracker tracker(ShenandoahPhaseTimings::complete_abbreviated_update_region_ages);
314 heap->update_region_ages(_generation->complete_marking_context());
315 }
316 }
317
318 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
319 ShenandoahHeap* const heap = ShenandoahHeap::heap();
320 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
321 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
322
323 heap->try_inject_alloc_failure();
324 VM_ShenandoahInitMark op(this);
325 VMThread::execute(&op); // jump to entry_init_mark() under safepoint
326 }
327
328 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
329 ShenandoahHeap* const heap = ShenandoahHeap::heap();
330 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
331 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
332
333 heap->try_inject_alloc_failure();
334 VM_ShenandoahFinalMarkStartEvac op(this);
335 VMThread::execute(&op); // jump to entry_final_mark under safepoint
338 void ShenandoahConcurrentGC::vmop_entry_init_update_refs() {
339 ShenandoahHeap* const heap = ShenandoahHeap::heap();
340 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
341 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
342
343 heap->try_inject_alloc_failure();
344 VM_ShenandoahInitUpdateRefs op(this);
345 VMThread::execute(&op);
346 }
347
348 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
349 ShenandoahHeap* const heap = ShenandoahHeap::heap();
350 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
351 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
352
353 heap->try_inject_alloc_failure();
354 VM_ShenandoahFinalUpdateRefs op(this);
355 VMThread::execute(&op);
356 }
357
358 void ShenandoahConcurrentGC::vmop_entry_final_roots(bool at_gc_end) {
359 ShenandoahHeap* const heap = ShenandoahHeap::heap();
360 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
361 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
362
363 // This phase does not use workers, no need for setup
364 heap->try_inject_alloc_failure();
365 VM_ShenandoahFinalRoots op(this, at_gc_end);
366 VMThread::execute(&op);
367 }
368
369 void ShenandoahConcurrentGC::entry_init_mark() {
370 const char* msg = init_mark_event_message();
371 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
372 EventMark em("%s", msg);
373
374 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
375 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
376 "init marking");
377
378 op_init_mark();
379 }
380
381 void ShenandoahConcurrentGC::entry_final_mark() {
382 const char* msg = final_mark_event_message();
383 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
384 EventMark em("%s", msg);
385
394 static const char* msg = "Pause Init Update Refs";
395 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
396 EventMark em("%s", msg);
397
398 // No workers used in this phase, no setup required
399 op_init_update_refs();
400 }
401
402 void ShenandoahConcurrentGC::entry_final_update_refs() {
403 static const char* msg = "Pause Final Update Refs";
404 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
405 EventMark em("%s", msg);
406
407 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
408 ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
409 "final reference update");
410
411 op_final_update_refs();
412 }
413
414 void ShenandoahConcurrentGC::entry_reset() {
415 ShenandoahHeap* const heap = ShenandoahHeap::heap();
416 heap->try_inject_alloc_failure();
417
418 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
419 {
420 const char* msg = conc_reset_event_message();
421 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
422 EventMark em("%s", msg);
423
424 ShenandoahWorkerScope scope(heap->workers(),
425 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
426 msg);
427 op_reset();
428 }
429 }
430
431 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
432 if (_generation->is_young()) {
433 ShenandoahHeap* const heap = ShenandoahHeap::heap();
567 heap->rebuild_free_set(true /*concurrent*/);
568 }
569 }
570
571 void ShenandoahConcurrentGC::entry_evacuate() {
572 ShenandoahHeap* const heap = ShenandoahHeap::heap();
573 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
574
575 static const char* msg = "Concurrent evacuation";
576 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
577 EventMark em("%s", msg);
578
579 ShenandoahWorkerScope scope(heap->workers(),
580 ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
581 "concurrent evacuation");
582
583 heap->try_inject_alloc_failure();
584 op_evacuate();
585 }
586
587 void ShenandoahConcurrentGC::entry_update_thread_roots() {
588 ShenandoahHeap* const heap = ShenandoahHeap::heap();
589 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
590
591 static const char* msg = "Concurrent update thread roots";
592 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
593 EventMark em("%s", msg);
594
595 // No workers used in this phase, no setup required
596 heap->try_inject_alloc_failure();
597 op_update_thread_roots();
598 }
599
600 void ShenandoahConcurrentGC::entry_update_refs() {
601 ShenandoahHeap* const heap = ShenandoahHeap::heap();
602 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
603 static const char* msg = "Concurrent update references";
604 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
605 EventMark em("%s", msg);
606
671 "Region %zu should already have correct TAMS", r->index());
672 }
673 }
674
675 bool is_thread_safe() { return true; }
676 };
677
678 void ShenandoahConcurrentGC::start_mark() {
679 _mark.start_mark();
680 }
681
682 void ShenandoahConcurrentGC::op_init_mark() {
683 ShenandoahHeap* const heap = ShenandoahHeap::heap();
684 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
685 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
686
687 assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
688 assert(!_generation->is_mark_complete(), "should not be complete");
689 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
690
691 // First pause in cycle, check that barriers were not left enabled.
692 ShenandoahCodeRoots::check_barriers();
693
694 if (heap->mode()->is_generational()) {
695 if (_generation->is_global()) {
696 heap->old_generation()->cancel_gc();
697 }
698
699 {
700 // After we swap card table below, the write-table is all clean, and the read table holds
701 // cards dirty prior to the start of GC. Young and bootstrap collection will update
702 // the write card table as a side effect of remembered set scanning. Global collection will
703 // update the card table as a side effect of global marking of old objects.
704 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
705 _generation->swap_card_tables();
706 }
707 }
708
709 if (ShenandoahVerify) {
710 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
711 heap->verifier()->verify_before_concmark(_generation);
712 }
713
781
782 // Has to be done after cset selection
783 heap->prepare_concurrent_roots();
784
785 if (!heap->collection_set()->is_empty()) {
786 LogTarget(Debug, gc, cset) lt;
787 if (lt.is_enabled()) {
788 ResourceMark rm;
789 LogStream ls(lt);
790 heap->collection_set()->print_on(&ls);
791 }
792
793 if (ShenandoahVerify) {
794 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
795 heap->verifier()->verify_before_evacuation(_generation);
796 }
797
798 heap->set_evacuation_in_progress(true);
799 // From here on, we need to update references.
800 heap->set_has_forwarded_objects(true);
801 } else {
802 if (ShenandoahVerify) {
803 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
804 if (has_in_place_promotions(heap)) {
805 heap->verifier()->verify_after_concmark_with_promotions(_generation);
806 } else {
807 heap->verifier()->verify_after_concmark(_generation);
808 }
809 }
810 }
811 }
812
813 // Arm nmethods/stack for concurrent processing
814 ShenandoahCodeRoots::arm_nmethods();
815 ShenandoahStackWatermark::change_epoch_id();
816
817 {
818 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
819 heap->propagate_gc_state_to_all_threads();
820 }
821 }
822
823 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
824 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
825 }
826
827 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
828 private:
829 OopClosure* const _oops;
830 public:
831 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
832
833 void do_thread(Thread* thread) override {
834 JavaThread* const jt = JavaThread::cast(thread);
835 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
836 }
1017
1018 void ShenandoahConcurrentGC::op_class_unloading() {
1019 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1020 assert (heap->is_concurrent_weak_root_in_progress() &&
1021 heap->unload_classes(),
1022 "Checked by caller");
1023 heap->do_class_unloading();
1024 }
1025
1026 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1027 private:
1028 ShenandoahEvacuateUpdateMetadataClosure _cl;
1029
1030 public:
1031 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1032
1033 void do_nmethod(nmethod* n) {
1034 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1035 ShenandoahNMethodLocker locker(data->lock());
1036 data->oops_do(&_cl, /* fix_relocations = */ true);
1037 ShenandoahNMethod::complete_and_disarm_nmethod_unlocked(n);
1038 }
1039 };
1040
1041 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1042 private:
1043 ShenandoahPhaseTimings::Phase _phase;
1044 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1045 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1046 _cld_roots;
1047 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1048
1049 public:
1050 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1051 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1052 _phase(phase),
1053 _vm_roots(phase),
1054 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1055 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1056
1057 void work(uint worker_id) {
1198
1199 // If we are running in generational mode, this will also age active regions that
1200 // haven't been used for allocation.
1201 heap->update_heap_region_states(true /*concurrent*/);
1202
1203 heap->set_update_refs_in_progress(false);
1204 heap->set_has_forwarded_objects(false);
1205
1206 if (ShenandoahVerify) {
1207 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1208 heap->verifier()->verify_after_update_refs(_generation);
1209 }
1210
1211 if (VerifyAfterGC) {
1212 Universe::verify();
1213 }
1214
1215 heap->rebuild_free_set(true /*concurrent*/);
1216 _generation->heuristics()->start_idle_span();
1217
1218 {
1219 // Final pause: update GC barriers to idle state.
1220 ShenandoahCodeRoots::arm_nmethods();
1221 ShenandoahStackWatermark::change_epoch_id();
1222 }
1223
1224 {
1225 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1226 heap->propagate_gc_state_to_all_threads();
1227 }
1228 }
1229
1230 void ShenandoahConcurrentGC::entry_final_roots(bool at_gc_end) {
1231 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1232 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
1233
1234 const char* msg = final_roots_event_message();
1235 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
1236 EventMark em("%s", msg);
1237 ShenandoahWorkerScope scope(heap->workers(),
1238 ParallelGCThreads,
1239 msg);
1240
1241 heap->op_final_roots(at_gc_end);
1242 }
1243
1244 void ShenandoahConcurrentGC::op_cleanup_complete() {
1245 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1246 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1247 "cleanup complete.");
1248 ShenandoahHeap::heap()->recycle_trash();
1249 }
1250
1251 void ShenandoahConcurrentGC::op_reset_after_collect() {
1252 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1253 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1254 "reset after collection.");
1255
1256 // Final concurrent phase: complete disabling all barriers.
1257 ShenandoahCodeRoots::disarm_nmethods();
1258
1259 // Check that barriers were not left enabled.
1260 ShenandoahCodeRoots::check_barriers();
1261
1262 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1263 if (heap->mode()->is_generational()) {
1264 // If we are in the midst of an old gc bootstrap or an old marking, we want to leave the mark bit map of
1265 // the young generation intact. In particular, reference processing in the old generation may potentially
1266 // need the reachability of a young generation referent of a Reference object in the old generation.
1267 if (!_do_old_gc_bootstrap && !heap->is_concurrent_old_mark_in_progress()) {
1268 heap->young_generation()->reset_mark_bitmap<false>();
1269 }
1270 } else {
1271 _generation->reset_mark_bitmap<false>();
1272 }
1273 }
1274
1275 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1276 if (ShenandoahHeap::heap()->cancelled_gc()) {
1277 _degen_point = point;
1278 return true;
1279 }
1280 return false;
1281 }
1312 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
1313 }
1314 }
1315
1316 const char* ShenandoahConcurrentGC::conc_reset_event_message() const {
1317 if (ShenandoahHeap::heap()->unload_classes()) {
1318 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)");
1319 } else {
1320 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", "");
1321 }
1322 }
1323
1324 const char* ShenandoahConcurrentGC::conc_reset_after_collect_event_message() const {
1325 if (ShenandoahHeap::heap()->unload_classes()) {
1326 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", " (unload classes)");
1327 } else {
1328 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset after collect", "");
1329 }
1330 }
1331
1332 const char* ShenandoahConcurrentGC::final_roots_event_message() const {
1333 if (ShenandoahHeap::heap()->unload_classes()) {
1334 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", " (unload classes)");
1335 } else {
1336 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", "");
1337 }
1338 }
1339
1340 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
1341 if (ShenandoahHeap::heap()->unload_classes()) {
1342 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)");
1343 } else {
1344 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", "");
1345 }
1346 }
1347
1348 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const {
1349 if (ShenandoahHeap::heap()->unload_classes()) {
1350 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)");
1351 } else {
1352 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", "");
1353 }
1354 }
1355
1356 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
|