197
198 // Perform concurrent class unloading before any regions get recycled. Class unloading may
199 // need to inspect unmarked objects in trashed regions.
200 if (heap->unload_classes()) {
201 entry_class_unloading();
202 }
203
204 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
205 // the space. This would be the last action if there is nothing to evacuate. Note that
206 // we will not age young-gen objects in the case that we skip evacuation.
207 entry_cleanup_early();
208
209 // Processing strong roots
210 // This may be skipped if there is nothing to update/evacuate.
211 // If so, strong_root_in_progress would be unset.
212 if (heap->is_concurrent_strong_root_in_progress()) {
213 entry_strong_roots();
214 }
215
216 // Roots processing is complete, put the weak roots flag down.
217 entry_final_roots();
218
219 // Continue the cycle with evacuation and optional update-refs.
220 // This may be skipped if there is nothing to evacuate.
221 // If so, evac_in_progress would be unset by collection set preparation code.
222 if (heap->is_evacuation_in_progress()) {
223 // Concurrently evacuate
224 entry_evacuate();
225 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
226 return false;
227 }
228
229 // Perform update-refs phase.
230 entry_concurrent_update_refs_prepare(heap);
231
232 if (ShenandoahHeap::heap()->mode()->is_generational()) {
233 entry_update_card_table();
234 }
235
236 if (ShenandoahVerify) {
237 vmop_entry_init_update_refs();
346 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
347 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
348
349 heap->try_inject_alloc_failure();
350 heap->try_inject_pin();
351 VM_ShenandoahInitUpdateRefs op(this);
352 VMThread::execute(&op);
353 }
354
355 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
356 ShenandoahHeap* const heap = ShenandoahHeap::heap();
357 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
358 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
359
360 heap->try_inject_alloc_failure();
361 heap->try_inject_pin();
362 VM_ShenandoahFinalUpdateRefs op(this);
363 VMThread::execute(&op);
364 }
365
366 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
367 ShenandoahHeap* const heap = ShenandoahHeap::heap();
368 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
369 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
370
371 // This phase does not use workers, no need for setup
372 heap->try_inject_alloc_failure();
373 heap->try_inject_pin();
374 VM_ShenandoahFinalVerify op(this);
375 VMThread::execute(&op);
376 }
377
378 void ShenandoahConcurrentGC::entry_init_mark() {
379 const char* msg = init_mark_event_message();
380 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
381 EventMark em("%s", msg);
382
383 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
384 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
385 "init marking");
806
807 // Has to be done after cset selection
808 heap->prepare_concurrent_roots();
809
810 if (!heap->collection_set()->is_empty()) {
811 LogTarget(Debug, gc, cset) lt;
812 if (lt.is_enabled()) {
813 ResourceMark rm;
814 LogStream ls(lt);
815 heap->collection_set()->print_on(&ls);
816 }
817
818 if (ShenandoahVerify) {
819 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
820 heap->verifier()->verify_before_evacuation(_generation);
821 }
822
823 heap->set_evacuation_in_progress(true);
824 // From here on, we need to update references.
825 heap->set_has_forwarded_objects(true);
826
827 // Arm nmethods/stack for concurrent processing
828 ShenandoahCodeRoots::arm_nmethods();
829 ShenandoahStackWatermark::change_epoch_id();
830
831 } else {
832 if (ShenandoahVerify) {
833 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
834 if (has_in_place_promotions(heap)) {
835 heap->verifier()->verify_after_concmark_with_promotions(_generation);
836 } else {
837 heap->verifier()->verify_after_concmark(_generation);
838 }
839 }
840 }
841 }
842
843 {
844 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
845 heap->propagate_gc_state_to_all_threads();
846 }
847 }
848
849 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
850 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
851 }
852
853 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
854 private:
855 OopClosure* const _oops;
856 public:
857 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
858
859 void do_thread(Thread* thread) override {
860 JavaThread* const jt = JavaThread::cast(thread);
861 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
862 }
1043
1044 void ShenandoahConcurrentGC::op_class_unloading() {
1045 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1046 assert (heap->is_concurrent_weak_root_in_progress() &&
1047 heap->unload_classes(),
1048 "Checked by caller");
1049 heap->do_class_unloading();
1050 }
1051
1052 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1053 private:
1054 ShenandoahEvacuateUpdateMetadataClosure _cl;
1055
1056 public:
1057 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1058
1059 void do_nmethod(nmethod* n) {
1060 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1061 ShenandoahNMethodLocker locker(data->lock());
1062 data->oops_do(&_cl, /* fix_relocations = */ true);
1063 ShenandoahNMethod::disarm_nmethod(n);
1064 }
1065 };
1066
1067 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1068 private:
1069 ShenandoahPhaseTimings::Phase _phase;
1070 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1071 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1072 _cld_roots;
1073 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1074
1075 public:
1076 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1077 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1078 _phase(phase),
1079 _vm_roots(phase),
1080 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1081 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1082
1083 void work(uint worker_id) {
1224
1225 // If we are running in generational mode, this will also age active regions that
1226 // haven't been used for allocation.
1227 heap->update_heap_region_states(true /*concurrent*/);
1228
1229 heap->set_update_refs_in_progress(false);
1230 heap->set_has_forwarded_objects(false);
1231
1232 if (ShenandoahVerify) {
1233 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1234 heap->verifier()->verify_after_update_refs(_generation);
1235 }
1236
1237 if (VerifyAfterGC) {
1238 Universe::verify();
1239 }
1240
1241 heap->rebuild_free_set(true /*concurrent*/);
1242 _generation->heuristics()->start_idle_span();
1243
1244 {
1245 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1246 heap->propagate_gc_state_to_all_threads();
1247 }
1248 }
1249
1250 void ShenandoahConcurrentGC::entry_final_roots() {
1251 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1252 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
1253 const char* msg = conc_final_roots_event_message();
1254 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_final_roots);
1255 EventMark em("%s", msg);
1256
1257 heap->concurrent_final_roots();
1258 }
1259
1260 void ShenandoahConcurrentGC::op_verify_final() {
1261 assert(ShenandoahVerify, "Should have been checked before");
1262 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1263 heap->verifier()->verify_after_gc(_generation);
1264 }
1265
1266 void ShenandoahConcurrentGC::op_cleanup_complete() {
1267 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1268 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1269 "cleanup complete.");
1270 ShenandoahHeap::heap()->recycle_trash();
1271 }
1272
1273 void ShenandoahConcurrentGC::op_reset_after_collect() {
1274 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1275 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1276 "reset after collection.");
1277
|
197
198 // Perform concurrent class unloading before any regions get recycled. Class unloading may
199 // need to inspect unmarked objects in trashed regions.
200 if (heap->unload_classes()) {
201 entry_class_unloading();
202 }
203
204 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
205 // the space. This would be the last action if there is nothing to evacuate. Note that
206 // we will not age young-gen objects in the case that we skip evacuation.
207 entry_cleanup_early();
208
209 // Processing strong roots
210 // This may be skipped if there is nothing to update/evacuate.
211 // If so, strong_root_in_progress would be unset.
212 if (heap->is_concurrent_strong_root_in_progress()) {
213 entry_strong_roots();
214 }
215
216 // Roots processing is complete, put the weak roots flag down.
217 entry_conc_roots();
218 vmop_entry_final_roots();
219
220 // Continue the cycle with evacuation and optional update-refs.
221 // This may be skipped if there is nothing to evacuate.
222 // If so, evac_in_progress would be unset by collection set preparation code.
223 if (heap->is_evacuation_in_progress()) {
224 // Concurrently evacuate
225 entry_evacuate();
226 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
227 return false;
228 }
229
230 // Perform update-refs phase.
231 entry_concurrent_update_refs_prepare(heap);
232
233 if (ShenandoahHeap::heap()->mode()->is_generational()) {
234 entry_update_card_table();
235 }
236
237 if (ShenandoahVerify) {
238 vmop_entry_init_update_refs();
347 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
348 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
349
350 heap->try_inject_alloc_failure();
351 heap->try_inject_pin();
352 VM_ShenandoahInitUpdateRefs op(this);
353 VMThread::execute(&op);
354 }
355
356 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
357 ShenandoahHeap* const heap = ShenandoahHeap::heap();
358 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
359 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
360
361 heap->try_inject_alloc_failure();
362 heap->try_inject_pin();
363 VM_ShenandoahFinalUpdateRefs op(this);
364 VMThread::execute(&op);
365 }
366
367 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
368 ShenandoahHeap* const heap = ShenandoahHeap::heap();
369 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
370 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
371
372 // This phase does not use workers, no need for setup
373 heap->try_inject_alloc_failure();
374 VM_ShenandoahFinalRoots op(this);
375 VMThread::execute(&op);
376 }
377
378 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
379 ShenandoahHeap* const heap = ShenandoahHeap::heap();
380 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
381 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
382
383 // This phase does not use workers, no need for setup
384 heap->try_inject_alloc_failure();
385 heap->try_inject_pin();
386 VM_ShenandoahFinalVerify op(this);
387 VMThread::execute(&op);
388 }
389
390 void ShenandoahConcurrentGC::entry_init_mark() {
391 const char* msg = init_mark_event_message();
392 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
393 EventMark em("%s", msg);
394
395 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
396 ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
397 "init marking");
818
819 // Has to be done after cset selection
820 heap->prepare_concurrent_roots();
821
822 if (!heap->collection_set()->is_empty()) {
823 LogTarget(Debug, gc, cset) lt;
824 if (lt.is_enabled()) {
825 ResourceMark rm;
826 LogStream ls(lt);
827 heap->collection_set()->print_on(&ls);
828 }
829
830 if (ShenandoahVerify) {
831 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
832 heap->verifier()->verify_before_evacuation(_generation);
833 }
834
835 heap->set_evacuation_in_progress(true);
836 // From here on, we need to update references.
837 heap->set_has_forwarded_objects(true);
838 } else {
839 if (ShenandoahVerify) {
840 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
841 if (has_in_place_promotions(heap)) {
842 heap->verifier()->verify_after_concmark_with_promotions(_generation);
843 } else {
844 heap->verifier()->verify_after_concmark(_generation);
845 }
846 }
847 }
848 }
849
850 // Arm nmethods/stack for concurrent processing
851 ShenandoahCodeRoots::arm_nmethods();
852 ShenandoahStackWatermark::change_epoch_id();
853
854 {
855 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
856 heap->propagate_gc_state_to_all_threads();
857 }
858 }
859
860 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
861 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
862 }
863
864 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
865 private:
866 OopClosure* const _oops;
867 public:
868 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
869
870 void do_thread(Thread* thread) override {
871 JavaThread* const jt = JavaThread::cast(thread);
872 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
873 }
1054
1055 void ShenandoahConcurrentGC::op_class_unloading() {
1056 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1057 assert (heap->is_concurrent_weak_root_in_progress() &&
1058 heap->unload_classes(),
1059 "Checked by caller");
1060 heap->do_class_unloading();
1061 }
1062
1063 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1064 private:
1065 ShenandoahEvacuateUpdateMetadataClosure _cl;
1066
1067 public:
1068 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1069
1070 void do_nmethod(nmethod* n) {
1071 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1072 ShenandoahNMethodLocker locker(data->lock());
1073 data->oops_do(&_cl, /* fix_relocations = */ true);
1074 }
1075 };
1076
1077 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1078 private:
1079 ShenandoahPhaseTimings::Phase _phase;
1080 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1081 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1082 _cld_roots;
1083 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1084
1085 public:
1086 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1087 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1088 _phase(phase),
1089 _vm_roots(phase),
1090 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1091 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1092
1093 void work(uint worker_id) {
1234
1235 // If we are running in generational mode, this will also age active regions that
1236 // haven't been used for allocation.
1237 heap->update_heap_region_states(true /*concurrent*/);
1238
1239 heap->set_update_refs_in_progress(false);
1240 heap->set_has_forwarded_objects(false);
1241
1242 if (ShenandoahVerify) {
1243 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1244 heap->verifier()->verify_after_update_refs(_generation);
1245 }
1246
1247 if (VerifyAfterGC) {
1248 Universe::verify();
1249 }
1250
1251 heap->rebuild_free_set(true /*concurrent*/);
1252 _generation->heuristics()->start_idle_span();
1253
1254 {
1255 // Final pause: update GC barriers to idle state.
1256 ShenandoahCodeRoots::arm_nmethods();
1257 ShenandoahStackWatermark::change_epoch_id();
1258 }
1259
1260 {
1261 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1262 heap->propagate_gc_state_to_all_threads();
1263 }
1264 }
1265
1266 void ShenandoahConcurrentGC::entry_conc_roots() {
1267 const char* msg = conc_final_roots_event_message();
1268 ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_roots);
1269 EventMark em("%s", msg);
1270
1271 ShenandoahHeap::heap()->op_conc_roots();
1272 }
1273
1274 void ShenandoahConcurrentGC::entry_final_roots() {
1275 const char* msg = "Pause Final Roots";
1276 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
1277 EventMark em("%s", msg);
1278
1279 ShenandoahHeap::heap()->op_final_roots();
1280 }
1281
1282 void ShenandoahConcurrentGC::op_verify_final() {
1283 assert(ShenandoahVerify, "Should have been checked before");
1284 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1285 heap->verifier()->verify_after_gc(_generation);
1286 }
1287
1288 void ShenandoahConcurrentGC::op_cleanup_complete() {
1289 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1290 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1291 "cleanup complete.");
1292 ShenandoahHeap::heap()->recycle_trash();
1293 }
1294
1295 void ShenandoahConcurrentGC::op_reset_after_collect() {
1296 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1297 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1298 "reset after collection.");
1299
|