199
200 // Perform concurrent class unloading before any regions get recycled. Class unloading may
201 // need to inspect unmarked objects in trashed regions.
202 if (heap->unload_classes()) {
203 entry_class_unloading();
204 }
205
206 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
207 // the space. This would be the last action if there is nothing to evacuate. Note that
208 // we will not age young-gen objects in the case that we skip evacuation.
209 entry_cleanup_early();
210
211 // Processing strong roots
212 // This may be skipped if there is nothing to update/evacuate.
213 // If so, strong_root_in_progress would be unset.
214 if (heap->is_concurrent_strong_root_in_progress()) {
215 entry_strong_roots();
216 }
217
218 // Roots processing is complete, put the weak roots flag down.
219 entry_final_roots();
220
221 // Continue the cycle with evacuation and optional update-refs.
222 // This may be skipped if there is nothing to evacuate.
223 // If so, evac_in_progress would be unset by collection set preparation code.
224 if (heap->is_evacuation_in_progress()) {
225 // Concurrently evacuate
226 entry_evacuate();
227 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
228 return false;
229 }
230
231 // Perform update-refs phase.
232 entry_concurrent_update_refs_prepare(heap);
233
234 if (ShenandoahHeap::heap()->mode()->is_generational()) {
235 entry_update_card_table();
236 }
237
238 if (ShenandoahVerify) {
239 vmop_entry_init_update_refs();
348 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
349 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
350
351 heap->try_inject_alloc_failure();
352 heap->try_inject_pin();
353 VM_ShenandoahInitUpdateRefs op(this);
354 VMThread::execute(&op);
355 }
356
357 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
358 ShenandoahHeap* const heap = ShenandoahHeap::heap();
359 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
360 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
361
362 heap->try_inject_alloc_failure();
363 heap->try_inject_pin();
364 VM_ShenandoahFinalUpdateRefs op(this);
365 VMThread::execute(&op);
366 }
367
368 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
369 ShenandoahHeap* const heap = ShenandoahHeap::heap();
370 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
371 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
372
373 // This phase does not use workers, no need for setup
374 heap->try_inject_alloc_failure();
375 heap->try_inject_pin();
376 VM_ShenandoahFinalVerify op(this);
377 VMThread::execute(&op);
378 }
379
380 void ShenandoahConcurrentGC::entry_init_mark() {
381 ShenandoahHeap* const heap = ShenandoahHeap::heap();
382 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
383
384 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Init Mark", "");
385 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
386 EventMark em("%s", msg);
387
760 // Update region state for both young and old regions
761 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
762 ShenandoahInitMarkUpdateRegionStateClosure cl;
763 heap->parallel_heap_region_iterate(&cl);
764 heap->old_generation()->ref_processor()->reset_thread_locals();
765 } else {
766 // Update region state for only young regions
767 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
768 ShenandoahInitMarkUpdateRegionStateClosure cl;
769 _generation->parallel_heap_region_iterate(&cl);
770 }
771
772 // Weak reference processing
773 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
774 rp->reset_thread_locals();
775
776 // Make above changes visible to worker threads
777 OrderAccess::fence();
778
779 // Arm nmethods/stack for concurrent processing
780 ShenandoahCodeRoots::arm_nmethods();
781 ShenandoahStackWatermark::change_epoch_id();
782
783 {
784 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_propagate_gc_state);
785 heap->propagate_gc_state_to_all_threads();
786 }
787 }
788
789 void ShenandoahConcurrentGC::op_mark_roots() {
790 _mark.mark_concurrent_roots();
791 }
792
793 void ShenandoahConcurrentGC::op_mark() {
794 _mark.concurrent_mark();
795 }
796
797 void ShenandoahConcurrentGC::op_final_mark() {
798 ShenandoahHeap* const heap = ShenandoahHeap::heap();
799 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
800 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
801
817
818 // Has to be done after cset selection
819 heap->prepare_concurrent_roots();
820
821 if (!heap->collection_set()->is_empty()) {
822 LogTarget(Debug, gc, cset) lt;
823 if (lt.is_enabled()) {
824 ResourceMark rm;
825 LogStream ls(lt);
826 heap->collection_set()->print_on(&ls);
827 }
828
829 if (ShenandoahVerify) {
830 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
831 heap->verifier()->verify_before_evacuation(_generation);
832 }
833
834 heap->set_evacuation_in_progress(true);
835 // From here on, we need to update references.
836 heap->set_has_forwarded_objects(true);
837
838 // Arm nmethods/stack for concurrent processing
839 ShenandoahCodeRoots::arm_nmethods();
840 ShenandoahStackWatermark::change_epoch_id();
841
842 } else {
843 if (ShenandoahVerify) {
844 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
845 if (has_in_place_promotions(heap)) {
846 heap->verifier()->verify_after_concmark_with_promotions(_generation);
847 } else {
848 heap->verifier()->verify_after_concmark(_generation);
849 }
850 }
851 }
852 }
853
854 {
855 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
856 heap->propagate_gc_state_to_all_threads();
857 }
858 }
859
860 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
861 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
862 }
863
864 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
865 private:
866 OopClosure* const _oops;
867 public:
868 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
869
870 void do_thread(Thread* thread) override {
871 JavaThread* const jt = JavaThread::cast(thread);
872 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
873 }
1054
1055 void ShenandoahConcurrentGC::op_class_unloading() {
1056 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1057 assert (heap->is_concurrent_weak_root_in_progress() &&
1058 heap->unload_classes(),
1059 "Checked by caller");
1060 heap->do_class_unloading();
1061 }
1062
1063 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1064 private:
1065 ShenandoahEvacuateUpdateMetadataClosure _cl;
1066
1067 public:
1068 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1069
1070 void do_nmethod(nmethod* n) {
1071 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1072 ShenandoahNMethodLocker locker(data->lock());
1073 data->oops_do(&_cl, /* fix_relocations = */ true);
1074 ShenandoahNMethod::disarm_nmethod(n);
1075 }
1076 };
1077
1078 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1079 private:
1080 ShenandoahPhaseTimings::Phase _phase;
1081 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1082 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1083 _cld_roots;
1084 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1085
1086 public:
1087 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1088 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1089 _phase(phase),
1090 _vm_roots(phase),
1091 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1092 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1093
1094 void work(uint worker_id) {
1235
1236 // If we are running in generational mode, this will also age active regions that
1237 // haven't been used for allocation.
1238 heap->update_heap_region_states(true /*concurrent*/);
1239
1240 heap->set_update_refs_in_progress(false);
1241 heap->set_has_forwarded_objects(false);
1242
1243 if (ShenandoahVerify) {
1244 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1245 heap->verifier()->verify_after_update_refs(_generation);
1246 }
1247
1248 if (VerifyAfterGC) {
1249 Universe::verify();
1250 }
1251
1252 heap->rebuild_free_set(true /*concurrent*/);
1253 _generation->heuristics()->start_idle_span();
1254
1255 {
1256 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1257 heap->propagate_gc_state_to_all_threads();
1258 }
1259 }
1260
1261 void ShenandoahConcurrentGC::entry_final_roots() {
1262 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1263 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
1264 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Concurrent final roots", "");
1265 ShenandoahConcurrentSubphase gc_phase(msg, ShenandoahPhaseTimings::conc_final_roots);
1266 EventMark em("%s", msg);
1267
1268 heap->concurrent_final_roots();
1269 }
1270
1271 void ShenandoahConcurrentGC::op_verify_final() {
1272 assert(ShenandoahVerify, "Should have been checked before");
1273 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1274 heap->verifier()->verify_after_gc(_generation);
1275 }
1276
1277 void ShenandoahConcurrentGC::op_cleanup_complete() {
1278 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1279 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1280 "cleanup complete.");
1281 ShenandoahHeap::heap()->recycle_trash();
1282 }
1283
1284 void ShenandoahConcurrentGC::op_reset_after_collect() {
1285 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1286 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1287 "reset after collection.");
1288
|
199
200 // Perform concurrent class unloading before any regions get recycled. Class unloading may
201 // need to inspect unmarked objects in trashed regions.
202 if (heap->unload_classes()) {
203 entry_class_unloading();
204 }
205
206 // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
207 // the space. This would be the last action if there is nothing to evacuate. Note that
208 // we will not age young-gen objects in the case that we skip evacuation.
209 entry_cleanup_early();
210
211 // Processing strong roots
212 // This may be skipped if there is nothing to update/evacuate.
213 // If so, strong_root_in_progress would be unset.
214 if (heap->is_concurrent_strong_root_in_progress()) {
215 entry_strong_roots();
216 }
217
218 // Roots processing is complete, put the weak roots flag down.
219 vmop_entry_final_roots();
220
221 // Continue the cycle with evacuation and optional update-refs.
222 // This may be skipped if there is nothing to evacuate.
223 // If so, evac_in_progress would be unset by collection set preparation code.
224 if (heap->is_evacuation_in_progress()) {
225 // Concurrently evacuate
226 entry_evacuate();
227 if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
228 return false;
229 }
230
231 // Perform update-refs phase.
232 entry_concurrent_update_refs_prepare(heap);
233
234 if (ShenandoahHeap::heap()->mode()->is_generational()) {
235 entry_update_card_table();
236 }
237
238 if (ShenandoahVerify) {
239 vmop_entry_init_update_refs();
348 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
349 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
350
351 heap->try_inject_alloc_failure();
352 heap->try_inject_pin();
353 VM_ShenandoahInitUpdateRefs op(this);
354 VMThread::execute(&op);
355 }
356
357 void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
358 ShenandoahHeap* const heap = ShenandoahHeap::heap();
359 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
360 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
361
362 heap->try_inject_alloc_failure();
363 heap->try_inject_pin();
364 VM_ShenandoahFinalUpdateRefs op(this);
365 VMThread::execute(&op);
366 }
367
368 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
369 ShenandoahHeap* const heap = ShenandoahHeap::heap();
370 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
371 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
372
373 // This phase does not use workers, no need for setup
374 heap->try_inject_pin();
375 VM_ShenandoahFinalRoots op(this);
376 VMThread::execute(&op);
377 }
378
379 void ShenandoahConcurrentGC::vmop_entry_final_verify() {
380 ShenandoahHeap* const heap = ShenandoahHeap::heap();
381 TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
382 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_verify_gross);
383
384 // This phase does not use workers, no need for setup
385 heap->try_inject_alloc_failure();
386 heap->try_inject_pin();
387 VM_ShenandoahFinalVerify op(this);
388 VMThread::execute(&op);
389 }
390
391 void ShenandoahConcurrentGC::entry_init_mark() {
392 ShenandoahHeap* const heap = ShenandoahHeap::heap();
393 assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
394
395 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Init Mark", "");
396 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
397 EventMark em("%s", msg);
398
771 // Update region state for both young and old regions
772 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
773 ShenandoahInitMarkUpdateRegionStateClosure cl;
774 heap->parallel_heap_region_iterate(&cl);
775 heap->old_generation()->ref_processor()->reset_thread_locals();
776 } else {
777 // Update region state for only young regions
778 ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
779 ShenandoahInitMarkUpdateRegionStateClosure cl;
780 _generation->parallel_heap_region_iterate(&cl);
781 }
782
783 // Weak reference processing
784 ShenandoahReferenceProcessor* rp = _generation->ref_processor();
785 rp->reset_thread_locals();
786
787 // Make above changes visible to worker threads
788 OrderAccess::fence();
789
790 // Arm nmethods/stack for concurrent processing
791 CodeCache::arm_all_nmethods();
792
793 {
794 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_propagate_gc_state);
795 heap->propagate_gc_state_to_all_threads();
796 }
797 }
798
799 void ShenandoahConcurrentGC::op_mark_roots() {
800 _mark.mark_concurrent_roots();
801 }
802
803 void ShenandoahConcurrentGC::op_mark() {
804 _mark.concurrent_mark();
805 }
806
807 void ShenandoahConcurrentGC::op_final_mark() {
808 ShenandoahHeap* const heap = ShenandoahHeap::heap();
809 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
810 assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
811
827
828 // Has to be done after cset selection
829 heap->prepare_concurrent_roots();
830
831 if (!heap->collection_set()->is_empty()) {
832 LogTarget(Debug, gc, cset) lt;
833 if (lt.is_enabled()) {
834 ResourceMark rm;
835 LogStream ls(lt);
836 heap->collection_set()->print_on(&ls);
837 }
838
839 if (ShenandoahVerify) {
840 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
841 heap->verifier()->verify_before_evacuation(_generation);
842 }
843
844 heap->set_evacuation_in_progress(true);
845 // From here on, we need to update references.
846 heap->set_has_forwarded_objects(true);
847 } else {
848 if (ShenandoahVerify) {
849 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
850 if (has_in_place_promotions(heap)) {
851 heap->verifier()->verify_after_concmark_with_promotions(_generation);
852 } else {
853 heap->verifier()->verify_after_concmark(_generation);
854 }
855 }
856 }
857 }
858
859 // Arm nmethods/stack for concurrent processing
860 CodeCache::arm_all_nmethods();
861
862 {
863 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
864 heap->propagate_gc_state_to_all_threads();
865 }
866 }
867
868 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
869 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
870 }
871
872 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
873 private:
874 OopClosure* const _oops;
875 public:
876 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
877
878 void do_thread(Thread* thread) override {
879 JavaThread* const jt = JavaThread::cast(thread);
880 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
881 }
1062
1063 void ShenandoahConcurrentGC::op_class_unloading() {
1064 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1065 assert (heap->is_concurrent_weak_root_in_progress() &&
1066 heap->unload_classes(),
1067 "Checked by caller");
1068 heap->do_class_unloading();
1069 }
1070
1071 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1072 private:
1073 ShenandoahEvacuateUpdateMetadataClosure _cl;
1074
1075 public:
1076 ShenandoahEvacUpdateCodeCacheClosure() : _cl() {}
1077
1078 void do_nmethod(nmethod* n) {
1079 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1080 ShenandoahNMethodLocker locker(data->lock());
1081 data->oops_do(&_cl, /* fix_relocations = */ true);
1082 }
1083 };
1084
1085 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1086 private:
1087 ShenandoahPhaseTimings::Phase _phase;
1088 ShenandoahVMRoots<true /*concurrent*/> _vm_roots;
1089 ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1090 _cld_roots;
1091 ShenandoahConcurrentNMethodIterator _nmethod_itr;
1092
1093 public:
1094 ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1095 WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1096 _phase(phase),
1097 _vm_roots(phase),
1098 _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1099 _nmethod_itr(ShenandoahCodeRoots::table()) {}
1100
1101 void work(uint worker_id) {
1242
1243 // If we are running in generational mode, this will also age active regions that
1244 // haven't been used for allocation.
1245 heap->update_heap_region_states(true /*concurrent*/);
1246
1247 heap->set_update_refs_in_progress(false);
1248 heap->set_has_forwarded_objects(false);
1249
1250 if (ShenandoahVerify) {
1251 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
1252 heap->verifier()->verify_after_update_refs(_generation);
1253 }
1254
1255 if (VerifyAfterGC) {
1256 Universe::verify();
1257 }
1258
1259 heap->rebuild_free_set(true /*concurrent*/);
1260 _generation->heuristics()->start_idle_span();
1261
1262 // Final pause: update GC barriers to idle state.
1263 CodeCache::arm_all_nmethods();
1264
1265 {
1266 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_propagate_gc_state);
1267 heap->propagate_gc_state_to_all_threads();
1268 }
1269 }
1270
1271 void ShenandoahConcurrentGC::entry_final_roots() {
1272 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1273 SHENANDOAH_EVENT_MESSAGE(msg, _generation->type(), "Pause Final Roots", "");
1274 ShenandoahPauseSubphase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
1275 EventMark em("%s", msg);
1276
1277 heap->op_final_roots();
1278 }
1279
1280 void ShenandoahConcurrentGC::op_verify_final() {
1281 assert(ShenandoahVerify, "Should have been checked before");
1282 ShenandoahHeap* const heap = ShenandoahHeap::heap();
1283 heap->verifier()->verify_after_gc(_generation);
1284 }
1285
1286 void ShenandoahConcurrentGC::op_cleanup_complete() {
1287 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1288 ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
1289 "cleanup complete.");
1290 ShenandoahHeap::heap()->recycle_trash();
1291 }
1292
1293 void ShenandoahConcurrentGC::op_reset_after_collect() {
1294 ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
1295 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
1296 "reset after collection.");
1297
|