814 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
815 if (has_in_place_promotions(heap)) {
816 heap->verifier()->verify_after_concmark_with_promotions();
817 } else {
818 heap->verifier()->verify_after_concmark();
819 }
820 }
821 }
822 }
823
824 {
825 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
826 heap->propagate_gc_state_to_all_threads();
827 }
828 }
829
830 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
831 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
832 }
833
834 template<bool GENERATIONAL>
835 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
836 private:
837 OopClosure* const _oops;
838 public:
839 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
840
841 void do_thread(Thread* thread) override {
842 JavaThread* const jt = JavaThread::cast(thread);
843 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
844 if (GENERATIONAL) {
845 ShenandoahThreadLocalData::enable_plab_promotions(thread);
846 }
847 }
848 };
849
850 template<bool GENERATIONAL>
851 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
852 private:
853 ShenandoahJavaThreadsIterator _java_threads;
854
855 public:
856 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
857 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
858 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
859 }
860
861 void work(uint worker_id) override {
862 if (GENERATIONAL) {
863 Thread* worker_thread = Thread::current();
864 ShenandoahThreadLocalData::enable_plab_promotions(worker_thread);
865 }
866
867 // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure.
868 // Otherwise, may deadlock with watermark lock
869 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
870 ShenandoahConcurrentEvacThreadClosure<GENERATIONAL> thr_cl(&oops_cl);
871 _java_threads.threads_do(&thr_cl, worker_id);
872 }
873 };
874
875 void ShenandoahConcurrentGC::op_thread_roots() {
876 ShenandoahHeap* const heap = ShenandoahHeap::heap();
877 assert(heap->is_evacuation_in_progress(), "Checked by caller");
878 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
879 if (heap->mode()->is_generational()) {
880 ShenandoahConcurrentEvacUpdateThreadTask<true> task(heap->workers()->active_workers());
881 heap->workers()->run_task(&task);
882 } else {
883 ShenandoahConcurrentEvacUpdateThreadTask<false> task(heap->workers()->active_workers());
884 heap->workers()->run_task(&task);
885 }
886 }
887
888 void ShenandoahConcurrentGC::op_weak_refs() {
889 ShenandoahHeap* const heap = ShenandoahHeap::heap();
890 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
891 // Concurrent weak refs processing
892 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
893 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
894 ShenandoahBreakpoint::at_after_reference_processing_started();
895 }
896 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
897 }
898
899 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
900 private:
901 ShenandoahHeap* const _heap;
902 ShenandoahMarkingContext* const _mark_context;
903 bool _evac_in_progress;
904 Thread* const _thread;
905
|
814 ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
815 if (has_in_place_promotions(heap)) {
816 heap->verifier()->verify_after_concmark_with_promotions();
817 } else {
818 heap->verifier()->verify_after_concmark();
819 }
820 }
821 }
822 }
823
824 {
825 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_propagate_gc_state);
826 heap->propagate_gc_state_to_all_threads();
827 }
828 }
829
830 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
831 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
832 }
833
834 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
835 private:
836 OopClosure* const _oops;
837 public:
838 explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
839
840 void do_thread(Thread* thread) override {
841 JavaThread* const jt = JavaThread::cast(thread);
842 StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
843 }
844 };
845
846 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
847 private:
848 ShenandoahJavaThreadsIterator _java_threads;
849
850 public:
851 explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
852 WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
853 _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
854 }
855
856 void work(uint worker_id) override {
857 // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure.
858 // Otherwise, may deadlock with watermark lock
859 ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
860 ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
861 _java_threads.threads_do(&thr_cl, worker_id);
862 }
863 };
864
865 void ShenandoahConcurrentGC::op_thread_roots() {
866 const ShenandoahHeap* const heap = ShenandoahHeap::heap();
867 assert(heap->is_evacuation_in_progress(), "Checked by caller");
868 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
869 ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
870 heap->workers()->run_task(&task);
871 }
872
873 void ShenandoahConcurrentGC::op_weak_refs() {
874 ShenandoahHeap* const heap = ShenandoahHeap::heap();
875 assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
876 // Concurrent weak refs processing
877 ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
878 if (heap->gc_cause() == GCCause::_wb_breakpoint) {
879 ShenandoahBreakpoint::at_after_reference_processing_started();
880 }
881 _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
882 }
883
884 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
885 private:
886 ShenandoahHeap* const _heap;
887 ShenandoahMarkingContext* const _mark_context;
888 bool _evac_in_progress;
889 Thread* const _thread;
890
|