< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp

Print this page

        

*** 21,33 **** * */ #include "precompiled.hpp" #include "code/codeCache.hpp" #include "code/nmethod.hpp" - #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahCodeRoots.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" ShenandoahParallelCodeCacheIterator::ShenandoahParallelCodeCacheIterator(const GrowableArray<CodeHeap*>* heaps) { --- 21,35 ---- * */ #include "precompiled.hpp" #include "code/codeCache.hpp" + #include "code/icBuffer.hpp" #include "code/nmethod.hpp" #include "gc/shenandoah/shenandoahCodeRoots.hpp" + #include "gc/shenandoah/shenandoahHeap.inline.hpp" + #include "gc/shenandoah/shenandoahNMethod.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" ShenandoahParallelCodeCacheIterator::ShenandoahParallelCodeCacheIterator(const GrowableArray<CodeHeap*>* heaps) {
*** 95,208 **** } _finished = true; } ! class ShenandoahNMethodOopDetector : public OopClosure { ! private: ! ResourceMark rm; // For growable array allocation below. ! GrowableArray<oop*> _oops; ! ! public: ! ShenandoahNMethodOopDetector() : _oops(10) {}; ! ! void do_oop(oop* o) { ! _oops.append(o); ! } ! void do_oop(narrowOop* o) { ! fatal("NMethods should not have compressed oops embedded."); ! } ! ! GrowableArray<oop*>* oops() { ! return &_oops; ! } ! ! bool has_oops() { ! return !_oops.is_empty(); ! } ! }; ! ! GrowableArray<ShenandoahNMethod*>* ShenandoahCodeRoots::_recorded_nms; ! ShenandoahLock ShenandoahCodeRoots::_recorded_nms_lock; void ShenandoahCodeRoots::initialize() { ! _recorded_nms = new (ResourceObj::C_HEAP, mtGC) GrowableArray<ShenandoahNMethod*>(100, true, mtGC); } ! void ShenandoahCodeRoots::add_nmethod(nmethod* nm) { switch (ShenandoahCodeRootsStyle) { case 0: case 1: break; case 2: { assert_locked_or_safepoint(CodeCache_lock); ! ShenandoahLocker locker(CodeCache_lock->owned_by_self() ? NULL : &_recorded_nms_lock); ! ! ShenandoahNMethodOopDetector detector; ! nm->oops_do(&detector); ! ! if (detector.has_oops()) { ! ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops()); ! nmr->assert_alive_and_correct(); ! int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod); ! if (idx != -1) { ! ShenandoahNMethod* old = _recorded_nms->at(idx); ! _recorded_nms->at_put(idx, nmr); ! delete old; ! } else { ! _recorded_nms->append(nmr); ! } ! } break; } default: ShouldNotReachHere(); } ! }; ! void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) { switch (ShenandoahCodeRootsStyle) { case 0: case 1: { break; } case 2: { assert_locked_or_safepoint(CodeCache_lock); ! ShenandoahLocker locker(CodeCache_lock->owned_by_self() ? NULL : &_recorded_nms_lock); ! ! ShenandoahNMethodOopDetector detector; ! nm->oops_do(&detector, /* allow_dead = */ true); ! if (detector.has_oops()) { ! int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod); ! assert(idx != -1, "nmethod " PTR_FORMAT " should be registered", p2i(nm)); ! ShenandoahNMethod* old = _recorded_nms->at(idx); ! old->assert_same_oops(detector.oops()); ! _recorded_nms->delete_at(idx); ! delete old; ! } break; } default: ShouldNotReachHere(); } } ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() : - _heap(ShenandoahHeap::heap()), _par_iterator(CodeCache::heaps()), ! _claimed(0) { assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint"); assert(!Thread::current()->is_Worker_thread(), "Should not be acquired by workers"); switch (ShenandoahCodeRootsStyle) { case 0: case 1: { // No need to do anything here break; } case 2: { CodeCache_lock->lock_without_safepoint_check(); break; } default: ShouldNotReachHere(); } --- 97,349 ---- } _finished = true; } ! ShenandoahNMethodTable* ShenandoahCodeRoots::_nmethod_table; ! int ShenandoahCodeRoots::_disarmed_value = 1; void ShenandoahCodeRoots::initialize() { ! _nmethod_table = new ShenandoahNMethodTable(); } ! void ShenandoahCodeRoots::register_nmethod(nmethod* nm) { switch (ShenandoahCodeRootsStyle) { case 0: case 1: break; case 2: { assert_locked_or_safepoint(CodeCache_lock); ! _nmethod_table->register_nmethod(nm); break; } default: ShouldNotReachHere(); } ! } ! void ShenandoahCodeRoots::unregister_nmethod(nmethod* nm) { switch (ShenandoahCodeRootsStyle) { case 0: case 1: { break; } case 2: { assert_locked_or_safepoint(CodeCache_lock); ! _nmethod_table->unregister_nmethod(nm); ! break; ! } ! default: ! ShouldNotReachHere(); ! } ! } ! void ShenandoahCodeRoots::flush_nmethod(nmethod* nm) { ! switch (ShenandoahCodeRootsStyle) { ! case 0: ! case 1: { ! break; ! } ! case 2: { ! assert_locked_or_safepoint(CodeCache_lock); ! _nmethod_table->flush_nmethod(nm); break; } default: ShouldNotReachHere(); } } + void ShenandoahCodeRoots::prepare_concurrent_unloading() { + assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint"); + _disarmed_value ++; + // 0 is reserved for new nmethod + if (_disarmed_value == 0) { + _disarmed_value = 1; + } + + JavaThreadIteratorWithHandle jtiwh; + for (JavaThread *thr = jtiwh.next(); thr != NULL; thr = jtiwh.next()) { + ShenandoahThreadLocalData::set_disarmed_value(thr, _disarmed_value); + } + } + + class ShenandoahNMethodUnlinkClosure : public NMethodClosure { + private: + bool _unloading_occurred; + volatile bool _failed; + ShenandoahHeap* _heap; + + void set_failed() { + Atomic::store(true, &_failed); + } + + void unlink(nmethod* nm) { + // Unlinking of the dependencies must happen before the + // handshake separating unlink and purge. + nm->flush_dependencies(false /* delete_immediately */); + + // We don't need to take the lock when unlinking nmethods from + // the Method, because it is only concurrently unlinked by + // the entry barrier, which acquires the per nmethod lock. + nm->unlink_from_method(false /* acquire_lock */); + + if (nm->is_osr_method()) { + // Invalidate the osr nmethod only once + nm->invalidate_osr_method(); + } + } + public: + ShenandoahNMethodUnlinkClosure(bool unloading_occurred) : + _unloading_occurred(unloading_occurred), + _failed(false), + _heap(ShenandoahHeap::heap()) {} + + virtual void do_nmethod(nmethod* nm) { + if (failed()) { + return; + } + + ShenandoahNMethod* nm_data = ShenandoahNMethod::gc_data(nm); + assert(!nm_data->is_unregistered(), "Should not see unregistered entry"); + + if (!nm->is_alive()) { + return; + } + + ShenandoahReentrantLocker locker(nm_data->lock()); + + if (nm->is_unloading()) { + unlink(nm); + return; + } + + // Heal oops and disarm + ShenandoahNMethod::heal_nmethod(nm); + ShenandoahNMethod::disarm_nmethod(nm); + + // Clear compiled ICs and exception caches + if (!nm->unload_nmethod_caches(_unloading_occurred)) { + set_failed(); + } + } + + bool failed() const { + return Atomic::load(&_failed); + } + }; + + class ShenandoahUnlinkTask : public AbstractGangTask { + private: + ShenandoahNMethodUnlinkClosure _cl; + ICRefillVerifier* _verifier; + ShenandoahConcurrentNMethodIterator _iterator; + + public: + ShenandoahUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) : + AbstractGangTask("ShenandoahNMethodUnlinkTask"), + _cl(unloading_occurred), + _verifier(verifier), + _iterator(ShenandoahCodeRoots::table()) { + MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + _iterator.nmethods_do_begin(); + } + + ~ShenandoahUnlinkTask() { + MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + _iterator.nmethods_do_end(); + } + + virtual void work(uint worker_id) { + ICRefillVerifierMark mark(_verifier); + ShenandoahEvacOOMScope scope; + _iterator.nmethods_do(&_cl); + } + + bool success() const { + return !_cl.failed(); + } + }; + + void ShenandoahCodeRoots::unlink(WorkGang* workers, bool unloading_occurred) { + assert(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(), + "Only when running concurrent class unloading"); + + for (;;) { + ICRefillVerifier verifier; + + { + ShenandoahUnlinkTask task(unloading_occurred, &verifier); + workers->run_task(&task); + if (task.success()) { + return; + } + } + + // Cleaning failed because we ran out of transitional IC stubs, + // so we have to refill and try again. Refilling requires taking + // a safepoint, so we temporarily leave the suspendible thread set. + SuspendibleThreadSetLeaver sts; + InlineCacheBuffer::refill_ic_stubs(); + } + } + + class ShenandoahNMethodPurgeClosure : public NMethodClosure { + public: + virtual void do_nmethod(nmethod* nm) { + if (nm->is_alive() && nm->is_unloading()) { + nm->make_unloaded(); + } + } + }; + + class ShenandoahNMethodPurgeTask : public AbstractGangTask { + private: + ShenandoahNMethodPurgeClosure _cl; + ShenandoahConcurrentNMethodIterator _iterator; + + public: + ShenandoahNMethodPurgeTask() : + AbstractGangTask("ShenandoahNMethodPurgeTask"), + _cl(), + _iterator(ShenandoahCodeRoots::table()) { + MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + _iterator.nmethods_do_begin(); + } + + ~ShenandoahNMethodPurgeTask() { + MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + _iterator.nmethods_do_end(); + } + + virtual void work(uint worker_id) { + _iterator.nmethods_do(&_cl); + } + }; + + void ShenandoahCodeRoots::purge(WorkGang* workers) { + assert(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(), + "Only when running concurrent class unloading"); + + ShenandoahNMethodPurgeTask task; + workers->run_task(&task); + } + ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() : _par_iterator(CodeCache::heaps()), ! _table_snapshot(NULL) { assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint"); assert(!Thread::current()->is_Worker_thread(), "Should not be acquired by workers"); switch (ShenandoahCodeRootsStyle) { case 0: case 1: { // No need to do anything here break; } case 2: { CodeCache_lock->lock_without_safepoint_check(); + _table_snapshot = ShenandoahCodeRoots::table()->snapshot_for_iteration(); break; } default: ShouldNotReachHere(); }
*** 214,223 **** --- 355,366 ---- case 1: { // No need to do anything here break; } case 2: { + ShenandoahCodeRoots::table()->finish_iteration(_table_snapshot); + _table_snapshot = NULL; CodeCache_lock->unlock(); break; } default: ShouldNotReachHere();
*** 255,329 **** } template <bool CSET_FILTER> void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) { assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint"); ! ! size_t stride = 256; // educated guess ! ! GrowableArray<ShenandoahNMethod*>* list = ShenandoahCodeRoots::_recorded_nms; ! ! size_t max = (size_t)list->length(); ! while (_claimed < max) { ! size_t cur = Atomic::add(stride, &_claimed) - stride; ! size_t start = cur; ! size_t end = MIN2(cur + stride, max); ! if (start >= max) break; ! ! for (size_t idx = start; idx < end; idx++) { ! ShenandoahNMethod* nmr = list->at((int) idx); ! nmr->assert_alive_and_correct(); ! ! if (CSET_FILTER && !nmr->has_cset_oops(_heap)) { ! continue; ! } ! ! f->do_code_blob(nmr->nm()); ! } ! } ! } ! ! ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>* oops) { ! _nm = nm; ! _oops = NEW_C_HEAP_ARRAY(oop*, oops->length(), mtGC); ! _oops_count = oops->length(); ! for (int c = 0; c < _oops_count; c++) { ! _oops[c] = oops->at(c); ! } } - ShenandoahNMethod::~ShenandoahNMethod() { - if (_oops != NULL) { - FREE_C_HEAP_ARRAY(oop*, _oops); - } - } - - bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) { - for (int c = 0; c < _oops_count; c++) { - oop o = RawAccess<>::oop_load(_oops[c]); - if (heap->in_collection_set(o)) { - return true; - } - } - return false; - } - - #ifdef ASSERT - void ShenandoahNMethod::assert_alive_and_correct() { - assert(_nm->is_alive(), "only alive nmethods here"); - assert(_oops_count > 0, "should have filtered nmethods without oops before"); - ShenandoahHeap* heap = ShenandoahHeap::heap(); - for (int c = 0; c < _oops_count; c++) { - oop *loc = _oops[c]; - assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*"); - oop o = RawAccess<>::oop_load(loc); - shenandoah_assert_correct_except(loc, o, o == NULL || heap->is_full_gc_move_in_progress()); - } - } - - void ShenandoahNMethod::assert_same_oops(GrowableArray<oop*>* oops) { - assert(_oops_count == oops->length(), "should have the same number of oop*"); - for (int c = 0; c < _oops_count; c++) { - assert(_oops[c] == oops->at(c), "should be the same oop*"); - } - } - #endif --- 398,406 ---- } template <bool CSET_FILTER> void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) { assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint"); ! assert(_table_snapshot != NULL, "Sanity"); ! _table_snapshot->parallel_blobs_do<CSET_FILTER>(f); }
< prev index next >