< prev index next >

src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp

Print this page

        

*** 21,36 **** * questions. * */ #include "precompiled.hpp" - #include "gc/g1/g1BufferNodeList.hpp" - #include "gc/g1/g1CardTableEntryClosure.hpp" #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/g1/g1DirtyCardQueue.hpp" #include "gc/g1/g1FreeIdSet.hpp" - #include "gc/g1/g1RedirtyCardsQueue.hpp" #include "gc/g1/g1RemSet.hpp" #include "gc/g1/g1ThreadLocalData.hpp" #include "gc/g1/heapRegionRemSet.hpp" #include "gc/shared/suspendibleThreadSet.hpp" #include "gc/shared/workgroup.hpp" --- 21,33 ----
*** 78,237 **** } else { allocate_buffer(); // Buffer enqueued, get a new one. } } ! G1DirtyCardQueueSet::G1DirtyCardQueueSet() : ! PtrQueueSet(), ! _cbl_mon(NULL), ! _completed_buffers_head(NULL), ! _completed_buffers_tail(NULL), ! _num_cards(0), ! _process_cards_threshold(ProcessCardsThresholdNever), ! _process_completed_buffers(false), ! _max_cards(MaxCardsUnlimited), ! _max_cards_padding(0), ! _free_ids(0, num_par_ids()), _processed_buffers_mut(0), ! _processed_buffers_rs_thread(0) { _all_active = true; } G1DirtyCardQueueSet::~G1DirtyCardQueueSet() { ! abandon_completed_buffers(); } // Determines how many mutator threads can process the buffers in parallel. uint G1DirtyCardQueueSet::num_par_ids() { return (uint)os::initial_active_processor_count(); } void G1DirtyCardQueueSet::initialize(Monitor* cbl_mon, ! BufferNode::Allocator* allocator) { ! PtrQueueSet::initialize(allocator); ! assert(_cbl_mon == NULL, "Init order issue?"); ! _cbl_mon = cbl_mon; } void G1DirtyCardQueueSet::handle_zero_index_for_thread(Thread* t) { G1ThreadLocalData::dirty_card_queue(t).handle_zero_index(); } - void G1DirtyCardQueueSet::enqueue_completed_buffer(BufferNode* cbn) { - MonitorLocker ml(_cbl_mon, Mutex::_no_safepoint_check_flag); - cbn->set_next(NULL); - if (_completed_buffers_tail == NULL) { - assert(_completed_buffers_head == NULL, "Well-formedness"); - _completed_buffers_head = cbn; - _completed_buffers_tail = cbn; - } else { - _completed_buffers_tail->set_next(cbn); - _completed_buffers_tail = cbn; - } - _num_cards += buffer_size() - cbn->index(); - - if (!process_completed_buffers() && - (num_cards() > process_cards_threshold())) { - set_process_completed_buffers(true); - ml.notify_all(); - } - verify_num_cards(); - } - - BufferNode* G1DirtyCardQueueSet::get_completed_buffer(size_t stop_at) { - MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); - - if (num_cards() <= stop_at) { - return NULL; - } - - assert(num_cards() > 0, "invariant"); - assert(_completed_buffers_head != NULL, "invariant"); - assert(_completed_buffers_tail != NULL, "invariant"); - - BufferNode* bn = _completed_buffers_head; - _num_cards -= buffer_size() - bn->index(); - _completed_buffers_head = bn->next(); - if (_completed_buffers_head == NULL) { - assert(num_cards() == 0, "invariant"); - _completed_buffers_tail = NULL; - set_process_completed_buffers(false); - } - verify_num_cards(); - bn->set_next(NULL); - return bn; - } - - #ifdef ASSERT - void G1DirtyCardQueueSet::verify_num_cards() const { - size_t actual = 0; - BufferNode* cur = _completed_buffers_head; - while (cur != NULL) { - actual += buffer_size() - cur->index(); - cur = cur->next(); - } - assert(actual == _num_cards, - "Num entries in completed buffers should be " SIZE_FORMAT " but are " SIZE_FORMAT, - _num_cards, actual); - } - #endif - - void G1DirtyCardQueueSet::abandon_completed_buffers() { - BufferNode* buffers_to_delete = NULL; - { - MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); - buffers_to_delete = _completed_buffers_head; - _completed_buffers_head = NULL; - _completed_buffers_tail = NULL; - _num_cards = 0; - set_process_completed_buffers(false); - } - while (buffers_to_delete != NULL) { - BufferNode* bn = buffers_to_delete; - buffers_to_delete = bn->next(); - bn->set_next(NULL); - deallocate_buffer(bn); - } - } - - void G1DirtyCardQueueSet::notify_if_necessary() { - MonitorLocker ml(_cbl_mon, Mutex::_no_safepoint_check_flag); - if (num_cards() > process_cards_threshold()) { - set_process_completed_buffers(true); - ml.notify_all(); - } - } - - // Merge lists of buffers. Notify the processing threads. - // The source queue is emptied as a result. The queues - // must share the monitor. - void G1DirtyCardQueueSet::merge_bufferlists(G1RedirtyCardsQueueSet* src) { - assert(allocator() == src->allocator(), "precondition"); - const G1BufferNodeList from = src->take_all_completed_buffers(); - if (from._head == NULL) return; - - MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag); - if (_completed_buffers_tail == NULL) { - assert(_completed_buffers_head == NULL, "Well-formedness"); - _completed_buffers_head = from._head; - _completed_buffers_tail = from._tail; - } else { - assert(_completed_buffers_head != NULL, "Well formedness"); - _completed_buffers_tail->set_next(from._head); - _completed_buffers_tail = from._tail; - } - _num_cards += from._entry_count; - - assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL || - _completed_buffers_head != NULL && _completed_buffers_tail != NULL, - "Sanity"); - verify_num_cards(); - } - bool G1DirtyCardQueueSet::apply_closure_to_buffer(G1CardTableEntryClosure* cl, BufferNode* node, uint worker_i) { if (cl == NULL) return true; bool result = true; void** buf = BufferNode::make_buffer_from_node(node); size_t i = node->index(); --- 75,121 ---- } else { allocate_buffer(); // Buffer enqueued, get a new one. } } ! G1DirtyCardQueueSet::G1DirtyCardQueueSet(bool notify_when_complete) : ! PtrQueueSet(notify_when_complete), ! _max_completed_buffers(MaxCompletedBuffersUnlimited), ! _completed_buffers_padding(0), ! _free_ids(NULL), _processed_buffers_mut(0), ! _processed_buffers_rs_thread(0), ! _cur_par_buffer_node(NULL) { _all_active = true; } G1DirtyCardQueueSet::~G1DirtyCardQueueSet() { ! delete _free_ids; } // Determines how many mutator threads can process the buffers in parallel. uint G1DirtyCardQueueSet::num_par_ids() { return (uint)os::initial_active_processor_count(); } void G1DirtyCardQueueSet::initialize(Monitor* cbl_mon, ! BufferNode::Allocator* allocator, ! bool init_free_ids) { ! PtrQueueSet::initialize(cbl_mon, allocator); ! if (init_free_ids) { ! _free_ids = new G1FreeIdSet(0, num_par_ids()); ! } } void G1DirtyCardQueueSet::handle_zero_index_for_thread(Thread* t) { G1ThreadLocalData::dirty_card_queue(t).handle_zero_index(); } bool G1DirtyCardQueueSet::apply_closure_to_buffer(G1CardTableEntryClosure* cl, BufferNode* node, + bool consume, uint worker_i) { if (cl == NULL) return true; bool result = true; void** buf = BufferNode::make_buffer_from_node(node); size_t i = node->index();
*** 242,253 **** if (!cl->do_card_ptr(card_ptr, worker_i)) { result = false; // Incomplete processing. break; } } ! assert(i <= buffer_size(), "invariant"); ! node->set_index(i); return result; } #ifndef ASSERT #define assert_fully_consumed(node, buffer_size) --- 126,139 ---- if (!cl->do_card_ptr(card_ptr, worker_i)) { result = false; // Incomplete processing. break; } } ! if (consume) { ! assert(i <= buffer_size(), "invariant"); ! node->set_index(i); ! } return result; } #ifndef ASSERT #define assert_fully_consumed(node, buffer_size)
*** 267,292 **** if (Thread::current()->is_Java_thread()) { // If the number of buffers exceeds the limit, make this Java // thread do the processing itself. We don't lock to access // buffer count or padding; it is fine to be imprecise here. The // add of padding could overflow, which is treated as unlimited. ! size_t limit = max_cards() + max_cards_padding(); ! if ((num_cards() > limit) && (limit >= max_cards())) { if (mut_process_buffer(node)) { return true; } } } enqueue_completed_buffer(node); return false; } bool G1DirtyCardQueueSet::mut_process_buffer(BufferNode* node) { ! uint worker_id = _free_ids.claim_par_id(); // temporarily claim an id G1RefineCardConcurrentlyClosure cl; ! bool result = apply_closure_to_buffer(&cl, node, worker_id); ! _free_ids.release_par_id(worker_id); // release the id if (result) { assert_fully_consumed(node, buffer_size()); Atomic::inc(&_processed_buffers_mut); } --- 153,181 ---- if (Thread::current()->is_Java_thread()) { // If the number of buffers exceeds the limit, make this Java // thread do the processing itself. We don't lock to access // buffer count or padding; it is fine to be imprecise here. The // add of padding could overflow, which is treated as unlimited. ! size_t max_buffers = max_completed_buffers(); ! size_t limit = max_buffers + completed_buffers_padding(); ! if ((completed_buffers_num() > limit) && (limit >= max_buffers)) { if (mut_process_buffer(node)) { return true; } } } enqueue_completed_buffer(node); return false; } bool G1DirtyCardQueueSet::mut_process_buffer(BufferNode* node) { ! guarantee(_free_ids != NULL, "must be"); ! ! uint worker_i = _free_ids->claim_par_id(); // temporarily claim an id G1RefineCardConcurrentlyClosure cl; ! bool result = apply_closure_to_buffer(&cl, node, true, worker_i); ! _free_ids->release_par_id(worker_i); // release the id if (result) { assert_fully_consumed(node, buffer_size()); Atomic::inc(&_processed_buffers_mut); }
*** 310,320 **** assert(!during_pause || stop_at == 0, "Should not leave any completed buffers during a pause"); BufferNode* nd = get_completed_buffer(stop_at); if (nd == NULL) { return false; } else { ! if (apply_closure_to_buffer(cl, nd, worker_i)) { assert_fully_consumed(nd, buffer_size()); // Done with fully processed buffer. deallocate_buffer(nd); Atomic::inc(&_processed_buffers_rs_thread); } else { --- 199,209 ---- assert(!during_pause || stop_at == 0, "Should not leave any completed buffers during a pause"); BufferNode* nd = get_completed_buffer(stop_at); if (nd == NULL) { return false; } else { ! if (apply_closure_to_buffer(cl, nd, true, worker_i)) { assert_fully_consumed(nd, buffer_size()); // Done with fully processed buffer. deallocate_buffer(nd); Atomic::inc(&_processed_buffers_rs_thread); } else {
*** 324,333 **** --- 213,237 ---- } return true; } } + void G1DirtyCardQueueSet::par_apply_closure_to_all_completed_buffers(G1CardTableEntryClosure* cl) { + BufferNode* nd = _cur_par_buffer_node; + while (nd != NULL) { + BufferNode* next = nd->next(); + BufferNode* actual = Atomic::cmpxchg(next, &_cur_par_buffer_node, nd); + if (actual == nd) { + bool b = apply_closure_to_buffer(cl, nd, false); + guarantee(b, "Should not stop early."); + nd = next; + } else { + nd = actual; + } + } + } + void G1DirtyCardQueueSet::abandon_logs() { assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); abandon_completed_buffers(); // Since abandon is done only at safepoints, we can safely manipulate
*** 345,356 **** void G1DirtyCardQueueSet::concatenate_logs() { // Iterate over all the threads, if we find a partial log add it to // the global list of logs. Temporarily turn off the limit on the number // of outstanding buffers. assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); ! size_t old_limit = max_cards(); ! set_max_cards(MaxCardsUnlimited); struct ConcatenateThreadLogClosure : public ThreadClosure { virtual void do_thread(Thread* t) { G1DirtyCardQueue& dcq = G1ThreadLocalData::dirty_card_queue(t); if (!dcq.is_empty()) { --- 249,260 ---- void G1DirtyCardQueueSet::concatenate_logs() { // Iterate over all the threads, if we find a partial log add it to // the global list of logs. Temporarily turn off the limit on the number // of outstanding buffers. assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); ! size_t old_limit = max_completed_buffers(); ! set_max_completed_buffers(MaxCompletedBuffersUnlimited); struct ConcatenateThreadLogClosure : public ThreadClosure { virtual void do_thread(Thread* t) { G1DirtyCardQueue& dcq = G1ThreadLocalData::dirty_card_queue(t); if (!dcq.is_empty()) {
*** 359,365 **** } } closure; Threads::threads_do(&closure); G1BarrierSet::shared_dirty_card_queue().flush(); ! set_max_cards(old_limit); } --- 263,269 ---- } } closure; Threads::threads_do(&closure); G1BarrierSet::shared_dirty_card_queue().flush(); ! set_max_completed_buffers(old_limit); }
< prev index next >