< prev index next >

src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp

Print this page




 675 void ConcurrentMarkSweepGeneration::print_statistics() {
 676   cmsSpace()->printFLCensus(0);
 677 }
 678 #endif
 679 
 680 size_t
 681 ConcurrentMarkSweepGeneration::contiguous_available() const {
 682   // dld proposes an improvement in precision here. If the committed
 683   // part of the space ends in a free block we should add that to
 684   // uncommitted size in the calculation below. Will make this
 685   // change later, staying with the approximation below for the
 686   // time being. -- ysr.
 687   return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
 688 }
 689 
 690 size_t
 691 ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {
 692   return _cmsSpace->max_alloc_in_words() * HeapWordSize;
 693 }
 694 
 695 size_t ConcurrentMarkSweepGeneration::used_stable() const {
 696   return cmsSpace()->used_stable();
 697 }
 698 
 699 size_t ConcurrentMarkSweepGeneration::max_available() const {
 700   return free() + _virtual_space.uncommitted_size();
 701 }
 702 
 703 bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
 704   size_t available = max_available();
 705   size_t av_promo  = (size_t)gc_stats()->avg_promoted()->padded_average();
 706   bool   res = (available >= av_promo) || (available >= max_promotion_in_bytes);
 707   log_trace(gc, promotion)("CMS: promo attempt is%s safe: available(" SIZE_FORMAT ") %s av_promo(" SIZE_FORMAT "), max_promo(" SIZE_FORMAT ")",
 708                            res? "":" not", available, res? ">=":"<", av_promo, max_promotion_in_bytes);
 709   return res;
 710 }
 711 
 712 // At a promotion failure dump information on block layout in heap
 713 // (cms old generation).
 714 void ConcurrentMarkSweepGeneration::promotion_failure_occurred() {
 715   Log(gc, promotion) log;
 716   if (log.is_trace()) {
 717     LogStream ls(log.trace());
 718     cmsSpace()->dump_at_safepoint_with_locks(collector(), &ls);


 997 // TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT
 998 //
 999 //
1000 // STATE TRANSITION DIAGRAM
1001 //
1002 //        mut / parnew                     mut  /  parnew
1003 // FREE --------------------> TRANSIENT ---------------------> OBJECT --|
1004 //  ^                                                                   |
1005 //  |------------------------ DEAD <------------------------------------|
1006 //         sweep                            mut
1007 //
1008 // While a block is in TRANSIENT state its size cannot be determined
1009 // so readers will either need to come back later or stall until
1010 // the size can be determined. Note that for the case of direct
1011 // allocation, P-bits, when available, may be used to determine the
1012 // size of an object that may not yet have been initialized.
1013 
1014 // Things to support parallel young-gen collection.
1015 oop
1016 ConcurrentMarkSweepGeneration::par_promote(int thread_num,
1017                                            oop old, markWord m,
1018                                            size_t word_sz) {
1019 #ifndef PRODUCT
1020   if (CMSHeap::heap()->promotion_should_fail()) {
1021     return NULL;
1022   }
1023 #endif  // #ifndef PRODUCT
1024 
1025   CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1026   PromotionInfo* promoInfo = &ps->promo;
1027   // if we are tracking promotions, then first ensure space for
1028   // promotion (including spooling space for saving header if necessary).
1029   // then allocate and copy, then track promoted info if needed.
1030   // When tracking (see PromotionInfo::track()), the mark word may
1031   // be displaced and in this case restoration of the mark word
1032   // occurs in the (oop_since_save_marks_)iterate phase.
1033   if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) {
1034     // Out of space for allocating spooling buffers;
1035     // try expanding and allocating spooling buffers.
1036     if (!expand_and_ensure_spooling_space(promoInfo)) {
1037       return NULL;


1510                                          max_eden_size,
1511                                          full,
1512                                          gc_cause,
1513                                          heap->soft_ref_policy());
1514 
1515   // Reset the expansion cause, now that we just completed
1516   // a collection cycle.
1517   clear_expansion_cause();
1518   _foregroundGCIsActive = false;
1519   return;
1520 }
1521 
1522 // Resize the tenured generation
1523 // after obtaining the free list locks for the
1524 // two generations.
1525 void CMSCollector::compute_new_size() {
1526   assert_locked_or_safepoint(Heap_lock);
1527   FreelistLocker z(this);
1528   MetaspaceGC::compute_new_size();
1529   _cmsGen->compute_new_size_free_list();
1530   // recalculate CMS used space after CMS collection
1531   _cmsGen->cmsSpace()->recalculate_used_stable();
1532 }
1533 
1534 // A work method used by the foreground collector to do
1535 // a mark-sweep-compact.
1536 void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
1537   CMSHeap* heap = CMSHeap::heap();
1538 
1539   STWGCTimer* gc_timer = GenMarkSweep::gc_timer();
1540   gc_timer->register_gc_start();
1541 
1542   SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer();
1543   gc_tracer->report_gc_start(heap->gc_cause(), gc_timer->gc_start());
1544 
1545   heap->pre_full_gc_dump(gc_timer);
1546 
1547   GCTraceTime(Trace, gc, phases) t("CMS:MSC");
1548 
1549   // Temporarily widen the span of the weak reference processing to
1550   // the entire heap.
1551   MemRegion new_span(CMSHeap::heap()->reserved_region());


2040   // there are any modified oops in the class. The remark phase also needs
2041   // that information. Tell the young collection to save the union of all
2042   // modified klasses.
2043   if (duringMarking) {
2044     _ct->cld_rem_set()->set_accumulate_modified_oops(true);
2045   }
2046 
2047   bool registerClosure = duringMarking;
2048 
2049   _cmsGen->gc_prologue_work(full, registerClosure, &_modUnionClosurePar);
2050 
2051   if (!full) {
2052     stats().record_gc0_begin();
2053   }
2054 }
2055 
2056 void ConcurrentMarkSweepGeneration::gc_prologue(bool full) {
2057 
2058   _capacity_at_prologue = capacity();
2059   _used_at_prologue = used();
2060   _cmsSpace->recalculate_used_stable();
2061 
2062   // We enable promotion tracking so that card-scanning can recognize
2063   // which objects have been promoted during this GC and skip them.
2064   for (uint i = 0; i < ParallelGCThreads; i++) {
2065     _par_gc_thread_states[i]->promo.startTrackingPromotions();
2066   }
2067 
2068   // Delegate to CMScollector which knows how to coordinate between
2069   // this and any other CMS generations that it is responsible for
2070   // collecting.
2071   collector()->gc_prologue(full);
2072 }
2073 
2074 // This is a "private" interface for use by this generation's CMSCollector.
2075 // Not to be called directly by any other entity (for instance,
2076 // GenCollectedHeap, which calls the "public" gc_prologue method above).
2077 void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,
2078   bool registerClosure, ModUnionClosure* modUnionClosure) {
2079   assert(!incremental_collection_failed(), "Shouldn't be set yet");
2080   assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL,


2113     // We have already been invoked; this is a gc_epilogue delegation
2114     // from yet another CMS generation that we are responsible for, just
2115     // ignore it since all relevant work has already been done.
2116     return;
2117   }
2118   assert(haveFreelistLocks(), "must have freelist locks");
2119   assert_lock_strong(bitMapLock());
2120 
2121   _ct->cld_rem_set()->set_accumulate_modified_oops(false);
2122 
2123   _cmsGen->gc_epilogue_work(full);
2124 
2125   if (_collectorState == AbortablePreclean || _collectorState == Precleaning) {
2126     // in case sampling was not already enabled, enable it
2127     _start_sampling = true;
2128   }
2129   // reset _eden_chunk_array so sampling starts afresh
2130   _eden_chunk_index = 0;
2131 
2132   size_t cms_used   = _cmsGen->cmsSpace()->used();
2133   _cmsGen->cmsSpace()->recalculate_used_stable();
2134 
2135   // update performance counters - this uses a special version of
2136   // update_counters() that allows the utilization to be passed as a
2137   // parameter, avoiding multiple calls to used().
2138   //
2139   _cmsGen->update_counters(cms_used);
2140 
2141   bitMapLock()->unlock();
2142   releaseFreelistLocks();
2143 
2144   if (!CleanChunkPoolAsync) {
2145     Chunk::clean_chunk_pool();
2146   }
2147 
2148   set_did_compact(false);
2149   _between_prologue_and_epilogue = false;  // ready for next cycle
2150 }
2151 
2152 void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) {
2153   collector()->gc_epilogue(full);


2807 // be approximate -- we'll do a catch up phase subsequently.]
2808 void CMSCollector::checkpointRootsInitial() {
2809   assert(_collectorState == InitialMarking, "Wrong collector state");
2810   check_correct_thread_executing();
2811   TraceCMSMemoryManagerStats tms(_collectorState, CMSHeap::heap()->gc_cause());
2812 
2813   save_heap_summary();
2814   report_heap_summary(GCWhen::BeforeGC);
2815 
2816   ReferenceProcessor* rp = ref_processor();
2817   assert(_restart_addr == NULL, "Control point invariant");
2818   {
2819     // acquire locks for subsequent manipulations
2820     MutexLocker x(bitMapLock(),
2821                   Mutex::_no_safepoint_check_flag);
2822     checkpointRootsInitialWork();
2823     // enable ("weak") refs discovery
2824     rp->enable_discovery();
2825     _collectorState = Marking;
2826   }
2827 
2828   _cmsGen->cmsSpace()->recalculate_used_stable();
2829 }
2830 
2831 void CMSCollector::checkpointRootsInitialWork() {
2832   assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
2833   assert(_collectorState == InitialMarking, "just checking");
2834 
2835   // Already have locks.
2836   assert_lock_strong(bitMapLock());
2837   assert(_markBitMap.isAllClear(), "was reset at end of previous cycle");
2838 
2839   // Setup the verification and class unloading state for this
2840   // CMS collection cycle.
2841   setup_cms_unloading_and_verification_state();
2842 
2843   GCTraceTime(Trace, gc, phases) ts("checkpointRootsInitialWork", _gc_timer_cm);
2844 
2845   // Reset all the PLAB chunk arrays if necessary.
2846   if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) {
2847     reset_survivor_plab_arrays();
2848   }


4170   log_debug(gc)("YG occupancy: " SIZE_FORMAT " K (" SIZE_FORMAT " K)",
4171                 _young_gen->used() / K, _young_gen->capacity() / K);
4172   {
4173     if (CMSScavengeBeforeRemark) {
4174       CMSHeap* heap = CMSHeap::heap();
4175       // Temporarily set flag to false, GCH->do_collection will
4176       // expect it to be false and set to true
4177       FlagSetting fl(heap->_is_gc_active, false);
4178 
4179       heap->do_collection(true,                      // full (i.e. force, see below)
4180                           false,                     // !clear_all_soft_refs
4181                           0,                         // size
4182                           false,                     // is_tlab
4183                           GenCollectedHeap::YoungGen // type
4184         );
4185     }
4186     FreelistLocker x(this);
4187     MutexLocker y(bitMapLock(),
4188                   Mutex::_no_safepoint_check_flag);
4189     checkpointRootsFinalWork();
4190     _cmsGen->cmsSpace()->recalculate_used_stable();
4191   }
4192   verify_work_stacks_empty();
4193   verify_overflow_empty();
4194 }
4195 
4196 void CMSCollector::checkpointRootsFinalWork() {
4197   GCTraceTime(Trace, gc, phases) tm("checkpointRootsFinalWork", _gc_timer_cm);
4198 
4199   assert(haveFreelistLocks(), "must have free list locks");
4200   assert_lock_strong(bitMapLock());
4201 
4202   ResourceMark rm;
4203   HandleMark   hm;
4204 
4205   CMSHeap* heap = CMSHeap::heap();
4206 
4207   assert(haveFreelistLocks(), "must have free list locks");
4208   assert_lock_strong(bitMapLock());
4209 
4210   // We might assume that we need not fill TLAB's when


4244       GCTraceTime(Debug, gc, phases) t("Rescan (parallel)", _gc_timer_cm);
4245       do_remark_parallel();
4246     } else {
4247       GCTraceTime(Debug, gc, phases) t("Rescan (non-parallel)", _gc_timer_cm);
4248       do_remark_non_parallel();
4249     }
4250   }
4251   verify_work_stacks_empty();
4252   verify_overflow_empty();
4253 
4254   {
4255     GCTraceTime(Trace, gc, phases) ts("refProcessingWork", _gc_timer_cm);
4256     refProcessingWork();
4257   }
4258   verify_work_stacks_empty();
4259   verify_overflow_empty();
4260 
4261   if (should_unload_classes()) {
4262     heap->prune_scavengable_nmethods();
4263   }

4264 
4265   // If we encountered any (marking stack / work queue) overflow
4266   // events during the current CMS cycle, take appropriate
4267   // remedial measures, where possible, so as to try and avoid
4268   // recurrence of that condition.
4269   assert(_markStack.isEmpty(), "No grey objects");
4270   size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
4271                      _ser_kac_ovflw        + _ser_kac_preclean_ovflw;
4272   if (ser_ovflw > 0) {
4273     log_trace(gc)("Marking stack overflow (benign) (pmc_pc=" SIZE_FORMAT ", pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT ", kac_preclean=" SIZE_FORMAT ")",
4274                          _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw, _ser_kac_ovflw, _ser_kac_preclean_ovflw);
4275     _markStack.expand();
4276     _ser_pmc_remark_ovflw = 0;
4277     _ser_pmc_preclean_ovflw = 0;
4278     _ser_kac_preclean_ovflw = 0;
4279     _ser_kac_ovflw = 0;
4280   }
4281   if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
4282      log_trace(gc)("Work queue overflow (benign) (pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT ")",
4283                           _par_pmc_remark_ovflw, _par_kac_ovflw);


5330 
5331   assert(!_intra_sweep_timer.is_active(), "Should not be active");
5332   _intra_sweep_timer.reset();
5333   _intra_sweep_timer.start();
5334   {
5335     GCTraceCPUTime tcpu;
5336     CMSPhaseAccounting pa(this, "Concurrent Sweep");
5337     // First sweep the old gen
5338     {
5339       CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
5340                                bitMapLock());
5341       sweepWork(_cmsGen);
5342     }
5343 
5344     // Update Universe::_heap_*_at_gc figures.
5345     // We need all the free list locks to make the abstract state
5346     // transition from Sweeping to Resetting. See detailed note
5347     // further below.
5348     {
5349       CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock());
5350 
5351       // Update heap occupancy information which is used as
5352       // input to soft ref clearing policy at the next gc.
5353       Universe::update_heap_info_at_gc();
5354 
5355       // recalculate CMS used space after CMS collection
5356       _cmsGen->cmsSpace()->recalculate_used_stable();
5357 
5358       _collectorState = Resizing;
5359     }
5360   }
5361   verify_work_stacks_empty();
5362   verify_overflow_empty();
5363 
5364   if (should_unload_classes()) {
5365     // Delay purge to the beginning of the next safepoint.  Metaspace::contains
5366     // requires that the virtual spaces are stable and not deleted.
5367     ClassLoaderDataGraph::set_should_purge(true);
5368   }
5369 
5370   _intra_sweep_timer.stop();
5371   _intra_sweep_estimate.sample(_intra_sweep_timer.seconds());
5372 
5373   _inter_sweep_timer.reset();
5374   _inter_sweep_timer.start();
5375 
5376   // We need to use a monotonically non-decreasing time in ms
5377   // or we will see time-warp warnings and os::javaTimeMillis()


5426   _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
5427 }
5428 
5429 bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
5430   return addr >= _cmsSpace->nearLargestChunk();
5431 }
5432 
5433 FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
5434   return _cmsSpace->find_chunk_at_end();
5435 }
5436 
5437 void ConcurrentMarkSweepGeneration::update_gc_stats(Generation* current_generation,
5438                                                     bool full) {
5439   // If the young generation has been collected, gather any statistics
5440   // that are of interest at this point.
5441   bool current_is_young = CMSHeap::heap()->is_young_gen(current_generation);
5442   if (!full && current_is_young) {
5443     // Gather statistics on the young generation collection.
5444     collector()->stats().record_gc0_end(used());
5445   }
5446   _cmsSpace->recalculate_used_stable();
5447 }
5448 
5449 void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* old_gen) {
5450   // We iterate over the space(s) underlying this generation,
5451   // checking the mark bit map to see if the bits corresponding
5452   // to specific blocks are marked or not. Blocks that are
5453   // marked are live and are not swept up. All remaining blocks
5454   // are swept up, with coalescing on-the-fly as we sweep up
5455   // contiguous free and/or garbage blocks:
5456   // We need to ensure that the sweeper synchronizes with allocators
5457   // and stop-the-world collectors. In particular, the following
5458   // locks are used:
5459   // . CMS token: if this is held, a stop the world collection cannot occur
5460   // . freelistLock: if this is held no allocation can occur from this
5461   //                 generation by another thread
5462   // . bitMapLock: if this is held, no other thread can access or update
5463   //
5464 
5465   // Note that we need to hold the freelistLock if we use
5466   // block iterate below; else the iterator might go awry if


6969 void CMSPrecleanRefsYieldClosure::do_yield_work() {
6970   Mutex* bml = _collector->bitMapLock();
6971   assert_lock_strong(bml);
6972   assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6973          "CMS thread should hold CMS token");
6974 
6975   bml->unlock();
6976   ConcurrentMarkSweepThread::desynchronize(true);
6977 
6978   _collector->stopTimer();
6979   _collector->incrementYields();
6980 
6981   // See the comment in coordinator_yield()
6982   for (unsigned i = 0; i < CMSYieldSleepCount &&
6983                        ConcurrentMarkSweepThread::should_yield() &&
6984                        !CMSCollector::foregroundGCIsActive(); ++i) {
6985     os::sleep(Thread::current(), 1, false);
6986   }
6987 
6988   ConcurrentMarkSweepThread::synchronize(true);
6989   bml->lock_without_safepoint_check();
6990 
6991   _collector->startTimer();
6992 }
6993 
6994 bool CMSPrecleanRefsYieldClosure::should_return() {
6995   if (ConcurrentMarkSweepThread::should_yield()) {
6996     do_yield_work();
6997   }
6998   return _collector->foregroundGCIsActive();
6999 }
7000 
7001 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
7002   assert(((size_t)mr.start())%CardTable::card_size_in_words == 0,
7003          "mr should be aligned to start at a card boundary");
7004   // We'd like to assert:
7005   // assert(mr.word_size()%CardTable::card_size_in_words == 0,
7006   //        "mr should be a range of cards");
7007   // However, that would be too strong in one case -- the last
7008   // partition ends at _unallocated_block which, in general, can be
7009   // an arbitrary boundary, not necessarily card aligned.


7776 bool CMSCollector::simulate_overflow() {
7777   if (_overflow_counter-- <= 0) { // just being defensive
7778     _overflow_counter = CMSMarkStackOverflowInterval;
7779     return true;
7780   } else {
7781     return false;
7782   }
7783 }
7784 
7785 bool CMSCollector::par_simulate_overflow() {
7786   return simulate_overflow();
7787 }
7788 #endif
7789 
7790 // Single-threaded
7791 bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) {
7792   assert(stack->isEmpty(), "Expected precondition");
7793   assert(stack->capacity() > num, "Shouldn't bite more than can chew");
7794   size_t i = num;
7795   oop  cur = _overflow_list;
7796   const markWord proto = markWord::prototype();
7797   NOT_PRODUCT(ssize_t n = 0;)
7798   for (oop next; i > 0 && cur != NULL; cur = next, i--) {
7799     next = oop(cur->mark_raw().to_pointer());
7800     cur->set_mark_raw(proto);   // until proven otherwise
7801     assert(oopDesc::is_oop(cur), "Should be an oop");
7802     bool res = stack->push(cur);
7803     assert(res, "Bit off more than can chew?");
7804     NOT_PRODUCT(n++;)
7805   }
7806   _overflow_list = cur;
7807 #ifndef PRODUCT
7808   assert(_num_par_pushes >= n, "Too many pops?");
7809   _num_par_pushes -=n;
7810 #endif
7811   return !stack->isEmpty();
7812 }
7813 
7814 #define BUSY  (cast_to_oop<intptr_t>(0x1aff1aff))
7815 // (MT-safe) Get a prefix of at most "num" from the list.
7816 // The overflow list is chained through the mark word of
7817 // each object in the list. We fetch the entire list,
7818 // break off a prefix of the right size and return the
7819 // remainder. If other threads try to take objects from


7863     }
7864   }
7865   // If the list was found to be empty, or we spun long
7866   // enough, we give up and return empty-handed. If we leave
7867   // the list in the BUSY state below, it must be the case that
7868   // some other thread holds the overflow list and will set it
7869   // to a non-BUSY state in the future.
7870   if (prefix == NULL || prefix == BUSY) {
7871      // Nothing to take or waited long enough
7872      if (prefix == NULL) {
7873        // Write back the NULL in case we overwrote it with BUSY above
7874        // and it is still the same value.
7875        Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY);
7876      }
7877      return false;
7878   }
7879   assert(prefix != NULL && prefix != BUSY, "Error");
7880   size_t i = num;
7881   oop cur = prefix;
7882   // Walk down the first "num" objects, unless we reach the end.
7883   for (; i > 1 && cur->mark_raw().to_pointer() != NULL; cur = oop(cur->mark_raw().to_pointer()), i--);
7884   if (cur->mark_raw().to_pointer() == NULL) {
7885     // We have "num" or fewer elements in the list, so there
7886     // is nothing to return to the global list.
7887     // Write back the NULL in lieu of the BUSY we wrote
7888     // above, if it is still the same value.
7889     if (_overflow_list == BUSY) {
7890       Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY);
7891     }
7892   } else {
7893     // Chop off the suffix and return it to the global list.
7894     assert(cur->mark_raw().to_pointer() != (void*)BUSY, "Error");
7895     oop suffix_head = oop(cur->mark_raw().to_pointer()); // suffix will be put back on global list
7896     cur->set_mark_raw(markWord::from_pointer(NULL));     // break off suffix
7897     // It's possible that the list is still in the empty(busy) state
7898     // we left it in a short while ago; in that case we may be
7899     // able to place back the suffix without incurring the cost
7900     // of a walk down the list.
7901     oop observed_overflow_list = _overflow_list;
7902     oop cur_overflow_list = observed_overflow_list;
7903     bool attached = false;
7904     while (observed_overflow_list == BUSY || observed_overflow_list == NULL) {
7905       observed_overflow_list =
7906         Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list);
7907       if (cur_overflow_list == observed_overflow_list) {
7908         attached = true;
7909         break;
7910       } else cur_overflow_list = observed_overflow_list;
7911     }
7912     if (!attached) {
7913       // Too bad, someone else sneaked in (at least) an element; we'll need
7914       // to do a splice. Find tail of suffix so we can prepend suffix to global
7915       // list.
7916       for (cur = suffix_head; cur->mark_raw().to_pointer() != NULL; cur = (oop)(cur->mark_raw().to_pointer()));
7917       oop suffix_tail = cur;
7918       assert(suffix_tail != NULL && suffix_tail->mark_raw().to_pointer() == NULL,
7919              "Tautology");
7920       observed_overflow_list = _overflow_list;
7921       do {
7922         cur_overflow_list = observed_overflow_list;
7923         if (cur_overflow_list != BUSY) {
7924           // Do the splice ...
7925           suffix_tail->set_mark_raw(markWord::from_pointer((void*)cur_overflow_list));
7926         } else { // cur_overflow_list == BUSY
7927           suffix_tail->set_mark_raw(markWord::from_pointer(NULL));
7928         }
7929         // ... and try to place spliced list back on overflow_list ...
7930         observed_overflow_list =
7931           Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list);
7932       } while (cur_overflow_list != observed_overflow_list);
7933       // ... until we have succeeded in doing so.
7934     }
7935   }
7936 
7937   // Push the prefix elements on work_q
7938   assert(prefix != NULL, "control point invariant");
7939   const markWord proto = markWord::prototype();
7940   oop next;
7941   NOT_PRODUCT(ssize_t n = 0;)
7942   for (cur = prefix; cur != NULL; cur = next) {
7943     next = oop(cur->mark_raw().to_pointer());
7944     cur->set_mark_raw(proto);   // until proven otherwise
7945     assert(oopDesc::is_oop(cur), "Should be an oop");
7946     bool res = work_q->push(cur);
7947     assert(res, "Bit off more than we can chew?");
7948     NOT_PRODUCT(n++;)
7949   }
7950 #ifndef PRODUCT
7951   assert(_num_par_pushes >= n, "Too many pops?");
7952   Atomic::sub(n, &_num_par_pushes);
7953 #endif
7954   return true;
7955 }
7956 
7957 // Single-threaded
7958 void CMSCollector::push_on_overflow_list(oop p) {
7959   NOT_PRODUCT(_num_par_pushes++;)
7960   assert(oopDesc::is_oop(p), "Not an oop");
7961   preserve_mark_if_necessary(p);
7962   p->set_mark_raw(markWord::from_pointer(_overflow_list));
7963   _overflow_list = p;
7964 }
7965 
7966 // Multi-threaded; use CAS to prepend to overflow list
7967 void CMSCollector::par_push_on_overflow_list(oop p) {
7968   NOT_PRODUCT(Atomic::inc(&_num_par_pushes);)
7969   assert(oopDesc::is_oop(p), "Not an oop");
7970   par_preserve_mark_if_necessary(p);
7971   oop observed_overflow_list = _overflow_list;
7972   oop cur_overflow_list;
7973   do {
7974     cur_overflow_list = observed_overflow_list;
7975     if (cur_overflow_list != BUSY) {
7976       p->set_mark_raw(markWord::from_pointer((void*)cur_overflow_list));
7977     } else {
7978       p->set_mark_raw(markWord::from_pointer(NULL));
7979     }
7980     observed_overflow_list =
7981       Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list);
7982   } while (cur_overflow_list != observed_overflow_list);
7983 }
7984 #undef BUSY
7985 
7986 // Single threaded
7987 // General Note on GrowableArray: pushes may silently fail
7988 // because we are (temporarily) out of C-heap for expanding
7989 // the stack. The problem is quite ubiquitous and affects
7990 // a lot of code in the JVM. The prudent thing for GrowableArray
7991 // to do (for now) is to exit with an error. However, that may
7992 // be too draconian in some cases because the caller may be
7993 // able to recover without much harm. For such cases, we
7994 // should probably introduce a "soft_push" method which returns
7995 // an indication of success or failure with the assumption that
7996 // the caller may be able to recover from a failure; code in
7997 // the VM can then be changed, incrementally, to deal with such
7998 // failures where possible, thus, incrementally hardening the VM
7999 // in such low resource situations.
8000 void CMSCollector::preserve_mark_work(oop p, markWord m) {
8001   _preserved_oop_stack.push(p);
8002   _preserved_mark_stack.push(m);
8003   assert(m == p->mark_raw(), "Mark word changed");
8004   assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
8005          "bijection");
8006 }
8007 
8008 // Single threaded
8009 void CMSCollector::preserve_mark_if_necessary(oop p) {
8010   markWord m = p->mark_raw();
8011   if (p->mark_must_be_preserved(m)) {
8012     preserve_mark_work(p, m);
8013   }
8014 }
8015 
8016 void CMSCollector::par_preserve_mark_if_necessary(oop p) {
8017   markWord m = p->mark_raw();
8018   if (p->mark_must_be_preserved(m)) {
8019     MutexLocker x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
8020     // Even though we read the mark word without holding
8021     // the lock, we are assured that it will not change
8022     // because we "own" this oop, so no other thread can
8023     // be trying to push it on the overflow list; see
8024     // the assertion in preserve_mark_work() that checks
8025     // that m == p->mark_raw().
8026     preserve_mark_work(p, m);
8027   }
8028 }
8029 
8030 // We should be able to do this multi-threaded,
8031 // a chunk of stack being a task (this is
8032 // correct because each oop only ever appears
8033 // once in the overflow list. However, it's
8034 // not very easy to completely overlap this with
8035 // other operations, so will generally not be done
8036 // until all work's been completed. Because we
8037 // expect the preserved oop stack (set) to be small,
8038 // it's probably fine to do this single-threaded.
8039 // We can explore cleverer concurrent/overlapped/parallel
8040 // processing of preserved marks if we feel the
8041 // need for this in the future. Stack overflow should
8042 // be so rare in practice and, when it happens, its
8043 // effect on performance so great that this will
8044 // likely just be in the noise anyway.
8045 void CMSCollector::restore_preserved_marks_if_any() {
8046   assert(SafepointSynchronize::is_at_safepoint(),
8047          "world should be stopped");
8048   assert(Thread::current()->is_ConcurrentGC_thread() ||
8049          Thread::current()->is_VM_thread(),
8050          "should be single-threaded");
8051   assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
8052          "bijection");
8053 
8054   while (!_preserved_oop_stack.is_empty()) {
8055     oop p = _preserved_oop_stack.pop();
8056     assert(oopDesc::is_oop(p), "Should be an oop");
8057     assert(_span.contains(p), "oop should be in _span");
8058     assert(p->mark_raw() == markWord::prototype(),
8059            "Set when taken from overflow list");
8060     markWord m = _preserved_mark_stack.pop();
8061     p->set_mark_raw(m);
8062   }
8063   assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(),
8064          "stacks were cleared above");
8065 }
8066 
8067 #ifndef PRODUCT
8068 bool CMSCollector::no_preserved_marks() const {
8069   return _preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty();
8070 }
8071 #endif
8072 
8073 // Transfer some number of overflown objects to usual marking
8074 // stack. Return true if some objects were transferred.
8075 bool MarkRefsIntoAndScanClosure::take_from_overflow_list() {
8076   size_t num = MIN2((size_t)(_mark_stack->capacity() - _mark_stack->length())/4,
8077                     (size_t)ParGCDesiredObjsFromOverflowList);
8078 
8079   bool res = _collector->take_from_overflow_list(num, _mark_stack);
8080   assert(_collector->overflow_list_is_empty() || res,




 675 void ConcurrentMarkSweepGeneration::print_statistics() {
 676   cmsSpace()->printFLCensus(0);
 677 }
 678 #endif
 679 
 680 size_t
 681 ConcurrentMarkSweepGeneration::contiguous_available() const {
 682   // dld proposes an improvement in precision here. If the committed
 683   // part of the space ends in a free block we should add that to
 684   // uncommitted size in the calculation below. Will make this
 685   // change later, staying with the approximation below for the
 686   // time being. -- ysr.
 687   return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
 688 }
 689 
 690 size_t
 691 ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {
 692   return _cmsSpace->max_alloc_in_words() * HeapWordSize;
 693 }
 694 




 695 size_t ConcurrentMarkSweepGeneration::max_available() const {
 696   return free() + _virtual_space.uncommitted_size();
 697 }
 698 
 699 bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
 700   size_t available = max_available();
 701   size_t av_promo  = (size_t)gc_stats()->avg_promoted()->padded_average();
 702   bool   res = (available >= av_promo) || (available >= max_promotion_in_bytes);
 703   log_trace(gc, promotion)("CMS: promo attempt is%s safe: available(" SIZE_FORMAT ") %s av_promo(" SIZE_FORMAT "), max_promo(" SIZE_FORMAT ")",
 704                            res? "":" not", available, res? ">=":"<", av_promo, max_promotion_in_bytes);
 705   return res;
 706 }
 707 
 708 // At a promotion failure dump information on block layout in heap
 709 // (cms old generation).
 710 void ConcurrentMarkSweepGeneration::promotion_failure_occurred() {
 711   Log(gc, promotion) log;
 712   if (log.is_trace()) {
 713     LogStream ls(log.trace());
 714     cmsSpace()->dump_at_safepoint_with_locks(collector(), &ls);


 993 // TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT
 994 //
 995 //
 996 // STATE TRANSITION DIAGRAM
 997 //
 998 //        mut / parnew                     mut  /  parnew
 999 // FREE --------------------> TRANSIENT ---------------------> OBJECT --|
1000 //  ^                                                                   |
1001 //  |------------------------ DEAD <------------------------------------|
1002 //         sweep                            mut
1003 //
1004 // While a block is in TRANSIENT state its size cannot be determined
1005 // so readers will either need to come back later or stall until
1006 // the size can be determined. Note that for the case of direct
1007 // allocation, P-bits, when available, may be used to determine the
1008 // size of an object that may not yet have been initialized.
1009 
1010 // Things to support parallel young-gen collection.
1011 oop
1012 ConcurrentMarkSweepGeneration::par_promote(int thread_num,
1013                                            oop old, markOop m,
1014                                            size_t word_sz) {
1015 #ifndef PRODUCT
1016   if (CMSHeap::heap()->promotion_should_fail()) {
1017     return NULL;
1018   }
1019 #endif  // #ifndef PRODUCT
1020 
1021   CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1022   PromotionInfo* promoInfo = &ps->promo;
1023   // if we are tracking promotions, then first ensure space for
1024   // promotion (including spooling space for saving header if necessary).
1025   // then allocate and copy, then track promoted info if needed.
1026   // When tracking (see PromotionInfo::track()), the mark word may
1027   // be displaced and in this case restoration of the mark word
1028   // occurs in the (oop_since_save_marks_)iterate phase.
1029   if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) {
1030     // Out of space for allocating spooling buffers;
1031     // try expanding and allocating spooling buffers.
1032     if (!expand_and_ensure_spooling_space(promoInfo)) {
1033       return NULL;


1506                                          max_eden_size,
1507                                          full,
1508                                          gc_cause,
1509                                          heap->soft_ref_policy());
1510 
1511   // Reset the expansion cause, now that we just completed
1512   // a collection cycle.
1513   clear_expansion_cause();
1514   _foregroundGCIsActive = false;
1515   return;
1516 }
1517 
1518 // Resize the tenured generation
1519 // after obtaining the free list locks for the
1520 // two generations.
1521 void CMSCollector::compute_new_size() {
1522   assert_locked_or_safepoint(Heap_lock);
1523   FreelistLocker z(this);
1524   MetaspaceGC::compute_new_size();
1525   _cmsGen->compute_new_size_free_list();


1526 }
1527 
1528 // A work method used by the foreground collector to do
1529 // a mark-sweep-compact.
1530 void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
1531   CMSHeap* heap = CMSHeap::heap();
1532 
1533   STWGCTimer* gc_timer = GenMarkSweep::gc_timer();
1534   gc_timer->register_gc_start();
1535 
1536   SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer();
1537   gc_tracer->report_gc_start(heap->gc_cause(), gc_timer->gc_start());
1538 
1539   heap->pre_full_gc_dump(gc_timer);
1540 
1541   GCTraceTime(Trace, gc, phases) t("CMS:MSC");
1542 
1543   // Temporarily widen the span of the weak reference processing to
1544   // the entire heap.
1545   MemRegion new_span(CMSHeap::heap()->reserved_region());


2034   // there are any modified oops in the class. The remark phase also needs
2035   // that information. Tell the young collection to save the union of all
2036   // modified klasses.
2037   if (duringMarking) {
2038     _ct->cld_rem_set()->set_accumulate_modified_oops(true);
2039   }
2040 
2041   bool registerClosure = duringMarking;
2042 
2043   _cmsGen->gc_prologue_work(full, registerClosure, &_modUnionClosurePar);
2044 
2045   if (!full) {
2046     stats().record_gc0_begin();
2047   }
2048 }
2049 
2050 void ConcurrentMarkSweepGeneration::gc_prologue(bool full) {
2051 
2052   _capacity_at_prologue = capacity();
2053   _used_at_prologue = used();

2054 
2055   // We enable promotion tracking so that card-scanning can recognize
2056   // which objects have been promoted during this GC and skip them.
2057   for (uint i = 0; i < ParallelGCThreads; i++) {
2058     _par_gc_thread_states[i]->promo.startTrackingPromotions();
2059   }
2060 
2061   // Delegate to CMScollector which knows how to coordinate between
2062   // this and any other CMS generations that it is responsible for
2063   // collecting.
2064   collector()->gc_prologue(full);
2065 }
2066 
2067 // This is a "private" interface for use by this generation's CMSCollector.
2068 // Not to be called directly by any other entity (for instance,
2069 // GenCollectedHeap, which calls the "public" gc_prologue method above).
2070 void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,
2071   bool registerClosure, ModUnionClosure* modUnionClosure) {
2072   assert(!incremental_collection_failed(), "Shouldn't be set yet");
2073   assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL,


2106     // We have already been invoked; this is a gc_epilogue delegation
2107     // from yet another CMS generation that we are responsible for, just
2108     // ignore it since all relevant work has already been done.
2109     return;
2110   }
2111   assert(haveFreelistLocks(), "must have freelist locks");
2112   assert_lock_strong(bitMapLock());
2113 
2114   _ct->cld_rem_set()->set_accumulate_modified_oops(false);
2115 
2116   _cmsGen->gc_epilogue_work(full);
2117 
2118   if (_collectorState == AbortablePreclean || _collectorState == Precleaning) {
2119     // in case sampling was not already enabled, enable it
2120     _start_sampling = true;
2121   }
2122   // reset _eden_chunk_array so sampling starts afresh
2123   _eden_chunk_index = 0;
2124 
2125   size_t cms_used   = _cmsGen->cmsSpace()->used();

2126 
2127   // update performance counters - this uses a special version of
2128   // update_counters() that allows the utilization to be passed as a
2129   // parameter, avoiding multiple calls to used().
2130   //
2131   _cmsGen->update_counters(cms_used);
2132 
2133   bitMapLock()->unlock();
2134   releaseFreelistLocks();
2135 
2136   if (!CleanChunkPoolAsync) {
2137     Chunk::clean_chunk_pool();
2138   }
2139 
2140   set_did_compact(false);
2141   _between_prologue_and_epilogue = false;  // ready for next cycle
2142 }
2143 
2144 void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) {
2145   collector()->gc_epilogue(full);


2799 // be approximate -- we'll do a catch up phase subsequently.]
2800 void CMSCollector::checkpointRootsInitial() {
2801   assert(_collectorState == InitialMarking, "Wrong collector state");
2802   check_correct_thread_executing();
2803   TraceCMSMemoryManagerStats tms(_collectorState, CMSHeap::heap()->gc_cause());
2804 
2805   save_heap_summary();
2806   report_heap_summary(GCWhen::BeforeGC);
2807 
2808   ReferenceProcessor* rp = ref_processor();
2809   assert(_restart_addr == NULL, "Control point invariant");
2810   {
2811     // acquire locks for subsequent manipulations
2812     MutexLocker x(bitMapLock(),
2813                   Mutex::_no_safepoint_check_flag);
2814     checkpointRootsInitialWork();
2815     // enable ("weak") refs discovery
2816     rp->enable_discovery();
2817     _collectorState = Marking;
2818   }


2819 }
2820 
2821 void CMSCollector::checkpointRootsInitialWork() {
2822   assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
2823   assert(_collectorState == InitialMarking, "just checking");
2824 
2825   // Already have locks.
2826   assert_lock_strong(bitMapLock());
2827   assert(_markBitMap.isAllClear(), "was reset at end of previous cycle");
2828 
2829   // Setup the verification and class unloading state for this
2830   // CMS collection cycle.
2831   setup_cms_unloading_and_verification_state();
2832 
2833   GCTraceTime(Trace, gc, phases) ts("checkpointRootsInitialWork", _gc_timer_cm);
2834 
2835   // Reset all the PLAB chunk arrays if necessary.
2836   if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) {
2837     reset_survivor_plab_arrays();
2838   }


4160   log_debug(gc)("YG occupancy: " SIZE_FORMAT " K (" SIZE_FORMAT " K)",
4161                 _young_gen->used() / K, _young_gen->capacity() / K);
4162   {
4163     if (CMSScavengeBeforeRemark) {
4164       CMSHeap* heap = CMSHeap::heap();
4165       // Temporarily set flag to false, GCH->do_collection will
4166       // expect it to be false and set to true
4167       FlagSetting fl(heap->_is_gc_active, false);
4168 
4169       heap->do_collection(true,                      // full (i.e. force, see below)
4170                           false,                     // !clear_all_soft_refs
4171                           0,                         // size
4172                           false,                     // is_tlab
4173                           GenCollectedHeap::YoungGen // type
4174         );
4175     }
4176     FreelistLocker x(this);
4177     MutexLocker y(bitMapLock(),
4178                   Mutex::_no_safepoint_check_flag);
4179     checkpointRootsFinalWork();

4180   }
4181   verify_work_stacks_empty();
4182   verify_overflow_empty();
4183 }
4184 
4185 void CMSCollector::checkpointRootsFinalWork() {
4186   GCTraceTime(Trace, gc, phases) tm("checkpointRootsFinalWork", _gc_timer_cm);
4187 
4188   assert(haveFreelistLocks(), "must have free list locks");
4189   assert_lock_strong(bitMapLock());
4190 
4191   ResourceMark rm;
4192   HandleMark   hm;
4193 
4194   CMSHeap* heap = CMSHeap::heap();
4195 
4196   assert(haveFreelistLocks(), "must have free list locks");
4197   assert_lock_strong(bitMapLock());
4198 
4199   // We might assume that we need not fill TLAB's when


4233       GCTraceTime(Debug, gc, phases) t("Rescan (parallel)", _gc_timer_cm);
4234       do_remark_parallel();
4235     } else {
4236       GCTraceTime(Debug, gc, phases) t("Rescan (non-parallel)", _gc_timer_cm);
4237       do_remark_non_parallel();
4238     }
4239   }
4240   verify_work_stacks_empty();
4241   verify_overflow_empty();
4242 
4243   {
4244     GCTraceTime(Trace, gc, phases) ts("refProcessingWork", _gc_timer_cm);
4245     refProcessingWork();
4246   }
4247   verify_work_stacks_empty();
4248   verify_overflow_empty();
4249 
4250   if (should_unload_classes()) {
4251     heap->prune_scavengable_nmethods();
4252   }
4253   JvmtiExport::gc_epilogue();
4254 
4255   // If we encountered any (marking stack / work queue) overflow
4256   // events during the current CMS cycle, take appropriate
4257   // remedial measures, where possible, so as to try and avoid
4258   // recurrence of that condition.
4259   assert(_markStack.isEmpty(), "No grey objects");
4260   size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
4261                      _ser_kac_ovflw        + _ser_kac_preclean_ovflw;
4262   if (ser_ovflw > 0) {
4263     log_trace(gc)("Marking stack overflow (benign) (pmc_pc=" SIZE_FORMAT ", pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT ", kac_preclean=" SIZE_FORMAT ")",
4264                          _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw, _ser_kac_ovflw, _ser_kac_preclean_ovflw);
4265     _markStack.expand();
4266     _ser_pmc_remark_ovflw = 0;
4267     _ser_pmc_preclean_ovflw = 0;
4268     _ser_kac_preclean_ovflw = 0;
4269     _ser_kac_ovflw = 0;
4270   }
4271   if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
4272      log_trace(gc)("Work queue overflow (benign) (pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT ")",
4273                           _par_pmc_remark_ovflw, _par_kac_ovflw);


5320 
5321   assert(!_intra_sweep_timer.is_active(), "Should not be active");
5322   _intra_sweep_timer.reset();
5323   _intra_sweep_timer.start();
5324   {
5325     GCTraceCPUTime tcpu;
5326     CMSPhaseAccounting pa(this, "Concurrent Sweep");
5327     // First sweep the old gen
5328     {
5329       CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
5330                                bitMapLock());
5331       sweepWork(_cmsGen);
5332     }
5333 
5334     // Update Universe::_heap_*_at_gc figures.
5335     // We need all the free list locks to make the abstract state
5336     // transition from Sweeping to Resetting. See detailed note
5337     // further below.
5338     {
5339       CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock());

5340       // Update heap occupancy information which is used as
5341       // input to soft ref clearing policy at the next gc.
5342       Universe::update_heap_info_at_gc();




5343       _collectorState = Resizing;
5344     }
5345   }
5346   verify_work_stacks_empty();
5347   verify_overflow_empty();
5348 
5349   if (should_unload_classes()) {
5350     // Delay purge to the beginning of the next safepoint.  Metaspace::contains
5351     // requires that the virtual spaces are stable and not deleted.
5352     ClassLoaderDataGraph::set_should_purge(true);
5353   }
5354 
5355   _intra_sweep_timer.stop();
5356   _intra_sweep_estimate.sample(_intra_sweep_timer.seconds());
5357 
5358   _inter_sweep_timer.reset();
5359   _inter_sweep_timer.start();
5360 
5361   // We need to use a monotonically non-decreasing time in ms
5362   // or we will see time-warp warnings and os::javaTimeMillis()


5411   _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
5412 }
5413 
5414 bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
5415   return addr >= _cmsSpace->nearLargestChunk();
5416 }
5417 
5418 FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
5419   return _cmsSpace->find_chunk_at_end();
5420 }
5421 
5422 void ConcurrentMarkSweepGeneration::update_gc_stats(Generation* current_generation,
5423                                                     bool full) {
5424   // If the young generation has been collected, gather any statistics
5425   // that are of interest at this point.
5426   bool current_is_young = CMSHeap::heap()->is_young_gen(current_generation);
5427   if (!full && current_is_young) {
5428     // Gather statistics on the young generation collection.
5429     collector()->stats().record_gc0_end(used());
5430   }

5431 }
5432 
5433 void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* old_gen) {
5434   // We iterate over the space(s) underlying this generation,
5435   // checking the mark bit map to see if the bits corresponding
5436   // to specific blocks are marked or not. Blocks that are
5437   // marked are live and are not swept up. All remaining blocks
5438   // are swept up, with coalescing on-the-fly as we sweep up
5439   // contiguous free and/or garbage blocks:
5440   // We need to ensure that the sweeper synchronizes with allocators
5441   // and stop-the-world collectors. In particular, the following
5442   // locks are used:
5443   // . CMS token: if this is held, a stop the world collection cannot occur
5444   // . freelistLock: if this is held no allocation can occur from this
5445   //                 generation by another thread
5446   // . bitMapLock: if this is held, no other thread can access or update
5447   //
5448 
5449   // Note that we need to hold the freelistLock if we use
5450   // block iterate below; else the iterator might go awry if


6953 void CMSPrecleanRefsYieldClosure::do_yield_work() {
6954   Mutex* bml = _collector->bitMapLock();
6955   assert_lock_strong(bml);
6956   assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6957          "CMS thread should hold CMS token");
6958 
6959   bml->unlock();
6960   ConcurrentMarkSweepThread::desynchronize(true);
6961 
6962   _collector->stopTimer();
6963   _collector->incrementYields();
6964 
6965   // See the comment in coordinator_yield()
6966   for (unsigned i = 0; i < CMSYieldSleepCount &&
6967                        ConcurrentMarkSweepThread::should_yield() &&
6968                        !CMSCollector::foregroundGCIsActive(); ++i) {
6969     os::sleep(Thread::current(), 1, false);
6970   }
6971 
6972   ConcurrentMarkSweepThread::synchronize(true);
6973   bml->lock();
6974 
6975   _collector->startTimer();
6976 }
6977 
6978 bool CMSPrecleanRefsYieldClosure::should_return() {
6979   if (ConcurrentMarkSweepThread::should_yield()) {
6980     do_yield_work();
6981   }
6982   return _collector->foregroundGCIsActive();
6983 }
6984 
6985 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
6986   assert(((size_t)mr.start())%CardTable::card_size_in_words == 0,
6987          "mr should be aligned to start at a card boundary");
6988   // We'd like to assert:
6989   // assert(mr.word_size()%CardTable::card_size_in_words == 0,
6990   //        "mr should be a range of cards");
6991   // However, that would be too strong in one case -- the last
6992   // partition ends at _unallocated_block which, in general, can be
6993   // an arbitrary boundary, not necessarily card aligned.


7760 bool CMSCollector::simulate_overflow() {
7761   if (_overflow_counter-- <= 0) { // just being defensive
7762     _overflow_counter = CMSMarkStackOverflowInterval;
7763     return true;
7764   } else {
7765     return false;
7766   }
7767 }
7768 
7769 bool CMSCollector::par_simulate_overflow() {
7770   return simulate_overflow();
7771 }
7772 #endif
7773 
7774 // Single-threaded
7775 bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) {
7776   assert(stack->isEmpty(), "Expected precondition");
7777   assert(stack->capacity() > num, "Shouldn't bite more than can chew");
7778   size_t i = num;
7779   oop  cur = _overflow_list;
7780   const markOop proto = markOopDesc::prototype();
7781   NOT_PRODUCT(ssize_t n = 0;)
7782   for (oop next; i > 0 && cur != NULL; cur = next, i--) {
7783     next = oop(cur->mark_raw());
7784     cur->set_mark_raw(proto);   // until proven otherwise
7785     assert(oopDesc::is_oop(cur), "Should be an oop");
7786     bool res = stack->push(cur);
7787     assert(res, "Bit off more than can chew?");
7788     NOT_PRODUCT(n++;)
7789   }
7790   _overflow_list = cur;
7791 #ifndef PRODUCT
7792   assert(_num_par_pushes >= n, "Too many pops?");
7793   _num_par_pushes -=n;
7794 #endif
7795   return !stack->isEmpty();
7796 }
7797 
7798 #define BUSY  (cast_to_oop<intptr_t>(0x1aff1aff))
7799 // (MT-safe) Get a prefix of at most "num" from the list.
7800 // The overflow list is chained through the mark word of
7801 // each object in the list. We fetch the entire list,
7802 // break off a prefix of the right size and return the
7803 // remainder. If other threads try to take objects from


7847     }
7848   }
7849   // If the list was found to be empty, or we spun long
7850   // enough, we give up and return empty-handed. If we leave
7851   // the list in the BUSY state below, it must be the case that
7852   // some other thread holds the overflow list and will set it
7853   // to a non-BUSY state in the future.
7854   if (prefix == NULL || prefix == BUSY) {
7855      // Nothing to take or waited long enough
7856      if (prefix == NULL) {
7857        // Write back the NULL in case we overwrote it with BUSY above
7858        // and it is still the same value.
7859        Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY);
7860      }
7861      return false;
7862   }
7863   assert(prefix != NULL && prefix != BUSY, "Error");
7864   size_t i = num;
7865   oop cur = prefix;
7866   // Walk down the first "num" objects, unless we reach the end.
7867   for (; i > 1 && cur->mark_raw() != NULL; cur = oop(cur->mark_raw()), i--);
7868   if (cur->mark_raw() == NULL) {
7869     // We have "num" or fewer elements in the list, so there
7870     // is nothing to return to the global list.
7871     // Write back the NULL in lieu of the BUSY we wrote
7872     // above, if it is still the same value.
7873     if (_overflow_list == BUSY) {
7874       Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY);
7875     }
7876   } else {
7877     // Chop off the suffix and return it to the global list.
7878     assert(cur->mark_raw() != BUSY, "Error");
7879     oop suffix_head = cur->mark_raw(); // suffix will be put back on global list
7880     cur->set_mark_raw(NULL);           // break off suffix
7881     // It's possible that the list is still in the empty(busy) state
7882     // we left it in a short while ago; in that case we may be
7883     // able to place back the suffix without incurring the cost
7884     // of a walk down the list.
7885     oop observed_overflow_list = _overflow_list;
7886     oop cur_overflow_list = observed_overflow_list;
7887     bool attached = false;
7888     while (observed_overflow_list == BUSY || observed_overflow_list == NULL) {
7889       observed_overflow_list =
7890         Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list);
7891       if (cur_overflow_list == observed_overflow_list) {
7892         attached = true;
7893         break;
7894       } else cur_overflow_list = observed_overflow_list;
7895     }
7896     if (!attached) {
7897       // Too bad, someone else sneaked in (at least) an element; we'll need
7898       // to do a splice. Find tail of suffix so we can prepend suffix to global
7899       // list.
7900       for (cur = suffix_head; cur->mark_raw() != NULL; cur = (oop)(cur->mark_raw()));
7901       oop suffix_tail = cur;
7902       assert(suffix_tail != NULL && suffix_tail->mark_raw() == NULL,
7903              "Tautology");
7904       observed_overflow_list = _overflow_list;
7905       do {
7906         cur_overflow_list = observed_overflow_list;
7907         if (cur_overflow_list != BUSY) {
7908           // Do the splice ...
7909           suffix_tail->set_mark_raw(markOop(cur_overflow_list));
7910         } else { // cur_overflow_list == BUSY
7911           suffix_tail->set_mark_raw(NULL);
7912         }
7913         // ... and try to place spliced list back on overflow_list ...
7914         observed_overflow_list =
7915           Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list);
7916       } while (cur_overflow_list != observed_overflow_list);
7917       // ... until we have succeeded in doing so.
7918     }
7919   }
7920 
7921   // Push the prefix elements on work_q
7922   assert(prefix != NULL, "control point invariant");
7923   const markOop proto = markOopDesc::prototype();
7924   oop next;
7925   NOT_PRODUCT(ssize_t n = 0;)
7926   for (cur = prefix; cur != NULL; cur = next) {
7927     next = oop(cur->mark_raw());
7928     cur->set_mark_raw(proto);   // until proven otherwise
7929     assert(oopDesc::is_oop(cur), "Should be an oop");
7930     bool res = work_q->push(cur);
7931     assert(res, "Bit off more than we can chew?");
7932     NOT_PRODUCT(n++;)
7933   }
7934 #ifndef PRODUCT
7935   assert(_num_par_pushes >= n, "Too many pops?");
7936   Atomic::sub(n, &_num_par_pushes);
7937 #endif
7938   return true;
7939 }
7940 
7941 // Single-threaded
7942 void CMSCollector::push_on_overflow_list(oop p) {
7943   NOT_PRODUCT(_num_par_pushes++;)
7944   assert(oopDesc::is_oop(p), "Not an oop");
7945   preserve_mark_if_necessary(p);
7946   p->set_mark_raw((markOop)_overflow_list);
7947   _overflow_list = p;
7948 }
7949 
7950 // Multi-threaded; use CAS to prepend to overflow list
7951 void CMSCollector::par_push_on_overflow_list(oop p) {
7952   NOT_PRODUCT(Atomic::inc(&_num_par_pushes);)
7953   assert(oopDesc::is_oop(p), "Not an oop");
7954   par_preserve_mark_if_necessary(p);
7955   oop observed_overflow_list = _overflow_list;
7956   oop cur_overflow_list;
7957   do {
7958     cur_overflow_list = observed_overflow_list;
7959     if (cur_overflow_list != BUSY) {
7960       p->set_mark_raw(markOop(cur_overflow_list));
7961     } else {
7962       p->set_mark_raw(NULL);
7963     }
7964     observed_overflow_list =
7965       Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list);
7966   } while (cur_overflow_list != observed_overflow_list);
7967 }
7968 #undef BUSY
7969 
7970 // Single threaded
7971 // General Note on GrowableArray: pushes may silently fail
7972 // because we are (temporarily) out of C-heap for expanding
7973 // the stack. The problem is quite ubiquitous and affects
7974 // a lot of code in the JVM. The prudent thing for GrowableArray
7975 // to do (for now) is to exit with an error. However, that may
7976 // be too draconian in some cases because the caller may be
7977 // able to recover without much harm. For such cases, we
7978 // should probably introduce a "soft_push" method which returns
7979 // an indication of success or failure with the assumption that
7980 // the caller may be able to recover from a failure; code in
7981 // the VM can then be changed, incrementally, to deal with such
7982 // failures where possible, thus, incrementally hardening the VM
7983 // in such low resource situations.
7984 void CMSCollector::preserve_mark_work(oop p, markOop m) {
7985   _preserved_oop_stack.push(p);
7986   _preserved_mark_stack.push(m);
7987   assert(m == p->mark_raw(), "Mark word changed");
7988   assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
7989          "bijection");
7990 }
7991 
7992 // Single threaded
7993 void CMSCollector::preserve_mark_if_necessary(oop p) {
7994   markOop m = p->mark_raw();
7995   if (m->must_be_preserved(p)) {
7996     preserve_mark_work(p, m);
7997   }
7998 }
7999 
8000 void CMSCollector::par_preserve_mark_if_necessary(oop p) {
8001   markOop m = p->mark_raw();
8002   if (m->must_be_preserved(p)) {
8003     MutexLocker x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
8004     // Even though we read the mark word without holding
8005     // the lock, we are assured that it will not change
8006     // because we "own" this oop, so no other thread can
8007     // be trying to push it on the overflow list; see
8008     // the assertion in preserve_mark_work() that checks
8009     // that m == p->mark_raw().
8010     preserve_mark_work(p, m);
8011   }
8012 }
8013 
8014 // We should be able to do this multi-threaded,
8015 // a chunk of stack being a task (this is
8016 // correct because each oop only ever appears
8017 // once in the overflow list. However, it's
8018 // not very easy to completely overlap this with
8019 // other operations, so will generally not be done
8020 // until all work's been completed. Because we
8021 // expect the preserved oop stack (set) to be small,
8022 // it's probably fine to do this single-threaded.
8023 // We can explore cleverer concurrent/overlapped/parallel
8024 // processing of preserved marks if we feel the
8025 // need for this in the future. Stack overflow should
8026 // be so rare in practice and, when it happens, its
8027 // effect on performance so great that this will
8028 // likely just be in the noise anyway.
8029 void CMSCollector::restore_preserved_marks_if_any() {
8030   assert(SafepointSynchronize::is_at_safepoint(),
8031          "world should be stopped");
8032   assert(Thread::current()->is_ConcurrentGC_thread() ||
8033          Thread::current()->is_VM_thread(),
8034          "should be single-threaded");
8035   assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
8036          "bijection");
8037 
8038   while (!_preserved_oop_stack.is_empty()) {
8039     oop p = _preserved_oop_stack.pop();
8040     assert(oopDesc::is_oop(p), "Should be an oop");
8041     assert(_span.contains(p), "oop should be in _span");
8042     assert(p->mark_raw() == markOopDesc::prototype(),
8043            "Set when taken from overflow list");
8044     markOop m = _preserved_mark_stack.pop();
8045     p->set_mark_raw(m);
8046   }
8047   assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(),
8048          "stacks were cleared above");
8049 }
8050 
8051 #ifndef PRODUCT
8052 bool CMSCollector::no_preserved_marks() const {
8053   return _preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty();
8054 }
8055 #endif
8056 
8057 // Transfer some number of overflown objects to usual marking
8058 // stack. Return true if some objects were transferred.
8059 bool MarkRefsIntoAndScanClosure::take_from_overflow_list() {
8060   size_t num = MIN2((size_t)(_mark_stack->capacity() - _mark_stack->length())/4,
8061                     (size_t)ParGCDesiredObjsFromOverflowList);
8062 
8063   bool res = _collector->take_from_overflow_list(num, _mark_stack);
8064   assert(_collector->overflow_list_is_empty() || res,


< prev index next >