< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp

Print this page

  1 /*
  2  * Copyright (c) 2015, 2020, Red Hat, Inc. All rights reserved.

  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
 26 #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
 27 
 28 #include "gc/shenandoah/shenandoahHeap.hpp"
 29 
 30 #include "classfile/javaClasses.inline.hpp"
 31 #include "gc/shared/markBitMap.inline.hpp"
 32 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
 33 #include "gc/shared/continuationGCSupport.inline.hpp"
 34 #include "gc/shared/suspendibleThreadSet.hpp"
 35 #include "gc/shared/tlab_globals.hpp"
 36 #include "gc/shenandoah/shenandoahAsserts.hpp"
 37 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
 38 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
 39 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
 40 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
 41 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
 42 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
 43 #include "gc/shenandoah/shenandoahControlThread.hpp"
 44 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
 45 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"

 46 #include "oops/compressedOops.inline.hpp"
 47 #include "oops/oop.inline.hpp"
 48 #include "runtime/atomic.hpp"
 49 #include "runtime/javaThread.hpp"
 50 #include "runtime/prefetch.inline.hpp"

 51 #include "utilities/copy.hpp"
 52 #include "utilities/globalDefinitions.hpp"
 53 
 54 inline ShenandoahHeap* ShenandoahHeap::heap() {
 55   return named_heap<ShenandoahHeap>(CollectedHeap::Shenandoah);
 56 }
 57 
 58 inline ShenandoahHeapRegion* ShenandoahRegionIterator::next() {
 59   size_t new_index = Atomic::add(&_index, (size_t) 1, memory_order_relaxed);
 60   // get_region() provides the bounds-check and returns null on OOB.
 61   return _heap->get_region(new_index - 1);
 62 }
 63 
 64 inline bool ShenandoahHeap::has_forwarded_objects() const {
 65   return _gc_state.is_set(HAS_FORWARDED);
 66 }
 67 
 68 inline WorkerThreads* ShenandoahHeap::workers() const {
 69   return _workers;
 70 }
 71 
 72 inline WorkerThreads* ShenandoahHeap::safepoint_workers() {
 73   return _safepoint_workers;
 74 }
 75 












 76 inline size_t ShenandoahHeap::heap_region_index_containing(const void* addr) const {
 77   uintptr_t region_start = ((uintptr_t) addr);
 78   uintptr_t index = (region_start - (uintptr_t) base()) >> ShenandoahHeapRegion::region_size_bytes_shift();
 79   assert(index < num_regions(), "Region index is in bounds: " PTR_FORMAT, p2i(addr));
 80   return index;
 81 }
 82 
 83 inline ShenandoahHeapRegion* const ShenandoahHeap::heap_region_containing(const void* addr) const {
 84   size_t index = heap_region_index_containing(addr);
 85   ShenandoahHeapRegion* const result = get_region(index);
 86   assert(addr >= result->bottom() && addr < result->end(), "Heap region contains the address: " PTR_FORMAT, p2i(addr));
 87   return result;
 88 }
 89 
 90 inline void ShenandoahHeap::enter_evacuation(Thread* t) {
 91   _oom_evac_handler.enter_evacuation(t);
 92 }
 93 
 94 inline void ShenandoahHeap::leave_evacuation(Thread* t) {
 95   _oom_evac_handler.leave_evacuation(t);
 96 }
 97 
 98 template <class T>
 99 inline void ShenandoahHeap::update_with_forwarded(T* p) {
100   T o = RawAccess<>::oop_load(p);
101   if (!CompressedOops::is_null(o)) {
102     oop obj = CompressedOops::decode_not_null(o);
103     if (in_collection_set(obj)) {

235 }
236 
237 inline void ShenandoahHeap::atomic_clear_oop(narrowOop* addr, narrowOop compare) {
238   assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr));
239   Atomic::cmpxchg(addr, compare, narrowOop(), memory_order_relaxed);
240 }
241 
242 inline bool ShenandoahHeap::cancelled_gc() const {
243   return _cancelled_gc.get() == CANCELLED;
244 }
245 
246 inline bool ShenandoahHeap::check_cancelled_gc_and_yield(bool sts_active) {
247   if (sts_active && !cancelled_gc()) {
248     if (SuspendibleThreadSet::should_yield()) {
249       SuspendibleThreadSet::yield();
250     }
251   }
252   return cancelled_gc();
253 }
254 
255 inline void ShenandoahHeap::clear_cancelled_gc() {
256   _cancelled_gc.set(CANCELLABLE);
257   _oom_evac_handler.clear();








258 }
259 
260 inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) {
261   assert(UseTLAB, "TLABs should be enabled");
262 
263   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
264   if (gclab == nullptr) {
265     assert(!thread->is_Java_thread() && !thread->is_Worker_thread(),
266            "Performance: thread should have GCLAB: %s", thread->name());
267     // No GCLABs in this thread, fallback to shared allocation
268     return nullptr;
269   }
270   HeapWord* obj = gclab->allocate(size);
271   if (obj != nullptr) {
272     return obj;
273   }
274   // Otherwise...
275   return allocate_from_gclab_slow(thread, size);
276 }
277 
278 inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
279   if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) {
280     // This thread went through the OOM during evac protocol and it is safe to return
281     // the forward pointer. It must not attempt to evacuate any more.
282     return ShenandoahBarrierSet::resolve_forwarded(p);






















283   }

284 
285   assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
















286 
287   size_t size = p->size();




288 
289   assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");




290 
291   bool alloc_from_gclab = true;
292   HeapWord* copy = nullptr;



293 
294 #ifdef ASSERT
295   if (ShenandoahOOMDuringEvacALot &&
296       (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
297         copy = nullptr;
298   } else {
299 #endif
300     if (UseTLAB) {
301       copy = allocate_from_gclab(thread, size);
302     }
303     if (copy == nullptr) {
304       ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size);
305       copy = allocate_memory(req);
306       alloc_from_gclab = false;
307     }
308 #ifdef ASSERT




309   }
310 #endif
311 
312   if (copy == nullptr) {
313     control_thread()->handle_alloc_failure_evac(size);

314 
315     _oom_evac_handler.handle_out_of_memory_during_evacuation();


316 
317     return ShenandoahBarrierSet::resolve_forwarded(p);
318   }

319 
320   // Copy the object:
321   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
322 
323   // Try to install the new forwarding pointer.
324   oop copy_val = cast_to_oop(copy);
325   ContinuationGCSupport::relativize_stack_chunk(copy_val);
326 
327   oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
328   if (result == copy_val) {
329     // Successfully evacuated. Our copy is now the public one!
330     shenandoah_assert_correct(nullptr, copy_val);
331     return copy_val;
332   }  else {
333     // Failed to evacuate. We need to deal with the object that is left behind. Since this
334     // new allocation is certainly after TAMS, it will be considered live in the next cycle.
335     // But if it happens to contain references to evacuated regions, those references would
336     // not get updated for this stale copy during this cycle, and we will crash while scanning
337     // it the next cycle.
338     //
339     // For GCLAB allocations, it is enough to rollback the allocation ptr. Either the next
340     // object will overwrite this stale copy, or the filler object on LAB retirement will
341     // do this. For non-GCLAB allocations, we have no way to retract the allocation, and
342     // have to explicitly overwrite the copy with the filler object. With that overwrite,
343     // we have to keep the fwdptr initialized and pointing to our (stale) copy.
344     if (alloc_from_gclab) {
345       ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
346     } else {
347       fill_with_object(copy, size);
348       shenandoah_assert_correct(nullptr, copy_val);
349     }
350     shenandoah_assert_correct(nullptr, result);
351     return result;
352   }
353 }
354 











355 inline bool ShenandoahHeap::requires_marking(const void* entry) const {
356   oop obj = cast_to_oop(entry);
357   return !_marking_context->is_marked_strong(obj);
358 }
359 
360 inline bool ShenandoahHeap::in_collection_set(oop p) const {
361   assert(collection_set() != nullptr, "Sanity");
362   return collection_set()->is_in(p);
363 }
364 
365 inline bool ShenandoahHeap::in_collection_set_loc(void* p) const {
366   assert(collection_set() != nullptr, "Sanity");
367   return collection_set()->is_in_loc(p);
368 }
369 

370 inline bool ShenandoahHeap::is_stable() const {
371   return _gc_state.is_clear();
372 }
373 
374 inline bool ShenandoahHeap::is_idle() const {
375   return _gc_state.is_unset(MARKING | EVACUATION | UPDATEREFS);
376 }
377 
378 inline bool ShenandoahHeap::is_concurrent_mark_in_progress() const {
379   return _gc_state.is_set(MARKING);
380 }
381 








382 inline bool ShenandoahHeap::is_evacuation_in_progress() const {
383   return _gc_state.is_set(EVACUATION);
384 }
385 
386 inline bool ShenandoahHeap::is_degenerated_gc_in_progress() const {
387   return _degenerated_gc_in_progress.is_set();
388 }
389 
390 inline bool ShenandoahHeap::is_full_gc_in_progress() const {
391   return _full_gc_in_progress.is_set();
392 }
393 
394 inline bool ShenandoahHeap::is_full_gc_move_in_progress() const {
395   return _full_gc_move_in_progress.is_set();
396 }
397 
398 inline bool ShenandoahHeap::is_update_refs_in_progress() const {
399   return _gc_state.is_set(UPDATEREFS);
400 }
401 

403   return is_full_gc_in_progress() || is_degenerated_gc_in_progress();
404 }
405 
406 inline bool ShenandoahHeap::is_concurrent_strong_root_in_progress() const {
407   return _concurrent_strong_root_in_progress.is_set();
408 }
409 
410 inline bool ShenandoahHeap::is_concurrent_weak_root_in_progress() const {
411   return _gc_state.is_set(WEAK_ROOTS);
412 }
413 
414 template<class T>
415 inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl) {
416   marked_object_iterate(region, cl, region->top());
417 }
418 
419 template<class T>
420 inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) {
421   assert(! region->is_humongous_continuation(), "no humongous continuation regions here");
422 
423   ShenandoahMarkingContext* const ctx = complete_marking_context();
424   assert(ctx->is_complete(), "sanity");
425 
426   HeapWord* tams = ctx->top_at_mark_start(region);
427 
428   size_t skip_bitmap_delta = 1;
429   HeapWord* start = region->bottom();
430   HeapWord* end = MIN2(tams, region->end());
431 
432   // Step 1. Scan below the TAMS based on bitmap data.
433   HeapWord* limit_bitmap = MIN2(limit, tams);
434 
435   // Try to scan the initial candidate. If the candidate is above the TAMS, it would
436   // fail the subsequent "< limit_bitmap" checks, and fall through to Step 2.
437   HeapWord* cb = ctx->get_next_marked_addr(start, end);
438 
439   intx dist = ShenandoahMarkScanPrefetch;
440   if (dist > 0) {
441     // Batched scan that prefetches the oop data, anticipating the access to
442     // either header, oop field, or forwarding pointer. Not that we cannot
443     // touch anything in oop, while it still being prefetched to get enough
444     // time for prefetch to work. This is why we try to scan the bitmap linearly,

527   void do_object(oop obj) {
528     obj->oop_iterate(_cl, _bounds);
529   }
530 };
531 
532 template<class T>
533 inline void ShenandoahHeap::marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* top) {
534   if (region->is_humongous()) {
535     HeapWord* bottom = region->bottom();
536     if (top > bottom) {
537       region = region->humongous_start_region();
538       ShenandoahObjectToOopBoundedClosure<T> objs(cl, bottom, top);
539       marked_object_iterate(region, &objs);
540     }
541   } else {
542     ShenandoahObjectToOopClosure<T> objs(cl);
543     marked_object_iterate(region, &objs, top);
544   }
545 }
546 
547 inline ShenandoahHeapRegion* const ShenandoahHeap::get_region(size_t region_idx) const {
548   if (region_idx < _num_regions) {
549     return _regions[region_idx];
550   } else {
551     return nullptr;
552   }
553 }
554 
555 inline void ShenandoahHeap::mark_complete_marking_context() {
556   _marking_context->mark_complete();
557 }
558 
559 inline void ShenandoahHeap::mark_incomplete_marking_context() {
560   _marking_context->mark_incomplete();
561 }
562 
563 inline ShenandoahMarkingContext* ShenandoahHeap::complete_marking_context() const {
564   assert (_marking_context->is_complete()," sanity");
565   return _marking_context;
566 }
567 
568 inline ShenandoahMarkingContext* ShenandoahHeap::marking_context() const {
569   return _marking_context;
570 }
571 
572 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP

  1 /*
  2  * Copyright (c) 2015, 2020, Red Hat, Inc. All rights reserved.
  3  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  *
 24  */
 25 
 26 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
 27 #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
 28 
 29 #include "gc/shenandoah/shenandoahHeap.hpp"
 30 
 31 #include "classfile/javaClasses.inline.hpp"
 32 #include "gc/shared/markBitMap.inline.hpp"
 33 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
 34 #include "gc/shared/continuationGCSupport.inline.hpp"
 35 #include "gc/shared/suspendibleThreadSet.hpp"
 36 #include "gc/shared/tlab_globals.hpp"
 37 #include "gc/shenandoah/shenandoahAsserts.hpp"
 38 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
 39 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
 40 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
 41 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
 42 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
 43 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
 44 #include "gc/shenandoah/shenandoahGeneration.hpp"
 45 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
 46 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 47 #include "gc/shenandoah/mode/shenandoahMode.hpp"
 48 #include "oops/compressedOops.inline.hpp"
 49 #include "oops/oop.inline.hpp"
 50 #include "runtime/atomic.hpp"
 51 #include "runtime/javaThread.hpp"
 52 #include "runtime/prefetch.inline.hpp"
 53 #include "runtime/objectMonitor.inline.hpp"
 54 #include "utilities/copy.hpp"
 55 #include "utilities/globalDefinitions.hpp"
 56 
 57 inline ShenandoahHeap* ShenandoahHeap::heap() {
 58   return named_heap<ShenandoahHeap>(CollectedHeap::Shenandoah);
 59 }
 60 
 61 inline ShenandoahHeapRegion* ShenandoahRegionIterator::next() {
 62   size_t new_index = Atomic::add(&_index, (size_t) 1, memory_order_relaxed);
 63   // get_region() provides the bounds-check and returns null on OOB.
 64   return _heap->get_region(new_index - 1);
 65 }
 66 
 67 inline bool ShenandoahHeap::has_forwarded_objects() const {
 68   return _gc_state.is_set(HAS_FORWARDED);
 69 }
 70 
 71 inline WorkerThreads* ShenandoahHeap::workers() const {
 72   return _workers;
 73 }
 74 
 75 inline WorkerThreads* ShenandoahHeap::safepoint_workers() {
 76   return _safepoint_workers;
 77 }
 78 
 79 inline void ShenandoahHeap::notify_gc_progress() {
 80   Atomic::store(&_gc_no_progress_count, (size_t) 0);
 81 
 82 }
 83 inline void ShenandoahHeap::notify_gc_no_progress() {
 84   Atomic::inc(&_gc_no_progress_count);
 85 }
 86 
 87 inline size_t ShenandoahHeap::get_gc_no_progress_count() const {
 88   return Atomic::load(&_gc_no_progress_count);
 89 }
 90 
 91 inline size_t ShenandoahHeap::heap_region_index_containing(const void* addr) const {
 92   uintptr_t region_start = ((uintptr_t) addr);
 93   uintptr_t index = (region_start - (uintptr_t) base()) >> ShenandoahHeapRegion::region_size_bytes_shift();
 94   assert(index < num_regions(), "Region index is in bounds: " PTR_FORMAT, p2i(addr));
 95   return index;
 96 }
 97 
 98 inline ShenandoahHeapRegion* ShenandoahHeap::heap_region_containing(const void* addr) const {
 99   size_t index = heap_region_index_containing(addr);
100   ShenandoahHeapRegion* const result = get_region(index);
101   assert(addr >= result->bottom() && addr < result->end(), "Heap region contains the address: " PTR_FORMAT, p2i(addr));
102   return result;
103 }
104 
105 inline void ShenandoahHeap::enter_evacuation(Thread* t) {
106   _oom_evac_handler.enter_evacuation(t);
107 }
108 
109 inline void ShenandoahHeap::leave_evacuation(Thread* t) {
110   _oom_evac_handler.leave_evacuation(t);
111 }
112 
113 template <class T>
114 inline void ShenandoahHeap::update_with_forwarded(T* p) {
115   T o = RawAccess<>::oop_load(p);
116   if (!CompressedOops::is_null(o)) {
117     oop obj = CompressedOops::decode_not_null(o);
118     if (in_collection_set(obj)) {

250 }
251 
252 inline void ShenandoahHeap::atomic_clear_oop(narrowOop* addr, narrowOop compare) {
253   assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr));
254   Atomic::cmpxchg(addr, compare, narrowOop(), memory_order_relaxed);
255 }
256 
257 inline bool ShenandoahHeap::cancelled_gc() const {
258   return _cancelled_gc.get() == CANCELLED;
259 }
260 
261 inline bool ShenandoahHeap::check_cancelled_gc_and_yield(bool sts_active) {
262   if (sts_active && !cancelled_gc()) {
263     if (SuspendibleThreadSet::should_yield()) {
264       SuspendibleThreadSet::yield();
265     }
266   }
267   return cancelled_gc();
268 }
269 
270 inline void ShenandoahHeap::clear_cancelled_gc(bool clear_oom_handler) {
271   _cancelled_gc.set(CANCELLABLE);
272   if (_cancel_requested_time > 0) {
273     double cancel_time = os::elapsedTime() - _cancel_requested_time;
274     log_info(gc)("GC cancellation took %.3fs", cancel_time);
275     _cancel_requested_time = 0;
276   }
277 
278   if (clear_oom_handler) {
279     _oom_evac_handler.clear();
280   }
281 }
282 
283 inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) {
284   assert(UseTLAB, "TLABs should be enabled");
285 
286   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
287   if (gclab == nullptr) {
288     assert(!thread->is_Java_thread() && !thread->is_Worker_thread(),
289            "Performance: thread should have GCLAB: %s", thread->name());
290     // No GCLABs in this thread, fallback to shared allocation
291     return nullptr;
292   }
293   HeapWord* obj = gclab->allocate(size);
294   if (obj != nullptr) {
295     return obj;
296   }

297   return allocate_from_gclab_slow(thread, size);
298 }
299 
300 void ShenandoahHeap::increase_object_age(oop obj, uint additional_age) {
301   // This operates on new copy of an object. This means that the object's mark-word
302   // is thread-local and therefore safe to access. However, when the mark is
303   // displaced (i.e. stack-locked or monitor-locked), then it must be considered
304   // a shared memory location. It can be accessed by other threads.
305   // In particular, a competing evacuating thread can succeed to install its copy
306   // as the forwardee and continue to unlock the object, at which point 'our'
307   // write to the foreign stack-location would potentially over-write random
308   // information on that stack. Writing to a monitor is less problematic,
309   // but still not safe: while the ObjectMonitor would not randomly disappear,
310   // the other thread would also write to the same displaced header location,
311   // possibly leading to increase the age twice.
312   // For all these reasons, we take the conservative approach and not attempt
313   // to increase the age when the header is displaced.
314   markWord w = obj->mark();
315   // The mark-word has been copied from the original object. It can not be
316   // inflating, because inflation can not be interrupted by a safepoint,
317   // and after a safepoint, a Java thread would first have to successfully
318   // evacuate the object before it could inflate the monitor.
319   assert(!w.is_being_inflated() || LockingMode == LM_LIGHTWEIGHT, "must not inflate monitor before evacuation of object succeeds");
320   // It is possible that we have copied the object after another thread has
321   // already successfully completed evacuation. While harmless (we would never
322   // publish our copy), don't even attempt to modify the age when that
323   // happens.
324   if (!w.has_displaced_mark_helper() && !w.is_marked()) {
325     w = w.set_age(MIN2(markWord::max_age, w.age() + additional_age));
326     obj->set_mark(w);
327   }
328 }
329 
330 // Return the object's age, or a sentinel value when the age can't
331 // necessarily be determined because of concurrent locking by the
332 // mutator
333 uint ShenandoahHeap::get_object_age(oop obj) {
334   // This is impossible to do unless we "freeze" ABA-type oscillations
335   // With Lilliput, we can do this more easily.
336   markWord w = obj->mark();
337   assert(!w.is_marked(), "must not be forwarded");
338   if (w.has_monitor()) {
339     w = w.monitor()->header();
340   } else if (w.is_being_inflated() || w.has_displaced_mark_helper()) {
341     // Informs caller that we aren't able to determine the age
342     return markWord::max_age + 1; // sentinel
343   }
344   assert(w.age() <= markWord::max_age, "Impossible!");
345   return w.age();
346 }
347 
348 bool ShenandoahHeap::is_in(const void* p) const {
349   HeapWord* heap_base = (HeapWord*) base();
350   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
351   return p >= heap_base && p < last_region_end;
352 }
353 
354 inline bool ShenandoahHeap::is_in_active_generation(oop obj) const {
355   if (!mode()->is_generational()) {
356     // everything is the same single generation
357     return true;
358   }
359 
360   if (active_generation() == nullptr) {
361     // no collection is happening, only expect this to be called
362     // when concurrent processing is active, but that could change
363     return false;
364   }
365 
366   assert(is_in(obj), "only check if is in active generation for objects (" PTR_FORMAT ") in heap", p2i(obj));
367   assert((active_generation() == (ShenandoahGeneration*) old_generation()) ||
368          (active_generation() == (ShenandoahGeneration*) young_generation()) ||
369          (active_generation() == global_generation()), "Active generation must be old, young, or global");
370 
371   size_t index = heap_region_containing(obj)->index();
372   switch (_affiliations[index]) {
373   case ShenandoahAffiliation::FREE:
374     // Free regions are in Old, Young, Global
375     return true;
376   case ShenandoahAffiliation::YOUNG_GENERATION:
377     // Young regions are in young_generation and global_generation, not in old_generation
378     return (active_generation() != (ShenandoahGeneration*) old_generation());
379   case ShenandoahAffiliation::OLD_GENERATION:
380     // Old regions are in old_generation and global_generation, not in young_generation
381     return (active_generation() != (ShenandoahGeneration*) young_generation());
382   default:
383     assert(false, "Bad affiliation (%d) for region " SIZE_FORMAT, _affiliations[index], index);
384     return false;
385   }
386 }
387 
388 inline bool ShenandoahHeap::is_in_young(const void* p) const {
389   return is_in(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahAffiliation::YOUNG_GENERATION);
390 }
391 
392 inline bool ShenandoahHeap::is_in_old(const void* p) const {
393   return is_in(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahAffiliation::OLD_GENERATION);
394 }
395 
396 inline bool ShenandoahHeap::is_old(oop obj) const {
397   return active_generation()->is_young() && is_in_old(obj);
398 }
399 
400 inline ShenandoahAffiliation ShenandoahHeap::region_affiliation(const ShenandoahHeapRegion *r) {
401   return (ShenandoahAffiliation) _affiliations[r->index()];
402 }
403 
404 inline void ShenandoahHeap::assert_lock_for_affiliation(ShenandoahAffiliation orig_affiliation,
405                                                         ShenandoahAffiliation new_affiliation) {
406   // A lock is required when changing from FREE to NON-FREE.  Though it may be possible to elide the lock when
407   // transitioning from in-use to FREE, the current implementation uses a lock for this transition.  A lock is
408   // not required to change from YOUNG to OLD (i.e. when promoting humongous region).
409   //
410   //         new_affiliation is:     FREE   YOUNG   OLD
411   //  orig_affiliation is:  FREE      X       L      L
412   //                       YOUNG      L       X
413   //                         OLD      L       X      X
414   //  X means state transition won't happen (so don't care)
415   //  L means lock should be held
416   //  Blank means no lock required because affiliation visibility will not be required until subsequent safepoint
417   //
418   // Note: during full GC, all transitions between states are possible.  During Full GC, we should be in a safepoint.
419 
420   if ((orig_affiliation == ShenandoahAffiliation::FREE) || (new_affiliation == ShenandoahAffiliation::FREE)) {
421     shenandoah_assert_heaplocked_or_fullgc_safepoint();










422   }
423 }
424 
425 inline void ShenandoahHeap::set_affiliation(ShenandoahHeapRegion* r, ShenandoahAffiliation new_affiliation) {
426 #ifdef ASSERT
427   assert_lock_for_affiliation(region_affiliation(r), new_affiliation);
428 #endif
429   _affiliations[r->index()] = (uint8_t) new_affiliation;
430 }
431 
432 inline ShenandoahAffiliation ShenandoahHeap::region_affiliation(size_t index) {
433   return (ShenandoahAffiliation) _affiliations[index];
434 }
435 
436 inline bool ShenandoahHeap::requires_marking(const void* entry) const {
437   oop obj = cast_to_oop(entry);
438   return !_marking_context->is_marked_strong(obj);
439 }
440 
441 inline bool ShenandoahHeap::in_collection_set(oop p) const {
442   assert(collection_set() != nullptr, "Sanity");
443   return collection_set()->is_in(p);
444 }
445 
446 inline bool ShenandoahHeap::in_collection_set_loc(void* p) const {
447   assert(collection_set() != nullptr, "Sanity");
448   return collection_set()->is_in_loc(p);
449 }
450 
451 
452 inline bool ShenandoahHeap::is_stable() const {
453   return _gc_state.is_clear();
454 }
455 
456 inline bool ShenandoahHeap::is_idle() const {
457   return _gc_state.is_unset(MARKING | EVACUATION | UPDATEREFS);
458 }
459 
460 inline bool ShenandoahHeap::is_concurrent_mark_in_progress() const {
461   return _gc_state.is_set(MARKING);
462 }
463 
464 inline bool ShenandoahHeap::is_concurrent_young_mark_in_progress() const {
465   return _gc_state.is_set(YOUNG_MARKING);
466 }
467 
468 inline bool ShenandoahHeap::is_concurrent_old_mark_in_progress() const {
469   return _gc_state.is_set(OLD_MARKING);
470 }
471 
472 inline bool ShenandoahHeap::is_evacuation_in_progress() const {
473   return _gc_state.is_set(EVACUATION);
474 }
475 
476 inline bool ShenandoahHeap::is_degenerated_gc_in_progress() const {
477   return _degenerated_gc_in_progress.is_set();
478 }
479 
480 inline bool ShenandoahHeap::is_full_gc_in_progress() const {
481   return _full_gc_in_progress.is_set();
482 }
483 
484 inline bool ShenandoahHeap::is_full_gc_move_in_progress() const {
485   return _full_gc_move_in_progress.is_set();
486 }
487 
488 inline bool ShenandoahHeap::is_update_refs_in_progress() const {
489   return _gc_state.is_set(UPDATEREFS);
490 }
491 

493   return is_full_gc_in_progress() || is_degenerated_gc_in_progress();
494 }
495 
496 inline bool ShenandoahHeap::is_concurrent_strong_root_in_progress() const {
497   return _concurrent_strong_root_in_progress.is_set();
498 }
499 
500 inline bool ShenandoahHeap::is_concurrent_weak_root_in_progress() const {
501   return _gc_state.is_set(WEAK_ROOTS);
502 }
503 
504 template<class T>
505 inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl) {
506   marked_object_iterate(region, cl, region->top());
507 }
508 
509 template<class T>
510 inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) {
511   assert(! region->is_humongous_continuation(), "no humongous continuation regions here");
512 
513   ShenandoahMarkingContext* const ctx = marking_context();

514 
515   HeapWord* tams = ctx->top_at_mark_start(region);
516 
517   size_t skip_bitmap_delta = 1;
518   HeapWord* start = region->bottom();
519   HeapWord* end = MIN2(tams, region->end());
520 
521   // Step 1. Scan below the TAMS based on bitmap data.
522   HeapWord* limit_bitmap = MIN2(limit, tams);
523 
524   // Try to scan the initial candidate. If the candidate is above the TAMS, it would
525   // fail the subsequent "< limit_bitmap" checks, and fall through to Step 2.
526   HeapWord* cb = ctx->get_next_marked_addr(start, end);
527 
528   intx dist = ShenandoahMarkScanPrefetch;
529   if (dist > 0) {
530     // Batched scan that prefetches the oop data, anticipating the access to
531     // either header, oop field, or forwarding pointer. Not that we cannot
532     // touch anything in oop, while it still being prefetched to get enough
533     // time for prefetch to work. This is why we try to scan the bitmap linearly,

616   void do_object(oop obj) {
617     obj->oop_iterate(_cl, _bounds);
618   }
619 };
620 
621 template<class T>
622 inline void ShenandoahHeap::marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* top) {
623   if (region->is_humongous()) {
624     HeapWord* bottom = region->bottom();
625     if (top > bottom) {
626       region = region->humongous_start_region();
627       ShenandoahObjectToOopBoundedClosure<T> objs(cl, bottom, top);
628       marked_object_iterate(region, &objs);
629     }
630   } else {
631     ShenandoahObjectToOopClosure<T> objs(cl);
632     marked_object_iterate(region, &objs, top);
633   }
634 }
635 
636 inline ShenandoahHeapRegion* ShenandoahHeap::get_region(size_t region_idx) const {
637   if (region_idx < _num_regions) {
638     return _regions[region_idx];
639   } else {
640     return nullptr;
641   }
642 }
643 








644 inline ShenandoahMarkingContext* ShenandoahHeap::complete_marking_context() const {
645   assert (_marking_context->is_complete()," sanity");
646   return _marking_context;
647 }
648 
649 inline ShenandoahMarkingContext* ShenandoahHeap::marking_context() const {
650   return _marking_context;
651 }
652 
653 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
< prev index next >