24
25 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
26 #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
27
28 #include "gc/shenandoah/shenandoahHeap.hpp"
29
30 #include "classfile/javaClasses.inline.hpp"
31 #include "gc/shared/markBitMap.inline.hpp"
32 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
33 #include "gc/shared/suspendibleThreadSet.hpp"
34 #include "gc/shared/tlab_globals.hpp"
35 #include "gc/shenandoah/shenandoahAsserts.hpp"
36 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
37 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
38 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
39 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
40 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
41 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
42 #include "gc/shenandoah/shenandoahControlThread.hpp"
43 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
44 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
45 #include "oops/compressedOops.inline.hpp"
46 #include "oops/oop.inline.hpp"
47 #include "runtime/atomic.hpp"
48 #include "runtime/prefetch.inline.hpp"
49 #include "runtime/thread.hpp"
50 #include "utilities/copy.hpp"
51 #include "utilities/globalDefinitions.hpp"
52
53 inline ShenandoahHeap* ShenandoahHeap::heap() {
54 return named_heap<ShenandoahHeap>(CollectedHeap::Shenandoah);
55 }
56
57 inline ShenandoahHeapRegion* ShenandoahRegionIterator::next() {
58 size_t new_index = Atomic::add(&_index, (size_t) 1, memory_order_relaxed);
59 // get_region() provides the bounds-check and returns NULL on OOB.
60 return _heap->get_region(new_index - 1);
61 }
62
63 inline bool ShenandoahHeap::has_forwarded_objects() const {
279 // No GCLABs in this thread, fallback to shared allocation
280 return NULL;
281 }
282 HeapWord* obj = gclab->allocate(size);
283 if (obj != NULL) {
284 return obj;
285 }
286 // Otherwise...
287 return allocate_from_gclab_slow(thread, size);
288 }
289
290 inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
291 if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) {
292 // This thread went through the OOM during evac protocol and it is safe to return
293 // the forward pointer. It must not attempt to evacuate any more.
294 return ShenandoahBarrierSet::resolve_forwarded(p);
295 }
296
297 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
298
299 size_t size = p->size();
300
301 assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
302
303 bool alloc_from_gclab = true;
304 HeapWord* copy = NULL;
305
306 #ifdef ASSERT
307 if (ShenandoahOOMDuringEvacALot &&
308 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
309 copy = NULL;
310 } else {
311 #endif
312 if (UseTLAB) {
313 copy = allocate_from_gclab(thread, size);
314 }
315 if (copy == NULL) {
316 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size);
317 copy = allocate_memory(req);
318 alloc_from_gclab = false;
319 }
496 assert(oopDesc::is_oop(obj), "sanity");
497 assert(ctx->is_marked(obj), "object expected to be marked");
498 cl->do_object(obj);
499 cb += skip_bitmap_delta;
500 if (cb < limit_bitmap) {
501 cb = ctx->get_next_marked_addr(cb, limit_bitmap);
502 }
503 }
504 }
505
506 // Step 2. Accurate size-based traversal, happens past the TAMS.
507 // This restarts the scan at TAMS, which makes sure we traverse all objects,
508 // regardless of what happened at Step 1.
509 HeapWord* cs = tams;
510 while (cs < limit) {
511 assert (cs >= tams, "only objects past TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams));
512 assert (cs < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit));
513 oop obj = cast_to_oop(cs);
514 assert(oopDesc::is_oop(obj), "sanity");
515 assert(ctx->is_marked(obj), "object expected to be marked");
516 size_t size = obj->size();
517 cl->do_object(obj);
518 cs += size;
519 }
520 }
521
522 template <class T>
523 class ShenandoahObjectToOopClosure : public ObjectClosure {
524 T* _cl;
525 public:
526 ShenandoahObjectToOopClosure(T* cl) : _cl(cl) {}
527
528 void do_object(oop obj) {
529 obj->oop_iterate(_cl);
530 }
531 };
532
533 template <class T>
534 class ShenandoahObjectToOopBoundedClosure : public ObjectClosure {
535 T* _cl;
536 MemRegion _bounds;
|
24
25 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
26 #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
27
28 #include "gc/shenandoah/shenandoahHeap.hpp"
29
30 #include "classfile/javaClasses.inline.hpp"
31 #include "gc/shared/markBitMap.inline.hpp"
32 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
33 #include "gc/shared/suspendibleThreadSet.hpp"
34 #include "gc/shared/tlab_globals.hpp"
35 #include "gc/shenandoah/shenandoahAsserts.hpp"
36 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
37 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
38 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
39 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
40 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
41 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
42 #include "gc/shenandoah/shenandoahControlThread.hpp"
43 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
44 #include "gc/shenandoah/shenandoahObjectUtils.inline.hpp"
45 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
46 #include "oops/compressedOops.inline.hpp"
47 #include "oops/oop.inline.hpp"
48 #include "runtime/atomic.hpp"
49 #include "runtime/prefetch.inline.hpp"
50 #include "runtime/thread.hpp"
51 #include "utilities/copy.hpp"
52 #include "utilities/globalDefinitions.hpp"
53
54 inline ShenandoahHeap* ShenandoahHeap::heap() {
55 return named_heap<ShenandoahHeap>(CollectedHeap::Shenandoah);
56 }
57
58 inline ShenandoahHeapRegion* ShenandoahRegionIterator::next() {
59 size_t new_index = Atomic::add(&_index, (size_t) 1, memory_order_relaxed);
60 // get_region() provides the bounds-check and returns NULL on OOB.
61 return _heap->get_region(new_index - 1);
62 }
63
64 inline bool ShenandoahHeap::has_forwarded_objects() const {
280 // No GCLABs in this thread, fallback to shared allocation
281 return NULL;
282 }
283 HeapWord* obj = gclab->allocate(size);
284 if (obj != NULL) {
285 return obj;
286 }
287 // Otherwise...
288 return allocate_from_gclab_slow(thread, size);
289 }
290
291 inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
292 if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) {
293 // This thread went through the OOM during evac protocol and it is safe to return
294 // the forward pointer. It must not attempt to evacuate any more.
295 return ShenandoahBarrierSet::resolve_forwarded(p);
296 }
297
298 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
299
300 size_t size = ShenandoahObjectUtils::size(p);
301
302 assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
303
304 bool alloc_from_gclab = true;
305 HeapWord* copy = NULL;
306
307 #ifdef ASSERT
308 if (ShenandoahOOMDuringEvacALot &&
309 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
310 copy = NULL;
311 } else {
312 #endif
313 if (UseTLAB) {
314 copy = allocate_from_gclab(thread, size);
315 }
316 if (copy == NULL) {
317 ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size);
318 copy = allocate_memory(req);
319 alloc_from_gclab = false;
320 }
497 assert(oopDesc::is_oop(obj), "sanity");
498 assert(ctx->is_marked(obj), "object expected to be marked");
499 cl->do_object(obj);
500 cb += skip_bitmap_delta;
501 if (cb < limit_bitmap) {
502 cb = ctx->get_next_marked_addr(cb, limit_bitmap);
503 }
504 }
505 }
506
507 // Step 2. Accurate size-based traversal, happens past the TAMS.
508 // This restarts the scan at TAMS, which makes sure we traverse all objects,
509 // regardless of what happened at Step 1.
510 HeapWord* cs = tams;
511 while (cs < limit) {
512 assert (cs >= tams, "only objects past TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams));
513 assert (cs < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit));
514 oop obj = cast_to_oop(cs);
515 assert(oopDesc::is_oop(obj), "sanity");
516 assert(ctx->is_marked(obj), "object expected to be marked");
517 size_t size = ShenandoahObjectUtils::size(obj);
518 cl->do_object(obj);
519 cs += size;
520 }
521 }
522
523 template <class T>
524 class ShenandoahObjectToOopClosure : public ObjectClosure {
525 T* _cl;
526 public:
527 ShenandoahObjectToOopClosure(T* cl) : _cl(cl) {}
528
529 void do_object(oop obj) {
530 obj->oop_iterate(_cl);
531 }
532 };
533
534 template <class T>
535 class ShenandoahObjectToOopBoundedClosure : public ObjectClosure {
536 T* _cl;
537 MemRegion _bounds;
|