< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp

Print this page

150       !_heap->marking_context()->is_marked_strong(obj)) {
151     return nullptr;
152   }
153 
154   // Allow runtime to see unreachable objects that are visited during concurrent class-unloading.
155   if ((decorators & AS_NO_KEEPALIVE) != 0 &&
156       _heap->is_concurrent_weak_root_in_progress() &&
157       !_heap->marking_context()->is_marked(obj)) {
158     return obj;
159   }
160 
161   oop fwd = load_reference_barrier(obj);
162   if (load_addr != nullptr && fwd != obj) {
163     // Since we are here and we know the load address, update the reference.
164     ShenandoahHeap::atomic_update_oop(fwd, load_addr, obj);
165   }
166 
167   return fwd;
168 }
169 
170 inline void ShenandoahBarrierSet::enqueue(oop obj) {
171   assert(obj != nullptr, "checked by caller");

172   assert(_satb_mark_queue_set.is_active(), "only get here when SATB active");
173 
174   // Filter marked objects before hitting the SATB queues. The same predicate would
175   // be used by SATBMQ::filter to eliminate already marked objects downstream, but
176   // filtering here helps to avoid wasteful SATB queueing work to begin with.
177   if (!_heap->requires_marking(obj)) return;
178 
179   SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(Thread::current());
180   _satb_mark_queue_set.enqueue_known_active(queue, obj);
181 }
182 
183 template <DecoratorSet decorators, typename T>
184 inline void ShenandoahBarrierSet::satb_barrier(T *field) {
185   // Uninitialized and no-keepalive stores do not need barrier.
186   if (HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value ||
187       HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
188     return;
189   }
190 
191   // Stores to weak/phantom require no barrier. The original references would
192   // have been enqueued in the SATB buffer by the load barrier if they were needed.
193   if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value ||
194       HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value) {
195     return;
196   }
197 

385   if (ShenandoahCardBarrier) {
386     bs->write_ref_field_post<decorators>(addr);
387   }
388   return result;
389 }
390 
391 template <DecoratorSet decorators, typename BarrierSetT>
392 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) {
393   assert((decorators & AS_NO_KEEPALIVE) == 0, "must be absent");
394   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
395   DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
396   auto addr = AccessInternal::oop_field_addr<decorators>(base, offset);
397   oop result = bs->oop_xchg(resolved_decorators, addr, new_value);
398   if (ShenandoahCardBarrier) {
399     bs->write_ref_field_post<decorators>(addr);
400   }
401   return result;
402 }
403 
404 // Clone barrier support


















































405 template <DecoratorSet decorators, typename BarrierSetT>
406 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
407   if (ShenandoahCloneBarrier) {
408     ShenandoahBarrierSet::barrier_set()->clone_barrier_runtime(src);









409   }
410   Raw::clone(src, dst, size);





411 }
412 
413 template <DecoratorSet decorators, typename BarrierSetT>
414 template <typename T>
415 OopCopyResult ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
416                                                                                                   arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
417                                                                                                   size_t length) {
418   T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
419   T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
420 
421   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
422   bs->arraycopy_barrier(src, dst, length);
423   OopCopyResult result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
424   if (ShenandoahCardBarrier) {
425     bs->write_ref_array((HeapWord*) dst, length);
426   }
427   return result;
428 }
429 
430 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>

150       !_heap->marking_context()->is_marked_strong(obj)) {
151     return nullptr;
152   }
153 
154   // Allow runtime to see unreachable objects that are visited during concurrent class-unloading.
155   if ((decorators & AS_NO_KEEPALIVE) != 0 &&
156       _heap->is_concurrent_weak_root_in_progress() &&
157       !_heap->marking_context()->is_marked(obj)) {
158     return obj;
159   }
160 
161   oop fwd = load_reference_barrier(obj);
162   if (load_addr != nullptr && fwd != obj) {
163     // Since we are here and we know the load address, update the reference.
164     ShenandoahHeap::atomic_update_oop(fwd, load_addr, obj);
165   }
166 
167   return fwd;
168 }
169 
170 inline void ShenandoahBarrierSet::enqueue(oop obj, bool filter) {
171   assert(obj != nullptr, "checked by caller");
172   shenandoah_assert_correct(nullptr, obj);
173   assert(_satb_mark_queue_set.is_active(), "only get here when SATB active");
174 
175   // Filter marked objects before hitting the SATB queues. The same predicate would
176   // be used by SATBMQ::filter to eliminate already marked objects downstream, but
177   // filtering here helps to avoid wasteful SATB queueing work to begin with.
178   if (filter && !_heap->requires_marking(obj)) return;
179 
180   SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(Thread::current());
181   _satb_mark_queue_set.enqueue_known_active(queue, obj);
182 }
183 
184 template <DecoratorSet decorators, typename T>
185 inline void ShenandoahBarrierSet::satb_barrier(T *field) {
186   // Uninitialized and no-keepalive stores do not need barrier.
187   if (HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value ||
188       HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
189     return;
190   }
191 
192   // Stores to weak/phantom require no barrier. The original references would
193   // have been enqueued in the SATB buffer by the load barrier if they were needed.
194   if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value ||
195       HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value) {
196     return;
197   }
198 

386   if (ShenandoahCardBarrier) {
387     bs->write_ref_field_post<decorators>(addr);
388   }
389   return result;
390 }
391 
392 template <DecoratorSet decorators, typename BarrierSetT>
393 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) {
394   assert((decorators & AS_NO_KEEPALIVE) == 0, "must be absent");
395   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
396   DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
397   auto addr = AccessInternal::oop_field_addr<decorators>(base, offset);
398   oop result = bs->oop_xchg(resolved_decorators, addr, new_value);
399   if (ShenandoahCardBarrier) {
400     bs->write_ref_field_post<decorators>(addr);
401   }
402   return result;
403 }
404 
405 // Clone barrier support
406 template <bool EVAC>
407 class ShenandoahUpdateEvacForCloneOopClosure : public BasicOopIterateClosure {
408 private:
409   ShenandoahHeap* const _heap;
410   const ShenandoahCollectionSet* const _cset;
411   Thread* const _thread;
412 
413   template <class T>
414   inline void do_oop_work(T* p) {
415     T o = RawAccess<>::oop_load(p);
416     if (!CompressedOops::is_null(o)) {
417       oop obj = CompressedOops::decode_not_null(o);
418       if (_cset->is_in(obj)) {
419         oop fwd = ShenandoahForwarding::get_forwardee(obj);
420         if (EVAC && obj == fwd) {
421           fwd = _heap->evacuate_object(obj, _thread);
422         }
423         shenandoah_assert_forwarded_except(p, obj, _heap->cancelled_gc());
424         ShenandoahHeap::atomic_update_oop(fwd, p, o);
425         obj = fwd;
426       }
427     }
428   }
429 
430 public:
431   ShenandoahUpdateEvacForCloneOopClosure() :
432           _heap(ShenandoahHeap::heap()),
433           _cset(_heap->collection_set()),
434           _thread(Thread::current()) {}
435 
436   virtual void do_oop(oop* p)       { do_oop_work(p); }
437   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
438 };
439 
440 void ShenandoahBarrierSet::clone_evacuation(oop obj) {
441   assert(_heap->is_evacuation_in_progress(), "only during evacuation");
442   if (need_bulk_update(cast_from_oop<HeapWord*>(obj))) {
443     ShenandoahUpdateEvacForCloneOopClosure<true> cl;
444     obj->oop_iterate(&cl);
445   }
446 }
447 
448 void ShenandoahBarrierSet::clone_update(oop obj) {
449   assert(_heap->is_update_refs_in_progress(), "only during update-refs");
450   if (need_bulk_update(cast_from_oop<HeapWord*>(obj))) {
451     ShenandoahUpdateEvacForCloneOopClosure<false> cl;
452     obj->oop_iterate(&cl);
453   }
454 }
455 
456 template <DecoratorSet decorators, typename BarrierSetT>
457 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t count) {
458   // Hot code path, called from compiler/runtime. Make sure fast path is fast.
459 
460   // Fix up src before doing the copy, if needed.
461   const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
462   if (gc_state != 0 && ShenandoahCloneBarrier) {
463     ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
464     if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
465       bs->clone_evacuation(src);
466     } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
467       bs->clone_update(src);
468     }
469   }
470 
471   Raw::clone(src, dst, count);
472 
473   // Current allocator never allocates in old, so clone destination is guaranteed to be in young.
474   // Otherwise we need card barriers.
475   shenandoah_assert_in_young_if(nullptr, dst, ShenandoahCardBarrier);
476 }
477 
478 template <DecoratorSet decorators, typename BarrierSetT>
479 template <typename T>
480 OopCopyResult ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
481                                                                                                   arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
482                                                                                                   size_t length) {
483   T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
484   T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
485 
486   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
487   bs->arraycopy_barrier(src, dst, length);
488   OopCopyResult result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
489   if (ShenandoahCardBarrier) {
490     bs->write_ref_array((HeapWord*) dst, length);
491   }
492   return result;
493 }
494 
495 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>
< prev index next >