src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp

@@ -54,20 +54,53 @@
    } else {
      return p;
    }
  }
  
- inline oop ShenandoahBarrierSet::resolve_forwarded_not_null_mutator(oop p) {
-   return ShenandoahForwarding::get_forwardee_mutator(p);
- }
- 
- template <class T>
+ template <DecoratorSet decorators, class T>
  inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, T* load_addr) {
-   assert(ShenandoahLoadRefBarrier, "should be enabled");
-   shenandoah_assert_in_cset(load_addr, obj);
+   assert(ShenandoahLoadRefBarrier, "Should be enabled");
+ 
+   bool on_weak    = HasDecorator<decorators, ON_WEAK_OOP_REF>::value;
+   bool on_phantom = HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value;
+ 
+   // Handle nulls. Strong loads filtered nulls with cset checks.
+   // Weak/phantom loads need to check for nulls here.
+   if (on_weak || on_phantom) {
+     if (obj == nullptr) {
+       return nullptr;
+     }
+   } else {
+     assert(obj != nullptr, "Should have been filtered before");
+   }
+ 
+   // Prevent resurrection of unreachable phantom (i.e. weak-native) references.
+   if (on_phantom &&
+       _heap->is_concurrent_weak_root_in_progress() &&
+       _heap->is_in_active_generation(obj) &&
+       !_heap->marking_context()->is_marked(obj)) {
+     return nullptr;
+   }
+ 
+   // Prevent resurrection of unreachable weak references.
+   if (on_weak &&
+       _heap->is_concurrent_weak_root_in_progress() &&
+       _heap->is_in_active_generation(obj) &&
+       !_heap->marking_context()->is_marked_strong(obj)) {
+     return nullptr;
+   }
+ 
+   // Weak/phantom loads need additional cset check.
+   if (on_phantom || on_weak) {
+     if (!_heap->has_forwarded_objects() || !_heap->in_collection_set(obj)) {
+       return obj;
+     }
+   } else {
+     shenandoah_assert_in_cset(load_addr, obj);
+   }
  
-   oop fwd = resolve_forwarded_not_null_mutator(obj);
+   oop fwd = ShenandoahForwarding::get_forwardee_mutator(obj);
    if (obj == fwd) {
      assert(_heap->is_evacuation_in_progress(), "evac should be in progress");
      Thread* const t = Thread::current();
      ShenandoahEvacOOMScope scope(t);
      fwd = _heap->evacuate_object(obj, t);

@@ -370,16 +403,80 @@
    }
    return result;
  }
  
  // Clone barrier support
+ template <bool EVAC>
+ class ShenandoahUpdateEvacForCloneOopClosure : public BasicOopIterateClosure {
+ private:
+   ShenandoahHeap* const _heap;
+   const ShenandoahCollectionSet* const _cset;
+   Thread* const _thread;
+ 
+   template <class T>
+   inline void do_oop_work(T* p) {
+     T o = RawAccess<>::oop_load(p);
+     if (!CompressedOops::is_null(o)) {
+       oop obj = CompressedOops::decode_not_null(o);
+       if (_cset->is_in(obj)) {
+         oop fwd = ShenandoahForwarding::get_forwardee(obj);
+         if (EVAC && obj == fwd) {
+           fwd = _heap->evacuate_object(obj, _thread);
+         }
+         shenandoah_assert_forwarded_except(p, obj, _heap->cancelled_gc());
+         ShenandoahHeap::atomic_update_oop(fwd, p, o);
+         obj = fwd;
+       }
+     }
+   }
+ 
+ public:
+   ShenandoahUpdateEvacForCloneOopClosure() :
+           _heap(ShenandoahHeap::heap()),
+           _cset(_heap->collection_set()),
+           _thread(Thread::current()) {}
+ 
+   virtual void do_oop(oop* p)       { do_oop_work(p); }
+   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+ };
+ 
+ void ShenandoahBarrierSet::clone_evacuation(oop obj) {
+   assert(_heap->is_evacuation_in_progress(), "only during evacuation");
+   if (need_bulk_update(cast_from_oop<HeapWord*>(obj))) {
+     ShenandoahEvacOOMScope oom_evac_scope;
+     ShenandoahUpdateEvacForCloneOopClosure<true> cl;
+     obj->oop_iterate(&cl);
+   }
+ }
+ 
+ void ShenandoahBarrierSet::clone_update(oop obj) {
+   assert(_heap->is_update_refs_in_progress(), "only during update-refs");
+   if (need_bulk_update(cast_from_oop<HeapWord*>(obj))) {
+     ShenandoahUpdateEvacForCloneOopClosure<false> cl;
+     obj->oop_iterate(&cl);
+   }
+ }
+ 
  template <DecoratorSet decorators, typename BarrierSetT>
- void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
-   if (ShenandoahCloneBarrier) {
-     ShenandoahBarrierSet::barrier_set()->clone_barrier_runtime(src);
+ void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t count) {
+   // Hot code path, called from compiler/runtime. Make sure fast path is fast.
+ 
+   // Fix up src before doing the copy, if needed.
+   const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
+   if (gc_state > 0 && ShenandoahCloneBarrier) {
+     ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
+     if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
+       bs->clone_evacuation(src);
+     } else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
+       bs->clone_update(src);
+     }
    }
-   Raw::clone(src, dst, size);
+ 
+   Raw::clone(src, dst, count);
+ 
+   // Safety: clone destination must be in young, otherwise we need card barriers.
+   shenandoah_assert_in_young_if(nullptr, dst, ShenandoahCardBarrier);
  }
  
  template <DecoratorSet decorators, typename BarrierSetT>
  template <typename T>
  OopCopyResult ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,