1 /*
   2  * Copyright (c) 2015, 2018, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP
  25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP
  26 
  27 #include "gc/shared/barrierSet.hpp"
  28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  29 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
  30 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
  31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  33 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
  35 #include "memory/iterator.inline.hpp"
  36 #include "oops/oop.inline.hpp"
  37 
  38 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) {
  39   return ShenandoahForwarding::get_forwardee(p);
  40 }
  41 
  42 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {
  43   if (((HeapWord*) p) != NULL) {
  44     return resolve_forwarded_not_null(p);
  45   } else {
  46     return p;
  47   }
  48 }
  49 
  50 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null_mutator(oop p) {
  51   return ShenandoahForwarding::get_forwardee_mutator(p);
  52 }
  53 
  54 template <class T>
  55 inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, T* load_addr) {
  56   assert(ShenandoahLoadRefBarrier, "should be enabled");
  57   shenandoah_assert_in_cset(load_addr, obj);
  58 
  59   oop fwd = resolve_forwarded_not_null_mutator(obj);
  60   if (obj == fwd) {
  61     assert(_heap->is_evacuation_in_progress(),
  62            "evac should be in progress");
  63     ShenandoahEvacOOMScope scope;
  64     fwd = _heap->evacuate_object(obj, Thread::current());
  65   }
  66 
  67   if (load_addr != NULL && fwd != obj) {
  68     // Since we are here and we know the load address, update the reference.
  69     ShenandoahHeap::cas_oop(fwd, load_addr, obj);
  70   }
  71 
  72   return fwd;
  73 }
  74 
  75 inline void ShenandoahBarrierSet::enqueue(oop obj) {
  76   assert(_satb_mark_queue_set.is_active(), "only get here when SATB active");
  77 
  78   // Filter marked objects before hitting the SATB queues. The same predicate would
  79   // be used by SATBMQ::filter to eliminate already marked objects downstream, but
  80   // filtering here helps to avoid wasteful SATB queueing work to begin with.
  81   if (!_heap->requires_marking(obj)) return;
  82 
  83   Thread* thr = Thread::current();
  84   if (thr->is_Java_thread()) {
  85     ShenandoahThreadLocalData::satb_mark_queue(thr).enqueue_known_active(obj);
  86   } else {
  87     MutexLockerEx x(Shared_SATB_Q_lock, Mutex::_no_safepoint_check_flag);
  88     _satb_mark_queue_set.shared_satb_queue()->enqueue_known_active(obj);
  89   }
  90 }
  91 
  92 template <DecoratorSet decorators, typename T>
  93 inline void ShenandoahBarrierSet::satb_barrier(T *field) {
  94   if (HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value ||
  95       HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
  96     return;
  97   }
  98   if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) {
  99     T heap_oop = RawAccess<>::oop_load(field);
 100     if (!CompressedOops::is_null(heap_oop)) {
 101       enqueue(CompressedOops::decode(heap_oop));
 102     }
 103   }
 104 }
 105 
 106 inline void ShenandoahBarrierSet::satb_enqueue(oop value) {
 107   assert(value != NULL, "checked before");
 108   if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) {
 109     enqueue(value);
 110   }
 111 }
 112 
 113 inline void ShenandoahBarrierSet::storeval_barrier(oop obj) {
 114   if (ShenandoahStoreValEnqueueBarrier && obj != NULL && _heap->is_concurrent_mark_in_progress()) {
 115     enqueue(obj);
 116   }
 117 }
 118 
 119 inline void ShenandoahBarrierSet::keep_alive_if_weak(DecoratorSet decorators, oop value) {
 120   assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known");
 121   const bool on_strong_oop_ref = (decorators & ON_STRONG_OOP_REF) != 0;
 122   const bool peek              = (decorators & AS_NO_KEEPALIVE) != 0;
 123   if (!peek && !on_strong_oop_ref) {
 124     satb_enqueue(value);
 125   }
 126 }
 127 
 128 template <DecoratorSet decorators>
 129 inline void ShenandoahBarrierSet::keep_alive_if_weak(oop value) {
 130   assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known");
 131   if (!HasDecorator<decorators, ON_STRONG_OOP_REF>::value &&
 132       !HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
 133     satb_enqueue(value);
 134   }
 135 }
 136 
 137 template <DecoratorSet decorators, typename BarrierSetT>
 138 template <typename T>
 139 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) {
 140   oop value = Raw::oop_load_not_in_heap(addr);
 141   if (value != NULL) {
 142     ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set();
 143     value = bs->load_reference_barrier_not_null(value);
 144     if (value != NULL) {
 145       bs->keep_alive_if_weak<decorators>(value);
 146     }
 147   }
 148   return value;
 149 }
 150 
 151 template <DecoratorSet decorators, typename BarrierSetT>
 152 template <typename T>
 153 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) {
 154   oop value = Raw::oop_load_in_heap(addr);
 155   if (value != NULL) {
 156     ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set();
 157     value = bs->load_reference_barrier_not_null(value);
 158     bs->keep_alive_if_weak<decorators>(value);
 159   }
 160   return value;
 161 }
 162 
 163 template <DecoratorSet decorators, typename BarrierSetT>
 164 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
 165   oop value = Raw::oop_load_in_heap_at(base, offset);
 166   if (value != NULL) {
 167     ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set();
 168     value = bs->load_reference_barrier_not_null(value);
 169     bs->keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset),
 170                            value);
 171   }
 172   return value;
 173 }
 174 
 175 template <DecoratorSet decorators, typename BarrierSetT>
 176 template <typename T>
 177 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(T* addr, oop value) {
 178   shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress());
 179   ShenandoahBarrierSet* const bs = ShenandoahBarrierSet::barrier_set();
 180   bs->storeval_barrier(value);
 181   bs->satb_barrier<decorators>(addr);
 182   Raw::oop_store(addr, value);
 183 }
 184 
 185 template <DecoratorSet decorators, typename BarrierSetT>
 186 template <typename T>
 187 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(T* addr, oop value) {
 188   shenandoah_assert_not_in_cset_loc_except(addr, ShenandoahHeap::heap()->cancelled_gc());
 189   shenandoah_assert_not_forwarded_except  (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress());
 190   shenandoah_assert_not_in_cset_except    (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress());
 191 
 192   oop_store_not_in_heap(addr, value);
 193 }
 194 
 195 template <DecoratorSet decorators, typename BarrierSetT>
 196 inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
 197   oop_store_in_heap(AccessInternal::oop_field_addr<decorators>(base, offset), value);
 198 }
 199 
 200 template <DecoratorSet decorators, typename BarrierSetT>
 201 template <typename T>
 202 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(oop new_value, T* addr, oop compare_value) {
 203   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
 204   bs->storeval_barrier(new_value);
 205 
 206   oop res;
 207   oop expected = compare_value;
 208   do {
 209     compare_value = expected;
 210     res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value);
 211     expected = res;
 212   } while (compare_value != expected && resolve_forwarded(compare_value) == resolve_forwarded(expected));
 213 
 214   // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway,
 215   // because it must be the previous value.
 216   if (res != NULL) {
 217     res = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res);
 218     bs->satb_enqueue(res);
 219   }
 220   return res;
 221 }
 222 
 223 template <DecoratorSet decorators, typename BarrierSetT>
 224 template <typename T>
 225 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {
 226   return oop_atomic_cmpxchg_not_in_heap(new_value, addr, compare_value);
 227 }
 228 
 229 template <DecoratorSet decorators, typename BarrierSetT>
 230 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) {
 231   return oop_atomic_cmpxchg_in_heap(new_value, AccessInternal::oop_field_addr<decorators>(base, offset), compare_value);
 232 }
 233 
 234 template <DecoratorSet decorators, typename BarrierSetT>
 235 template <typename T>
 236 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(oop new_value, T* addr) {
 237   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
 238   bs->storeval_barrier(new_value);
 239 
 240   oop previous = Raw::oop_atomic_xchg(new_value, addr);
 241 
 242   // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway,
 243   // because it must be the previous value.
 244   if (previous != NULL) {
 245     previous = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous);
 246     bs->satb_enqueue(previous);
 247  }
 248  return previous;
 249 }
 250 
 251 template <DecoratorSet decorators, typename BarrierSetT>
 252 template <typename T>
 253 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) {
 254   return oop_atomic_xchg_in_heap_impl(new_value, addr);
 255 }
 256 
 257 template <DecoratorSet decorators, typename BarrierSetT>
 258 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) {
 259   return oop_atomic_xchg_in_heap_impl(new_value, AccessInternal::oop_field_addr<decorators>(base, offset));
 260 }
 261 
 262 // Clone barrier support
 263 template <DecoratorSet decorators, typename BarrierSetT>
 264 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
 265   if (ShenandoahCloneBarrier) {
 266     ShenandoahBarrierSet::barrier_set()->clone_barrier_runtime(src);
 267   }
 268   Raw::clone(src, dst, size);
 269 }
 270 
 271 template <DecoratorSet decorators, typename BarrierSetT>
 272 template <typename T>
 273 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 274                                                                                          arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 275                                                                                          size_t length) {
 276   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
 277   bs->arraycopy_barrier(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw),
 278                         arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw),
 279                         length);
 280   return Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
 281 }
 282 
 283 template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>
 284 void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
 285   assert(HAS_FWD == _heap->has_forwarded_objects(), "Forwarded object status is sane");
 286 
 287   Thread* thread = Thread::current();
 288   ShenandoahSATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
 289   ShenandoahMarkingContext* ctx = _heap->marking_context();
 290   const ShenandoahCollectionSet* const cset = _heap->collection_set();
 291   T* end = src + count;
 292   for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) {
 293     T o = RawAccess<>::oop_load(elem_ptr);
 294     if (!CompressedOops::is_null(o)) {
 295       oop obj = CompressedOops::decode_not_null(o);
 296       if (HAS_FWD && cset->is_in(obj)) {
 297         oop fwd = resolve_forwarded_not_null(obj);
 298         if (EVAC && obj == fwd) {
 299           fwd = _heap->evacuate_object(obj, thread);
 300         }
 301         assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded");
 302         oop witness = ShenandoahHeap::cas_oop(fwd, elem_ptr, o);
 303         obj = fwd;
 304       }
 305       if (ENQUEUE && !ctx->is_marked(obj)) {
 306         queue.enqueue_known_active(obj);
 307       }
 308     }
 309   }
 310 }
 311 
 312 template <class T>
 313 void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
 314   if (count == 0) {
 315     return;
 316   }
 317   int gc_state = _heap->gc_state();
 318   if ((gc_state & ShenandoahHeap::MARKING) != 0) {
 319     arraycopy_marking(src, dst, count);
 320   } else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
 321     arraycopy_evacuation(src, count);
 322   } else if ((gc_state & ShenandoahHeap::UPDATEREFS) != 0) {
 323     arraycopy_update(src, count);
 324   }
 325 }
 326 
 327 template <class T>
 328 void ShenandoahBarrierSet::arraycopy_marking(T* src, T* dst, size_t count) {
 329   assert(_heap->is_concurrent_mark_in_progress(), "only during marking");
 330   T* array = ShenandoahSATBBarrier ? dst : src;
 331   if (!_heap->marking_context()->allocated_after_mark_start(reinterpret_cast<HeapWord*>(array))) {
 332     arraycopy_work<T, false, false, true>(array, count);
 333   }
 334 }
 335 
 336 inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* ary) {
 337   return ary < _heap->heap_region_containing(ary)->get_update_watermark();
 338 }
 339 
 340 template <class T>
 341 void ShenandoahBarrierSet::arraycopy_evacuation(T* src, size_t count) {
 342   assert(_heap->is_evacuation_in_progress(), "only during evacuation");
 343   if (need_bulk_update(reinterpret_cast<HeapWord*>(src))) {
 344     ShenandoahEvacOOMScope oom_evac;
 345     arraycopy_work<T, true, true, false>(src, count);
 346   }
 347 }
 348 
 349 template <class T>
 350 void ShenandoahBarrierSet::arraycopy_update(T* src, size_t count) {
 351   assert(_heap->is_update_refs_in_progress(), "only during update-refs");
 352   if (need_bulk_update(reinterpret_cast<HeapWord*>(src))) {
 353     arraycopy_work<T, true, false, false>(src, count);
 354   }
 355 }
 356 
 357 #endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP